First commit

setup/e2mgr/E2Manager/services/rmrreceiver/rmr_receiver.go

@@ -0,0 +1,58 @@
+//
+// Copyright 2019 AT&T Intellectual Property
+// Copyright 2019 Nokia
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//  This source code is part of the near-RT RIC (RAN Intelligent Controller)
+//  platform project (RICP).
+
+
+package rmrreceiver
+
+import (
+	"e2mgr/logger"
+	"e2mgr/managers/notificationmanager"
+	"e2mgr/rmrCgo"
+)
+
+type RmrReceiver struct {
+	logger    *logger.Logger
+	nManager  *notificationmanager.NotificationManager
+	messenger rmrCgo.RmrMessenger
+}
+
+func NewRmrReceiver(logger *logger.Logger, messenger rmrCgo.RmrMessenger, nManager *notificationmanager.NotificationManager) *RmrReceiver {
+	return &RmrReceiver{
+		logger:    logger,
+		nManager:  nManager,
+		messenger: messenger,
+	}
+}
+
+func (r *RmrReceiver) ListenAndHandle() {
+
+	for {
+		mbuf, err := r.messenger.RecvMsg()
+
+		if err != nil {
+			r.logger.Errorf("#RmrReceiver.ListenAndHandle - error: %s", err)
+			continue
+		}
+
+		r.logger.Debugf("#RmrReceiver.ListenAndHandle - Going to handle received message: %#v\n", mbuf)
+
+		// TODO: go routine?
+		_ = r.nManager.HandleMessage(mbuf)
+	}
+}

setup/e2mgr/E2Manager/services/rmrreceiver/rmr_receiver_test.go

@@ -0,0 +1,80 @@
+//
+// Copyright 2019 AT&T Intellectual Property
+// Copyright 2019 Nokia
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//  This source code is part of the near-RT RIC (RAN Intelligent Controller)
+//  platform project (RICP).
+
+
+package rmrreceiver
+
+import (
+	"e2mgr/clients"
+	"e2mgr/configuration"
+	"e2mgr/logger"
+	"e2mgr/managers"
+	"e2mgr/managers/notificationmanager"
+	"e2mgr/mocks"
+	"e2mgr/providers/rmrmsghandlerprovider"
+	"e2mgr/rmrCgo"
+	"e2mgr/services"
+	"e2mgr/services/rmrsender"
+	"e2mgr/tests"
+	"fmt"
+	"testing"
+	"time"
+)
+
+func TestListenAndHandle(t *testing.T) {
+	log, err := logger.InitLogger(logger.DebugLevel)
+	if err != nil {
+		t.Errorf("#rmr_service_test.TestListenAndHandle - failed to initialize logger, error: %s", err)
+	}
+	rmrReceiver := initRmrReceiver(log)
+	go rmrReceiver.ListenAndHandle()
+	time.Sleep(time.Microsecond * 10)
+}
+
+func initRmrMessenger(log *logger.Logger) rmrCgo.RmrMessenger {
+	rmrMessengerMock := &mocks.RmrMessengerMock{}
+	rmrMessenger := rmrCgo.RmrMessenger(rmrMessengerMock)
+	rmrMessengerMock.On("Init", tests.GetPort(), tests.MaxMsgSize, tests.Flags, log).Return(&rmrMessenger)
+
+	// TODO: that's not good since we don't actually test anything. if the error is populated then the loop will just continue and it's sort of a "workaround" for that method to be called
+	var buf *rmrCgo.MBuf
+	e := fmt.Errorf("test error")
+	rmrMessengerMock.On("RecvMsg").Return(buf, e)
+	return rmrMessenger
+}
+
+func initRmrReceiver(logger *logger.Logger) *RmrReceiver {
+	config := &configuration.Configuration{RnibRetryIntervalMs: 10, MaxRnibConnectionAttempts: 3}
+
+	readerMock := &mocks.RnibReaderMock{}
+	writerMock := &mocks.RnibWriterMock{}
+	httpClient := &mocks.HttpClientMock{}
+
+	rnibDataService := services.NewRnibDataService(logger, config, readerMock, writerMock)
+	rmrMessenger := initRmrMessenger(logger)
+	rmrSender := rmrsender.NewRmrSender(logger, rmrMessenger)
+	ranSetupManager := managers.NewRanSetupManager(logger, rmrSender, rnibDataService)
+	e2tInstancesManager := managers.NewE2TInstancesManager(rnibDataService, logger)
+	routingManagerClient := clients.NewRoutingManagerClient(logger, config, httpClient)
+	e2tAssociationManager := managers.NewE2TAssociationManager(logger, rnibDataService, e2tInstancesManager, routingManagerClient)
+	rmrNotificationHandlerProvider := rmrmsghandlerprovider.NewNotificationHandlerProvider()
+	rmrNotificationHandlerProvider.Init(logger, config, rnibDataService, rmrSender, ranSetupManager, e2tInstancesManager, routingManagerClient, e2tAssociationManager)
+	notificationManager := notificationmanager.NewNotificationManager(logger, rmrNotificationHandlerProvider)
+	return NewRmrReceiver(logger, rmrMessenger, notificationManager)
+}

setup/e2mgr/E2Manager/services/rmrsender/rmr_sender.go

@@ -0,0 +1,80 @@
+//
+// Copyright 2019 AT&T Intellectual Property
+// Copyright 2019 Nokia
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//  This source code is part of the near-RT RIC (RAN Intelligent Controller)
+//  platform project (RICP).
+
+
+package rmrsender
+
+import (
+	"e2mgr/logger"
+	"e2mgr/models"
+	"e2mgr/rmrCgo"
+)
+
+type RmrSender struct {
+	logger    *logger.Logger
+	messenger rmrCgo.RmrMessenger
+}
+
+func NewRmrSender(logger *logger.Logger, messenger rmrCgo.RmrMessenger) *RmrSender {
+	return &RmrSender{
+		logger:    logger,
+		messenger: messenger,
+	}
+}
+
+func (r *RmrSender) WhSend(rmrMessage *models.RmrMessage) error {
+	msg := rmrCgo.NewMBuf(rmrMessage.MsgType, len(rmrMessage.Payload), rmrMessage.RanName, &rmrMessage.Payload, &rmrMessage.XAction, rmrMessage.GetMsgSrc())
+
+	_, err := r.messenger.WhSendMsg(msg, true)
+
+	if err != nil {
+		r.logger.Errorf("#RmrSender.WhSend - RAN name: %s , Message type: %d - Failed sending message. Error: %v", rmrMessage.RanName, rmrMessage.MsgType, err)
+		return err
+	}
+
+	r.logger.Infof("#RmrSender.WhSend - RAN name: %s , Message type: %d - Successfully sent RMR message", rmrMessage.RanName, rmrMessage.MsgType)
+	return nil
+}
+
+func (r *RmrSender) Send(rmrMessage *models.RmrMessage) error {
+	msg := rmrCgo.NewMBuf(rmrMessage.MsgType, len(rmrMessage.Payload), rmrMessage.RanName, &rmrMessage.Payload, &rmrMessage.XAction, rmrMessage.GetMsgSrc())
+
+	_, err := r.messenger.SendMsg(msg, true)
+
+	if err != nil {
+		r.logger.Errorf("#RmrSender.Send - RAN name: %s , Message type: %d - Failed sending message. Error: %v", rmrMessage.RanName, rmrMessage.MsgType, err)
+		return err
+	}
+
+	r.logger.Infof("#RmrSender.Send - RAN name: %s , Message type: %d - Successfully sent RMR message", rmrMessage.RanName, rmrMessage.MsgType)
+	return nil
+}
+
+func (r *RmrSender) SendWithoutLogs(rmrMessage *models.RmrMessage) error {
+	msg := rmrCgo.NewMBuf(rmrMessage.MsgType, len(rmrMessage.Payload), rmrMessage.RanName, &rmrMessage.Payload, &rmrMessage.XAction, rmrMessage.GetMsgSrc())
+
+	_, err := r.messenger.SendMsg(msg, false)
+
+	if err != nil {
+		r.logger.Errorf("#RmrSender.Send - RAN name: %s , Message type: %d - Failed sending message. Error: %v", rmrMessage.RanName, rmrMessage.MsgType, err)
+		return err
+	}
+
+	return nil
+}

setup/e2mgr/E2Manager/services/rmrsender/rmr_sender_test.go

@@ -0,0 +1,94 @@
+//
+// Copyright 2019 AT&T Intellectual Property
+// Copyright 2019 Nokia
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//  This source code is part of the near-RT RIC (RAN Intelligent Controller)
+//  platform project (RICP).
+
+
+package rmrsender
+
+import (
+	"e2mgr/logger"
+	"e2mgr/mocks"
+	"e2mgr/models"
+	"e2mgr/rmrCgo"
+	"fmt"
+	"github.com/stretchr/testify/assert"
+	"testing"
+	"unsafe"
+)
+
+func initRmrSenderTest(t *testing.T) (*logger.Logger, *mocks.RmrMessengerMock) {
+	log := initLog(t)
+	rmrMessengerMock := &mocks.RmrMessengerMock{}
+	rmrMessengerMock.On("IsReady").Return(true)
+	rmrMessengerMock.On("Close").Return()
+	return log, rmrMessengerMock
+}
+
+//func TestRmrSender_CloseContext(t *testing.T) {
+//	logger, rmrMessengerMock := initRmrSenderTest(t)
+//
+//	rmrMessenger := rmrCgo.RmrMessenger(rmrMessengerMock)
+//	rmrSender := NewRmrSender(logger, &rmrMessenger)
+//
+//	rmrSender.CloseContext()
+//	time.Sleep(time.Microsecond * 10)
+//}
+
+func TestRmrSenderSendSuccess(t *testing.T) {
+	logger, rmrMessengerMock := initRmrSenderTest(t)
+
+	ranName := "test"
+	payload := []byte("some payload")
+	var xAction []byte
+	var msgSrc unsafe.Pointer
+	mbuf := rmrCgo.NewMBuf(123, len(payload), ranName, &payload, &xAction, msgSrc)
+	rmrMessengerMock.On("SendMsg", mbuf, true).Return(&rmrCgo.MBuf{}, nil)
+	rmrMsg := models.NewRmrMessage(123, ranName, payload, xAction, nil)
+	rmrMessenger := rmrCgo.RmrMessenger(rmrMessengerMock)
+	rmrSender := NewRmrSender(logger, rmrMessenger)
+	err := rmrSender.Send(rmrMsg)
+	assert.Nil(t, err)
+	rmrMessengerMock.AssertCalled(t, "SendMsg",mbuf, true)
+
+}
+
+func TestRmrSenderSendFailure(t *testing.T) {
+	logger, rmrMessengerMock := initRmrSenderTest(t)
+
+	ranName := "test"
+	payload := []byte("some payload")
+	var xAction []byte
+	var msgSrc unsafe.Pointer
+	mbuf := rmrCgo.NewMBuf(123, len(payload), ranName, &payload, &xAction, msgSrc)
+	rmrMessengerMock.On("SendMsg", mbuf, true).Return(mbuf, fmt.Errorf("rmr send failure"))
+	rmrMsg := models.NewRmrMessage(123, ranName, payload, xAction, nil)
+	rmrMessenger := rmrCgo.RmrMessenger(rmrMessengerMock)
+	rmrSender := NewRmrSender(logger, rmrMessenger)
+	err := rmrSender.Send(rmrMsg)
+	rmrMessengerMock.AssertCalled(t, "SendMsg",mbuf, true)
+	assert.NotNil(t, err)
+}
+
+// TODO: extract to test_utils
+func initLog(t *testing.T) *logger.Logger {
+	log, err := logger.InitLogger(logger.InfoLevel)
+	if err != nil {
+		t.Fatalf("#initLog - failed to initialize logger, error: %s", err)
+	}
+	return log
+}

setup/e2mgr/E2Manager/services/rnib_data_service.go

@@ -0,0 +1,310 @@
+//
+// Copyright 2019 AT&T Intellectual Property
+// Copyright 2019 Nokia
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//  This source code is part of the near-RT RIC (RAN Intelligent Controller)
+//  platform project (RICP).
+
+package services
+
+import (
+	"e2mgr/configuration"
+	"e2mgr/logger"
+	"e2mgr/rNibWriter"
+	"gerrit.o-ran-sc.org/r/ric-plt/nodeb-rnib.git/common"
+	"gerrit.o-ran-sc.org/r/ric-plt/nodeb-rnib.git/entities"
+	"gerrit.o-ran-sc.org/r/ric-plt/nodeb-rnib.git/reader"
+	"net"
+	"time"
+)
+
+type RNibDataService interface {
+	SaveNodeb(nbIdentity *entities.NbIdentity, nb *entities.NodebInfo) error
+	UpdateNodebInfo(nodebInfo *entities.NodebInfo) error
+	SaveRanLoadInformation(inventoryName string, ranLoadInformation *entities.RanLoadInformation) error
+	GetNodeb(ranName string) (*entities.NodebInfo, error)
+	GetListNodebIds() ([]*entities.NbIdentity, error)
+	PingRnib() bool
+	GetE2TInstance(address string) (*entities.E2TInstance, error)
+	GetE2TInstances(addresses []string) ([]*entities.E2TInstance, error)
+	GetE2TAddresses() ([]string, error)
+	SaveE2TInstance(e2tInstance *entities.E2TInstance) error
+	SaveE2TAddresses(addresses []string) error
+	GetE2TInstanceNoLogs(address string) (*entities.E2TInstance, error)
+	GetE2TInstancesNoLogs(addresses []string) ([]*entities.E2TInstance, error)
+	SaveE2TInstanceNoLogs(e2tInstance *entities.E2TInstance) error
+	GetE2TAddressesNoLogs() ([]string, error)
+	RemoveE2TInstance(e2tAddress string) error
+	UpdateGnbCells(nodebInfo *entities.NodebInfo, servedNrCells []*entities.ServedNRCell) error
+	RemoveServedNrCells(inventoryName string, servedNrCells []*entities.ServedNRCell) error
+}
+
+type rNibDataService struct {
+	logger        *logger.Logger
+	rnibReader    reader.RNibReader
+	rnibWriter    rNibWriter.RNibWriter
+	maxAttempts   int
+	retryInterval time.Duration
+}
+
+func NewRnibDataService(logger *logger.Logger, config *configuration.Configuration, rnibReader reader.RNibReader, rnibWriter rNibWriter.RNibWriter) *rNibDataService {
+	return &rNibDataService{
+		logger:        logger,
+		rnibReader:    rnibReader,
+		rnibWriter:    rnibWriter,
+		maxAttempts:   config.MaxRnibConnectionAttempts,
+		retryInterval: time.Duration(config.RnibRetryIntervalMs) * time.Millisecond,
+	}
+}
+
+func (w *rNibDataService) RemoveServedNrCells(inventoryName string, servedNrCells []*entities.ServedNRCell) error {
+	err := w.retry("RemoveServedNrCells", func() (err error) {
+		err = w.rnibWriter.RemoveServedNrCells(inventoryName, servedNrCells)
+		return
+	})
+
+	return err
+}
+
+func (w *rNibDataService) UpdateGnbCells(nodebInfo *entities.NodebInfo, servedNrCells []*entities.ServedNRCell) error {
+	w.logger.Infof("#RnibDataService.UpdateGnbCells - nodebInfo: %s, servedNrCells: %s", nodebInfo, servedNrCells)
+
+	err := w.retry("UpdateGnbCells", func() (err error) {
+		err = w.rnibWriter.UpdateGnbCells(nodebInfo, servedNrCells)
+		return
+	})
+
+	return err
+}
+
+func (w *rNibDataService) UpdateNodebInfo(nodebInfo *entities.NodebInfo) error {
+	w.logger.Infof("#RnibDataService.UpdateNodebInfo - nodebInfo: %s", nodebInfo)
+
+	err := w.retry("UpdateNodebInfo", func() (err error) {
+		err = w.rnibWriter.UpdateNodebInfo(nodebInfo)
+		return
+	})
+
+	return err
+}
+
+func (w *rNibDataService) SaveNodeb(nbIdentity *entities.NbIdentity, nb *entities.NodebInfo) error {
+	w.logger.Infof("#RnibDataService.SaveNodeb - nbIdentity: %s, nodebInfo: %s", nbIdentity, nb)
+
+	err := w.retry("SaveNodeb", func() (err error) {
+		err = w.rnibWriter.SaveNodeb(nbIdentity, nb)
+		return
+	})
+
+	return err
+}
+
+func (w *rNibDataService) SaveRanLoadInformation(inventoryName string, ranLoadInformation *entities.RanLoadInformation) error {
+	w.logger.Infof("#RnibDataService.SaveRanLoadInformation - inventoryName: %s, ranLoadInformation: %s", inventoryName, ranLoadInformation)
+
+	err := w.retry("SaveRanLoadInformation", func() (err error) {
+		err = w.rnibWriter.SaveRanLoadInformation(inventoryName, ranLoadInformation)
+		return
+	})
+
+	return err
+}
+
+func (w *rNibDataService) GetNodeb(ranName string) (*entities.NodebInfo, error) {
+
+	var nodeb *entities.NodebInfo = nil
+
+	err := w.retry("GetNodeb", func() (err error) {
+		nodeb, err = w.rnibReader.GetNodeb(ranName)
+		return
+	})
+
+	if err == nil {
+		w.logger.Infof("#RnibDataService.GetNodeb - RAN name: %s, connection status: %s, associated E2T: %s", nodeb.RanName, nodeb.ConnectionStatus, nodeb.AssociatedE2TInstanceAddress)
+	}
+
+	return nodeb, err
+}
+
+func (w *rNibDataService) GetListNodebIds() ([]*entities.NbIdentity, error) {
+	var nodeIds []*entities.NbIdentity = nil
+
+	err := w.retry("GetListNodebIds", func() (err error) {
+		nodeIds, err = w.rnibReader.GetListNodebIds()
+		return
+	})
+
+	if err == nil {
+		w.logger.Infof("#RnibDataService.GetListNodebIds - RANs count: %d", len(nodeIds))
+	}
+
+	return nodeIds, err
+}
+
+func (w *rNibDataService) GetE2TInstance(address string) (*entities.E2TInstance, error) {
+	var e2tInstance *entities.E2TInstance = nil
+
+	err := w.retry("GetE2TInstance", func() (err error) {
+		e2tInstance, err = w.rnibReader.GetE2TInstance(address)
+		return
+	})
+
+	if err == nil {
+		w.logger.Infof("#RnibDataService.GetE2TInstance - E2T instance address: %s, state: %s, associated RANs count: %d, keep Alive ts: %d", e2tInstance.Address, e2tInstance.State, len(e2tInstance.AssociatedRanList), e2tInstance.KeepAliveTimestamp)
+	}
+
+	return e2tInstance, err
+}
+
+func (w *rNibDataService) GetE2TInstanceNoLogs(address string) (*entities.E2TInstance, error) {
+	var e2tInstance *entities.E2TInstance = nil
+
+	err := w.retry("GetE2TInstance", func() (err error) {
+		e2tInstance, err = w.rnibReader.GetE2TInstance(address)
+		return
+	})
+
+	return e2tInstance, err
+}
+
+func (w *rNibDataService) GetE2TInstances(addresses []string) ([]*entities.E2TInstance, error) {
+	w.logger.Infof("#RnibDataService.GetE2TInstances - addresses: %s", addresses)
+	var e2tInstances []*entities.E2TInstance = nil
+
+	err := w.retry("GetE2TInstance", func() (err error) {
+		e2tInstances, err = w.rnibReader.GetE2TInstances(addresses)
+		return
+	})
+
+	return e2tInstances, err
+}
+
+func (w *rNibDataService) GetE2TInstancesNoLogs(addresses []string) ([]*entities.E2TInstance, error) {
+
+	var e2tInstances []*entities.E2TInstance = nil
+
+	err := w.retry("GetE2TInstance", func() (err error) {
+		e2tInstances, err = w.rnibReader.GetE2TInstances(addresses)
+		return
+	})
+
+	return e2tInstances, err
+}
+
+func (w *rNibDataService) GetE2TAddresses() ([]string, error) {
+
+	var e2tAddresses []string = nil
+
+	err := w.retry("GetE2TAddresses", func() (err error) {
+		e2tAddresses, err = w.rnibReader.GetE2TAddresses()
+		return
+	})
+
+	if err == nil {
+		w.logger.Infof("#RnibDataService.GetE2TAddresses - addresses: %s", e2tAddresses)
+	}
+
+	return e2tAddresses, err
+}
+
+func (w *rNibDataService) GetE2TAddressesNoLogs() ([]string, error) {
+
+	var e2tAddresses []string = nil
+
+	err := w.retry("GetE2TAddresses", func() (err error) {
+		e2tAddresses, err = w.rnibReader.GetE2TAddresses()
+		return
+	})
+
+	return e2tAddresses, err
+}
+
+func (w *rNibDataService) SaveE2TInstance(e2tInstance *entities.E2TInstance) error {
+	w.logger.Infof("#RnibDataService.SaveE2TInstance - E2T instance address: %s, podName: %s, state: %s, associated RANs count: %d, keep Alive ts: %d", e2tInstance.Address, e2tInstance.PodName, e2tInstance.State, len(e2tInstance.AssociatedRanList), e2tInstance.KeepAliveTimestamp)
+
+	return w.SaveE2TInstanceNoLogs(e2tInstance)
+}
+
+func (w *rNibDataService) SaveE2TInstanceNoLogs(e2tInstance *entities.E2TInstance) error {
+
+	err := w.retry("SaveE2TInstance", func() (err error) {
+		err = w.rnibWriter.SaveE2TInstance(e2tInstance)
+		return
+	})
+
+	return err
+}
+
+func (w *rNibDataService) SaveE2TAddresses(addresses []string) error {
+	w.logger.Infof("#RnibDataService.SaveE2TAddresses - addresses: %s", addresses)
+
+	err := w.retry("SaveE2TAddresses", func() (err error) {
+		err = w.rnibWriter.SaveE2TAddresses(addresses)
+		return
+	})
+
+	return err
+}
+
+func (w *rNibDataService) RemoveE2TInstance(e2tAddress string) error {
+	w.logger.Infof("#RnibDataService.RemoveE2TInstance - e2tAddress: %s", e2tAddress)
+
+	err := w.retry("RemoveE2TInstance", func() (err error) {
+		err = w.rnibWriter.RemoveE2TInstance(e2tAddress)
+		return
+	})
+
+	return err
+}
+
+func (w *rNibDataService) PingRnib() bool {
+	err := w.retry("GetListNodebIds", func() (err error) {
+		_, err = w.rnibReader.GetListNodebIds()
+		return
+	})
+
+	return !isRnibConnectionError(err)
+}
+
+func (w *rNibDataService) retry(rnibFunc string, f func() error) (err error) {
+	attempts := w.maxAttempts
+
+	for i := 1; ; i++ {
+		err = f()
+		if err == nil {
+			return
+		}
+		if !isRnibConnectionError(err) {
+			return err
+		}
+		if i >= attempts {
+			w.logger.Errorf("#RnibDataService.retry - after %d attempts of %s, last error: %s", attempts, rnibFunc, err)
+			return err
+		}
+		time.Sleep(w.retryInterval)
+
+		w.logger.Infof("#RnibDataService.retry - retrying %d %s after error: %s", i, rnibFunc, err)
+	}
+}
+
+func isRnibConnectionError(err error) bool {
+	internalErr, ok := err.(*common.InternalError)
+	if !ok {
+		return false
+	}
+	_, ok = internalErr.Err.(*net.OpError)
+
+	return ok
+}

setup/e2mgr/E2Manager/services/rnib_data_service_test.go

@@ -0,0 +1,314 @@
+//
+// Copyright 2019 AT&T Intellectual Property
+// Copyright 2019 Nokia
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//  This source code is part of the near-RT RIC (RAN Intelligent Controller)
+//  platform project (RICP).
+
+
+package services
+
+import (
+	"e2mgr/configuration"
+	"e2mgr/logger"
+	"e2mgr/mocks"
+	"fmt"
+	"gerrit.o-ran-sc.org/r/ric-plt/nodeb-rnib.git/common"
+	"gerrit.o-ran-sc.org/r/ric-plt/nodeb-rnib.git/entities"
+	"github.com/stretchr/testify/assert"
+	"net"
+	"strings"
+	"testing"
+)
+
+func setupRnibDataServiceTest(t *testing.T) (*rNibDataService, *mocks.RnibReaderMock, *mocks.RnibWriterMock) {
+	return setupRnibDataServiceTestWithMaxAttempts(t, 3)
+}
+
+func setupRnibDataServiceTestWithMaxAttempts(t *testing.T, maxAttempts int) (*rNibDataService, *mocks.RnibReaderMock, *mocks.RnibWriterMock) {
+	logger, err := logger.InitLogger(logger.DebugLevel)
+	if err != nil {
+		t.Errorf("#... - failed to initialize logger, error: %s", err)
+	}
+
+	config := &configuration.Configuration{RnibRetryIntervalMs: 10, MaxRnibConnectionAttempts: maxAttempts}
+
+	readerMock := &mocks.RnibReaderMock{}
+
+
+	writerMock := &mocks.RnibWriterMock{}
+
+
+	rnibDataService := NewRnibDataService(logger, config, readerMock, writerMock)
+	assert.NotNil(t, rnibDataService)
+
+	return rnibDataService, readerMock, writerMock
+}
+
+func TestSuccessfulSaveNodeb(t *testing.T) {
+	rnibDataService, _, writerMock := setupRnibDataServiceTest(t)
+
+	nodebInfo := &entities.NodebInfo{}
+	nbIdentity := &entities.NbIdentity{}
+	writerMock.On("SaveNodeb", nbIdentity, nodebInfo).Return(nil)
+
+	rnibDataService.SaveNodeb(nbIdentity, nodebInfo)
+	writerMock.AssertNumberOfCalls(t, "SaveNodeb", 1)
+}
+
+func TestConnFailureSaveNodeb(t *testing.T) {
+	rnibDataService, _, writerMock := setupRnibDataServiceTest(t)
+
+	nodebInfo := &entities.NodebInfo{}
+	nbIdentity := &entities.NbIdentity{}
+	mockErr := &common.InternalError{Err: &net.OpError{Err: fmt.Errorf("connection error")}}
+	writerMock.On("SaveNodeb", nbIdentity, nodebInfo).Return(mockErr)
+
+	rnibDataService.SaveNodeb(nbIdentity, nodebInfo)
+	writerMock.AssertNumberOfCalls(t, "SaveNodeb", 3)
+}
+
+func TestNonConnFailureSaveNodeb(t *testing.T) {
+	rnibDataService, _, writerMock := setupRnibDataServiceTest(t)
+
+	nodebInfo := &entities.NodebInfo{}
+	nbIdentity := &entities.NbIdentity{}
+	mockErr := &common.InternalError{Err: fmt.Errorf("non connection failure")}
+	writerMock.On("SaveNodeb", nbIdentity, nodebInfo).Return(mockErr)
+
+	rnibDataService.SaveNodeb(nbIdentity, nodebInfo)
+	writerMock.AssertNumberOfCalls(t, "SaveNodeb", 1)
+}
+
+func TestSuccessfulUpdateNodebInfo(t *testing.T) {
+	rnibDataService, _, writerMock := setupRnibDataServiceTest(t)
+
+	nodebInfo := &entities.NodebInfo{}
+	writerMock.On("UpdateNodebInfo", nodebInfo).Return(nil)
+
+	rnibDataService.UpdateNodebInfo(nodebInfo)
+	writerMock.AssertNumberOfCalls(t, "UpdateNodebInfo", 1)
+}
+
+func TestConnFailureUpdateNodebInfo(t *testing.T) {
+	rnibDataService, _, writerMock := setupRnibDataServiceTest(t)
+
+	nodebInfo := &entities.NodebInfo{}
+	mockErr := &common.InternalError{Err: &net.OpError{Err: fmt.Errorf("connection error")}}
+	writerMock.On("UpdateNodebInfo", nodebInfo).Return(mockErr)
+
+	rnibDataService.UpdateNodebInfo(nodebInfo)
+	writerMock.AssertNumberOfCalls(t, "UpdateNodebInfo", 3)
+}
+
+func TestSuccessfulSaveRanLoadInformation(t *testing.T) {
+	rnibDataService, _, writerMock := setupRnibDataServiceTest(t)
+
+	var ranName string = "abcd"
+	ranLoadInformation := &entities.RanLoadInformation{}
+	writerMock.On("SaveRanLoadInformation", ranName, ranLoadInformation).Return(nil)
+
+	rnibDataService.SaveRanLoadInformation(ranName, ranLoadInformation)
+	writerMock.AssertNumberOfCalls(t, "SaveRanLoadInformation", 1)
+}
+
+func TestConnFailureSaveRanLoadInformation(t *testing.T) {
+	rnibDataService, _, writerMock := setupRnibDataServiceTest(t)
+
+	var ranName string = "abcd"
+	ranLoadInformation := &entities.RanLoadInformation{}
+	mockErr := &common.InternalError{Err: &net.OpError{Err: fmt.Errorf("connection error")}}
+	writerMock.On("SaveRanLoadInformation", ranName, ranLoadInformation).Return(mockErr)
+
+	rnibDataService.SaveRanLoadInformation(ranName, ranLoadInformation)
+	writerMock.AssertNumberOfCalls(t, "SaveRanLoadInformation", 3)
+}
+
+func TestSuccessfulGetNodeb(t *testing.T) {
+	rnibDataService, readerMock, _ := setupRnibDataServiceTest(t)
+
+	invName := "abcd"
+	nodebInfo := &entities.NodebInfo{}
+	readerMock.On("GetNodeb", invName).Return(nodebInfo, nil)
+
+	res, err := rnibDataService.GetNodeb(invName)
+	readerMock.AssertNumberOfCalls(t, "GetNodeb", 1)
+	assert.Equal(t, nodebInfo, res)
+	assert.Nil(t, err)
+}
+
+func TestConnFailureGetNodeb(t *testing.T) {
+	rnibDataService, readerMock, _ := setupRnibDataServiceTest(t)
+
+	invName := "abcd"
+	var nodeb *entities.NodebInfo = nil
+	mockErr := &common.InternalError{Err: &net.OpError{Err: fmt.Errorf("connection error")}}
+	readerMock.On("GetNodeb", invName).Return(nodeb, mockErr)
+
+	res, err := rnibDataService.GetNodeb(invName)
+	readerMock.AssertNumberOfCalls(t, "GetNodeb", 3)
+	assert.True(t, strings.Contains(err.Error(), "connection error", ))
+	assert.Equal(t, nodeb, res)
+}
+
+func TestSuccessfulGetNodebIdList(t *testing.T) {
+	rnibDataService, readerMock, _ := setupRnibDataServiceTest(t)
+
+	nodeIds := []*entities.NbIdentity{}
+	readerMock.On("GetListNodebIds").Return(nodeIds, nil)
+
+	res, err := rnibDataService.GetListNodebIds()
+	readerMock.AssertNumberOfCalls(t, "GetListNodebIds", 1)
+	assert.Equal(t, nodeIds, res)
+	assert.Nil(t, err)
+}
+
+func TestConnFailureGetNodebIdList(t *testing.T) {
+	rnibDataService, readerMock, _ := setupRnibDataServiceTest(t)
+
+	var nodeIds []*entities.NbIdentity = nil
+	mockErr := &common.InternalError{Err: &net.OpError{Err: fmt.Errorf("connection error")}}
+	readerMock.On("GetListNodebIds").Return(nodeIds, mockErr)
+
+	res, err := rnibDataService.GetListNodebIds()
+	readerMock.AssertNumberOfCalls(t, "GetListNodebIds", 3)
+	assert.True(t, strings.Contains(err.Error(), "connection error", ))
+	assert.Equal(t, nodeIds, res)
+}
+
+func TestConnFailureTwiceGetNodebIdList(t *testing.T) {
+	rnibDataService, readerMock, _ := setupRnibDataServiceTest(t)
+
+	invName := "abcd"
+	var nodeb *entities.NodebInfo = nil
+	var nodeIds []*entities.NbIdentity = nil
+	mockErr := &common.InternalError{Err: &net.OpError{Err: fmt.Errorf("connection error")}}
+	readerMock.On("GetNodeb", invName).Return(nodeb, mockErr)
+	readerMock.On("GetListNodebIds").Return(nodeIds, mockErr)
+
+	res, err := rnibDataService.GetListNodebIds()
+	readerMock.AssertNumberOfCalls(t, "GetListNodebIds", 3)
+	assert.True(t, strings.Contains(err.Error(), "connection error", ))
+	assert.Equal(t, nodeIds, res)
+
+	res2, err := rnibDataService.GetNodeb(invName)
+	readerMock.AssertNumberOfCalls(t, "GetNodeb", 3)
+	assert.True(t, strings.Contains(err.Error(), "connection error", ))
+	assert.Equal(t, nodeb, res2)
+}
+
+func TestConnFailureWithAnotherConfig(t *testing.T) {
+	rnibDataService, readerMock, _ := setupRnibDataServiceTestWithMaxAttempts(t, 5)
+
+	var nodeIds []*entities.NbIdentity = nil
+	mockErr := &common.InternalError{Err: &net.OpError{Err: fmt.Errorf("connection error")}}
+	readerMock.On("GetListNodebIds").Return(nodeIds, mockErr)
+
+	res, err := rnibDataService.GetListNodebIds()
+	readerMock.AssertNumberOfCalls(t, "GetListNodebIds", 5)
+	assert.True(t, strings.Contains(err.Error(), "connection error", ))
+	assert.Equal(t, nodeIds, res)
+}
+
+func TestPingRnibConnFailure(t *testing.T) {
+	rnibDataService, readerMock, _ := setupRnibDataServiceTest(t)
+
+	var nodeIds []*entities.NbIdentity = nil
+	mockErr := &common.InternalError{Err: &net.OpError{Err: fmt.Errorf("connection error")}}
+	readerMock.On("GetListNodebIds").Return(nodeIds, mockErr)
+
+	res := rnibDataService.PingRnib()
+	readerMock.AssertNumberOfCalls(t, "GetListNodebIds", 3)
+	assert.False(t, res)
+}
+
+func TestPingRnibOkNoError(t *testing.T) {
+	rnibDataService, readerMock, _ := setupRnibDataServiceTest(t)
+
+	var nodeIds []*entities.NbIdentity = nil
+	readerMock.On("GetListNodebIds").Return(nodeIds, nil)
+
+	res := rnibDataService.PingRnib()
+	readerMock.AssertNumberOfCalls(t, "GetListNodebIds", 1)
+	assert.True(t, res)
+}
+
+func TestPingRnibOkOtherError(t *testing.T) {
+	rnibDataService, readerMock, _ := setupRnibDataServiceTest(t)
+
+	var nodeIds []*entities.NbIdentity = nil
+	mockErr := &common.InternalError{Err: fmt.Errorf("non connection error")}
+	readerMock.On("GetListNodebIds").Return(nodeIds, mockErr)
+
+	res := rnibDataService.PingRnib()
+	readerMock.AssertNumberOfCalls(t, "GetListNodebIds", 1)
+	assert.True(t, res)
+}
+
+//func TestConnFailureThenSuccessGetNodebIdList(t *testing.T) {
+//	rnibDataService, readerMock, _ := setupRnibDataServiceTest(t)
+//
+//	var nilNodeIds []*entities.NbIdentity = nil
+//	nodeIds := []*entities.NbIdentity{}
+//	mockErr := &common.InternalError{Err: &net.OpError{Err:fmt.Errorf("connection error")}}
+//	//readerMock.On("GetListNodebIds").Return(nilNodeIds, mockErr)
+//	//readerMock.On("GetListNodebIds").Return(nodeIds, nil)
+//
+//	res, err := rnibDataService.GetListNodebIds()
+//	readerMock.AssertNumberOfCalls(t, "GetListNodebIds", 2)
+//	assert.True(t, strings.Contains(err.Error(),"connection failure", ))
+//	assert.Equal(t, nodeIds, res)
+//}
+
+func TestGetE2TInstanceConnFailure(t *testing.T) {
+	rnibDataService, readerMock, _ := setupRnibDataServiceTest(t)
+
+	address := "10.10.5.20:3200"
+	var e2tInstance *entities.E2TInstance = nil
+	mockErr := &common.InternalError{Err: &net.OpError{Err: fmt.Errorf("connection error")}}
+	readerMock.On("GetE2TInstance", address).Return(e2tInstance, mockErr)
+
+	res, err := rnibDataService.GetE2TInstance(address)
+	readerMock.AssertNumberOfCalls(t, "GetE2TInstance", 3)
+	assert.Nil(t, res)
+	assert.NotNil(t, err)
+}
+
+func TestGetE2TInstanceOkNoError(t *testing.T) {
+	rnibDataService, readerMock, _ := setupRnibDataServiceTest(t)
+
+	address := "10.10.5.20:3200"
+	e2tInstance := &entities.E2TInstance{}
+	readerMock.On("GetE2TInstance", address).Return(e2tInstance, nil)
+
+	res, err := rnibDataService.GetE2TInstance(address)
+	readerMock.AssertNumberOfCalls(t, "GetE2TInstance", 1)
+	assert.Nil(t, err)
+	assert.Equal(t, e2tInstance, res)
+}
+
+func TestGetE2TInstanceOkOtherError(t *testing.T) {
+	rnibDataService, readerMock, _ := setupRnibDataServiceTest(t)
+
+	address := "10.10.5.20:3200"
+	var e2tInstance *entities.E2TInstance = nil
+	mockErr := &common.InternalError{Err: fmt.Errorf("non connection error")}
+	readerMock.On("GetE2TInstance", address).Return(e2tInstance, mockErr)
+
+	res, err := rnibDataService.GetE2TInstance(address)
+	readerMock.AssertNumberOfCalls(t, "GetE2TInstance", 1)
+	assert.Nil(t, res)
+	assert.NotNil(t, err)
+}