VOL-4077: Improve storage usage on etcd
- the onu adapter will now receive the tp instance on the
  inter-container kafka message and need not reach the etcd store
  to fetch it.
- on reconcile, the onu adapter need to go to the kv store to
  fetch the tp instance, but request the tp instance via a new
  API towards openolt adapter which in-turn will fetch from the
  cache.
- re-org the code in onu-metrics-manager to restore pm-data
  on reconcile to avoid panics by accessing uninitialzed data
  if ani-fsm were to try adding gem port for monitoring
  before pm-data is initialized properly.

Change-Id: I82a6de2772155f6e08390b671fe26d692dd02c99
diff --git a/internal/pkg/onuadaptercore/onu_uni_port.go b/internal/pkg/onuadaptercore/onu_uni_port.go
index 660c043..bfd52f9 100644
--- a/internal/pkg/onuadaptercore/onu_uni_port.go
+++ b/internal/pkg/onuadaptercore/onu_uni_port.go
@@ -22,12 +22,13 @@
 	"fmt"
 	"strconv"
 	"strings"
+	"time"
 
 	//"sync"
 	//"time"
 
-	//"github.com/opencord/voltha-lib-go/v4/pkg/kafka"
-	"github.com/opencord/voltha-lib-go/v4/pkg/log"
+	//"github.com/opencord/voltha-lib-go/v5/pkg/kafka"
+	"github.com/opencord/voltha-lib-go/v5/pkg/log"
 	vc "github.com/opencord/voltha-protos/v4/go/common"
 	of "github.com/opencord/voltha-protos/v4/go/openflow_13"
 	"github.com/opencord/voltha-protos/v4/go/voltha"
@@ -129,21 +130,29 @@
 			MaxSpeed:   1000,
 		},
 	}
-	if pUniPort != nil {
-		if err := apDeviceHandler.coreProxy.PortCreated(log.WithSpanFromContext(context.TODO(), ctx),
+	maxRetry := 3
+	retryCnt := 0
+	var err error
+	for retryCnt = 0; retryCnt < maxRetry; retryCnt++ {
+		if err = apDeviceHandler.coreProxy.PortCreated(log.WithSpanFromContext(context.TODO(), ctx),
 			apDeviceHandler.deviceID, pUniPort); err != nil {
-			logger.Fatalf(ctx, "adding-uni-port: create-VOLTHA-Port-failed-%s", err)
-			return err
+			logger.Errorf(ctx, "Device FSM: PortCreated-failed-%s, retrying after a delay", err)
+			// retry after a sleep
+			time.Sleep(2 * time.Second)
+		} else {
+			// success, break from retry loop
+			break
 		}
-		logger.Infow(ctx, "Voltha onuUniPort-added", log.Fields{
-			"device-id": apDeviceHandler.device.Id, "PortNo": oo.portNo})
-		oo.pPort = pUniPort
-		oo.operState = vc.OperStatus_DISCOVERED
-	} else {
-		logger.Warnw(ctx, "could not create Voltha UniPort", log.Fields{
-			"device-id": apDeviceHandler.device.Id, "PortNo": oo.portNo})
-		return fmt.Errorf("create Voltha UniPort %d failed on %s", oo.portNo, apDeviceHandler.device.Id)
 	}
+	if retryCnt == maxRetry { // maxed out..
+		logger.Errorf(ctx, "Device FSM: PortCreated-failed-%s", err)
+		return fmt.Errorf("device-fsm-port-create-failed-%s", err)
+	}
+	logger.Infow(ctx, "Voltha onuUniPort-added", log.Fields{
+		"device-id": apDeviceHandler.device.Id, "PortNo": oo.portNo})
+	oo.pPort = pUniPort
+	oo.operState = vc.OperStatus_DISCOVERED
+
 	return nil
 }