[VOL-2628] Fix issue with peer ports

There is a scenario where a request to add a PON peer port
is received before the OLT PON port has been created.  When this occurs
the relationship between the parent device and that child device is
lost.  Routes needed for flow decomposition cannot be created and
any request to retrieve child devices will fail.  This fix creates
a default PON port on the OLT when the peer request is received
and update the PON port with the latest info when the actual OLT
PON port creation is received.

Change-Id: I81807b3f4cbe7e9c1e3bbcb138fa8d4f20c6edeb
diff --git a/rw_core/core/device_agent.go b/rw_core/core/device_agent.go
index 7dd5062..d20874a 100755
--- a/rw_core/core/device_agent.go
+++ b/rw_core/core/device_agent.go
@@ -1226,9 +1226,10 @@
 func (agent *DeviceAgent) addPort(ctx context.Context, port *voltha.Port) error {
 	agent.lockDevice.Lock()
 	defer agent.lockDevice.Unlock()
-	log.Debugw("addPort", log.Fields{"deviceId": agent.deviceID})
+	log.Debugw("addPort", log.Fields{"deviceId": agent.deviceID, "port": port})
 
 	cloned := agent.getDeviceWithoutLock()
+	updatePort := false
 	if cloned.Ports == nil {
 		//	First port
 		log.Debugw("addPort-first-port-to-add", log.Fields{"deviceId": agent.deviceID})
@@ -1236,35 +1237,60 @@
 	} else {
 		for _, p := range cloned.Ports {
 			if p.Type == port.Type && p.PortNo == port.PortNo {
-				log.Debugw("port already exists", log.Fields{"port": *port})
+				if p.Label == "" && p.Type == voltha.Port_PON_OLT {
+					//Creation of OLT PON port is being processed after a default PON port was created.  Just update it.
+					log.Infow("update-pon-port-created-by-default", log.Fields{"default-port": p, "port-to-add": port})
+					p.Label = port.Label
+					p.OperStatus = port.OperStatus
+					updatePort = true
+					break
+				}
+				log.Debugw("port already exists", log.Fields{"port": port})
 				return nil
 			}
 		}
 	}
-	cp := proto.Clone(port).(*voltha.Port)
-	// Set the admin state of the port to ENABLE
-	cp.AdminState = voltha.AdminState_ENABLED
-	cloned.Ports = append(cloned.Ports, cp)
+	if !updatePort {
+		cp := proto.Clone(port).(*voltha.Port)
+		// Set the admin state of the port to ENABLE
+		cp.AdminState = voltha.AdminState_ENABLED
+		cloned.Ports = append(cloned.Ports, cp)
+	}
 	// Store the device
 	return agent.updateDeviceInStoreWithoutLock(ctx, cloned, false, "")
 }
 
-func (agent *DeviceAgent) addPeerPort(ctx context.Context, port *voltha.Port_PeerPort) error {
+func (agent *DeviceAgent) addPeerPort(ctx context.Context, peerPort *voltha.Port_PeerPort) error {
 	agent.lockDevice.Lock()
 	defer agent.lockDevice.Unlock()
-	log.Debug("addPeerPort")
+	log.Debugw("adding-peer-peerPort", log.Fields{"device-id": agent.deviceID, "peer-peerPort": peerPort})
 
 	cloned := agent.getDeviceWithoutLock()
 
-	// Get the peer port on the device based on the port no
-	for _, peerPort := range cloned.Ports {
-		if peerPort.PortNo == port.PortNo { // found port
-			cp := proto.Clone(port).(*voltha.Port_PeerPort)
-			peerPort.Peers = append(peerPort.Peers, cp)
-			log.Debugw("found-peer", log.Fields{"portNo": port.PortNo, "deviceId": agent.deviceID})
+	// Get the peer port on the device based on the peerPort no
+	found := false
+	for _, port := range cloned.Ports {
+		if port.PortNo == peerPort.PortNo { // found peerPort
+			cp := proto.Clone(peerPort).(*voltha.Port_PeerPort)
+			port.Peers = append(port.Peers, cp)
+			log.Debugw("found-peer", log.Fields{"device-id": agent.deviceID, "portNo": peerPort.PortNo, "deviceId": agent.deviceID})
+			found = true
 			break
 		}
 	}
+	if !found && agent.isRootdevice {
+		// An ONU PON port has been created before the corresponding creation of the OLT PON port.  Create the OLT PON port
+		// with default values which will be updated once the OLT PON port creation is processed.
+		ponPort := &voltha.Port{
+			PortNo:     peerPort.PortNo,
+			Type:       voltha.Port_PON_OLT,
+			AdminState: voltha.AdminState_ENABLED,
+			DeviceId:   agent.deviceID,
+			Peers:      []*voltha.Port_PeerPort{proto.Clone(peerPort).(*voltha.Port_PeerPort)},
+		}
+		cloned.Ports = append(cloned.Ports, ponPort)
+		log.Infow("adding-default-pon-port", log.Fields{"device-id": agent.deviceID, "peer": peerPort, "pon-port": ponPort})
+	}
 	// Store the device
 	return agent.updateDeviceInStoreWithoutLock(ctx, cloned, false, "")
 }
diff --git a/rw_core/core/logical_device_agent.go b/rw_core/core/logical_device_agent.go
index 857776f..cfd486a 100644
--- a/rw_core/core/logical_device_agent.go
+++ b/rw_core/core/logical_device_agent.go
@@ -209,7 +209,7 @@
 	if loadFromdB {
 		go func() {
 			if err := agent.buildRoutes(context.Background()); err != nil {
-				log.Warn("routes-not-ready", log.Fields{"logical-device-id": agent.logicalDeviceID, "error": err})
+				log.Warnw("routes-not-ready", log.Fields{"logical-device-id": agent.logicalDeviceID, "error": err})
 			}
 		}()
 	}
@@ -1320,7 +1320,7 @@
 		// Reset the logical device routes
 		go func() {
 			if err := agent.buildRoutes(context.Background()); err != nil {
-				log.Warn("device-routes-not-ready", log.Fields{"logicalDeviceId": agent.logicalDeviceID, "error": err})
+				log.Warnw("device-routes-not-ready", log.Fields{"logicalDeviceId": agent.logicalDeviceID, "error": err})
 			}
 		}()
 	}
@@ -1355,7 +1355,7 @@
 	// Reset the logical device routes
 	go func() {
 		if err := agent.buildRoutes(context.Background()); err != nil {
-			log.Warn("routes-not-ready", log.Fields{"logical-device-id": agent.logicalDeviceID, "error": err})
+			log.Warnw("routes-not-ready", log.Fields{"logical-device-id": agent.logicalDeviceID, "error": err})
 		}
 	}()
 
@@ -1507,7 +1507,7 @@
 
 //rebuildRoutes rebuilds the device routes
 func (agent *LogicalDeviceAgent) buildRoutes(ctx context.Context) error {
-	log.Debugf("building-routes", log.Fields{"logical-device-id": agent.logicalDeviceID})
+	log.Debugw("building-routes", log.Fields{"logical-device-id": agent.logicalDeviceID})
 	agent.lockLogicalDevice.Lock()
 	defer agent.lockLogicalDevice.Unlock()
 	if agent.deviceRoutes == nil {
diff --git a/rw_core/route/device_route.go b/rw_core/route/device_route.go
index b80c20c..c22ff2a 100644
--- a/rw_core/route/device_route.go
+++ b/rw_core/route/device_route.go
@@ -143,6 +143,7 @@
 		}
 		for _, rootDevicePort := range rootDevice.Ports {
 			if rootDevicePort.Type == voltha.Port_PON_OLT {
+				log.Debugw("peers", log.Fields{"root-device-id": rootDevice.Id, "port-no": rootDevicePort.PortNo, "len-peers": len(rootDevicePort.Peers)})
 				for _, rootDevicePeer := range rootDevicePort.Peers {
 					childDevice, err = dr.getDevice(ctx, rootDevicePeer.DeviceId)
 					if err != nil {