VOL-4511: Fix logical Meters are not deleted from the kv store after olt
device delete some times - part 3
- Delay logical device deletion from model until logical meters are cleared
- During meter updates have additional checks to see if the logical device
  exists in model.

Change-Id: I09788b6cdc00779c5f9f5de0e691de79608262fb
diff --git a/rw_core/core/device/logical_agent.go b/rw_core/core/device/logical_agent.go
index bcc9aef..b0eaf38 100644
--- a/rw_core/core/device/logical_agent.go
+++ b/rw_core/core/device/logical_agent.go
@@ -204,6 +204,11 @@
 				log.Fields{"error": err, "logical-device-id": agent.logicalDeviceID})
 		}
 
+		if err := agent.ldeviceMgr.deleteAllLogicalMetersForLogicalDevice(ctx, agent.logicalDeviceID); err != nil {
+			// Just log the error.   The logical device or port may already have been deleted before this callback is invoked.
+			logger.Warnw(ctx, "delete-logical-meters-error", log.Fields{"device-id": agent.logicalDeviceID, "error": err})
+		}
+
 		//Remove the logical device from the model
 		if err := agent.ldProxy.Remove(ctx, agent.logicalDeviceID); err != nil {
 			returnErr = err
@@ -211,11 +216,6 @@
 			logger.Debugw(ctx, "logical-device-removed", log.Fields{"logical-device-id": agent.logicalDeviceID})
 		}
 
-		if err := agent.ldeviceMgr.deleteAllLogicalMeters(ctx, agent.logicalDeviceID); err != nil {
-			// Just log the error.   The logical device or port may already have been deleted before this callback is invoked.
-			logger.Warnw(ctx, "delete-logical-meters-error", log.Fields{"device-id": agent.logicalDeviceID, "error": err})
-		}
-
 		// TODO: remove all entries from all loaders
 		// TODO: don't allow any more modifications to flows/groups/meters/ports or to any logical device field
 
diff --git a/rw_core/core/device/logical_agent_meter.go b/rw_core/core/device/logical_agent_meter.go
index a82052a..d6e949a 100644
--- a/rw_core/core/device/logical_agent_meter.go
+++ b/rw_core/core/device/logical_agent_meter.go
@@ -59,9 +59,11 @@
 	}
 	defer agent.requestQueue.RequestComplete()
 
-	// If the logical-device-agent is stopped, return
 	if agent.stopped {
-		logger.Warnw(ctx, "logical-agent-stopped-not-handling-meter", log.Fields{"logical-device-id": agent.logicalDeviceID})
+		logger.Warnw(ctx, "logical-device-already-stopped-not-handling-meter", log.Fields{"logical-device-id": agent.logicalDeviceID})
+		return nil
+	} else if _, err := agent.ldeviceMgr.getLogicalDeviceFromModel(ctx, agent.logicalDeviceID); err != nil {
+		logger.Warnw(ctx, "logical-device-already-removed-not-handling-meter", log.Fields{"logical-device-id": agent.logicalDeviceID})
 		return nil
 	}
 
diff --git a/rw_core/core/device/logical_manager.go b/rw_core/core/device/logical_manager.go
index 5e2187d..3f08195 100644
--- a/rw_core/core/device/logical_manager.go
+++ b/rw_core/core/device/logical_manager.go
@@ -424,15 +424,10 @@
 	return nil
 }
 
-// deleteAllLogicalMeters removes the logical meters associated with a child device
-func (ldMgr *LogicalManager) deleteAllLogicalMeters(ctx context.Context, deviceID string) error {
-	logger.Debugw(ctx, "delete-logical-meters", log.Fields{"device-id": deviceID})
-	// Get logical port
-	ldID, err := ldMgr.getLogicalDeviceIDFromDeviceID(ctx, deviceID)
-	if err != nil {
-		return err
-	}
-	if agent := ldMgr.getLogicalDeviceAgent(ctx, *ldID); agent != nil {
+// deleteAllLogicalMetersForLogicalDevice removes the logical meters associated with a the Logical Device ID
+func (ldMgr *LogicalManager) deleteAllLogicalMetersForLogicalDevice(ctx context.Context, ldID string) error {
+	logger.Debugw(ctx, "delete-logical-meters", log.Fields{"logical-device-id": ldID})
+	if agent := ldMgr.getLogicalDeviceAgent(ctx, ldID); agent != nil {
 		for meterID := range agent.meterLoader.ListIDs() {
 			if meterHandle, have := agent.meterLoader.Lock(meterID); have {
 				// Update the store and cache
diff --git a/rw_core/core/device/manager_state_callback.go b/rw_core/core/device/manager_state_callback.go
index 53ce6af..b833250 100644
--- a/rw_core/core/device/manager_state_callback.go
+++ b/rw_core/core/device/manager_state_callback.go
@@ -178,7 +178,12 @@
 
 func (dMgr *Manager) DeleteAllLogicalMeters(ctx context.Context, cDevice *voltha.Device) error {
 	logger.Debugw(ctx, "delete-all-logical-device-meters", log.Fields{"device-id": cDevice.Id})
-	if err := dMgr.logicalDeviceMgr.deleteAllLogicalMeters(ctx, cDevice.Id); err != nil {
+	// Get logical device id
+	ldID, err := dMgr.logicalDeviceMgr.getLogicalDeviceIDFromDeviceID(ctx, cDevice.Id)
+	if err != nil {
+		return err
+	}
+	if err := dMgr.logicalDeviceMgr.deleteAllLogicalMetersForLogicalDevice(ctx, *ldID); err != nil {
 		// Just log the error.   The logical device or port may already have been deleted before this callback is invoked.
 		logger.Warnw(ctx, "delete-logical-meters-error", log.Fields{"device-id": cDevice.Id, "error": err})
 	}