[VOL-3731] Delete etcd stale data after olt reboot

Change-Id: I345b1af33de943c795888ad2defe2e470b6ea2c4
diff --git a/VERSION b/VERSION
index 93ea0c9..1acd4da 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.9.4
+2.9.5
diff --git a/go.mod b/go.mod
index 0ba5a2a..f2e11e6 100644
--- a/go.mod
+++ b/go.mod
@@ -8,7 +8,7 @@
 	github.com/golang/mock v1.5.0
 	github.com/golang/protobuf v1.3.2
 	github.com/google/uuid v1.1.1
-	github.com/opencord/voltha-lib-go/v5 v5.0.5
+	github.com/opencord/voltha-lib-go/v5 v5.0.6
 	github.com/opencord/voltha-protos/v4 v4.2.0
 	github.com/opentracing/opentracing-go v1.1.0
 	github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2
diff --git a/go.sum b/go.sum
index 5d75105..8a35b97 100644
--- a/go.sum
+++ b/go.sum
@@ -143,8 +143,8 @@
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.4.2 h1:3mYCb7aPxS/RU7TI1y4rkEn1oKmPRjNJLNEXgw7MH2I=
 github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/opencord/voltha-lib-go/v5 v5.0.5 h1:S3m984EaHNGJY68Hrv9LOf0rwd92UiVLFBBbTaL+fNg=
-github.com/opencord/voltha-lib-go/v5 v5.0.5/go.mod h1:i1fwPMicFccG38L200+IQAlfHSbszWg//jF1pDQxTPQ=
+github.com/opencord/voltha-lib-go/v5 v5.0.6 h1:vJ6ZTNSjOiWAViCT5BfxEgv596d4KBov28a3rE5DDPI=
+github.com/opencord/voltha-lib-go/v5 v5.0.6/go.mod h1:i1fwPMicFccG38L200+IQAlfHSbszWg//jF1pDQxTPQ=
 github.com/opencord/voltha-protos/v4 v4.2.0 h1:QJZqHPRKa1E1xh40F3UA4xSjBI+6EmW7OfIcJqPNc4A=
 github.com/opencord/voltha-protos/v4 v4.2.0/go.mod h1:wNzWqmTwe7+DbYbpmOX6eMlglREtMkNxIDv3lyI2bco=
 github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU=
diff --git a/rw_core/core/device/agent_port.go b/rw_core/core/device/agent_port.go
index f9bf7de..004ab8e 100644
--- a/rw_core/core/device/agent_port.go
+++ b/rw_core/core/device/agent_port.go
@@ -127,7 +127,7 @@
 		return err
 	}
 
-	if device.AdminState != voltha.AdminState_DISABLED && !agent.isDeletionInProgress() {
+	if !agent.isDeletionInProgress() {
 		err := status.Error(codes.FailedPrecondition, fmt.Sprintf("invalid-admin-state-%v",
 			device.AdminState))
 		logger.Warnw(ctx, "invalid-state-removing-ports", log.Fields{"state": device.AdminState, "error": err})
diff --git a/rw_core/core/device/logical_manager.go b/rw_core/core/device/logical_manager.go
index 26021ca..17ffe9f 100644
--- a/rw_core/core/device/logical_manager.go
+++ b/rw_core/core/device/logical_manager.go
@@ -19,13 +19,14 @@
 import (
 	"context"
 	"errors"
-	"github.com/opencord/voltha-lib-go/v5/pkg/probe"
 	"io"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/opencord/voltha-lib-go/v5/pkg/probe"
+
 	"github.com/golang/protobuf/ptypes/empty"
 	"github.com/opencord/voltha-go/db/model"
 	"github.com/opencord/voltha-go/rw_core/core/device/event"
@@ -421,6 +422,30 @@
 	return nil
 }
 
+// deleteAllLogicalMeters removes the logical meters associated with a child device
+func (ldMgr *LogicalManager) deleteAllLogicalMeters(ctx context.Context, deviceID string) error {
+	logger.Debugw(ctx, "delete-logical-meters", log.Fields{"device-id": deviceID})
+	// Get logical port
+	ldID, err := ldMgr.getLogicalDeviceIDFromDeviceID(ctx, deviceID)
+	if err != nil {
+		return err
+	}
+	if agent := ldMgr.getLogicalDeviceAgent(ctx, *ldID); agent != nil {
+		for meterID := range agent.meterLoader.ListIDs() {
+			if meterHandle, have := agent.meterLoader.Lock(meterID); have {
+				// Update the store and cache
+				if err := meterHandle.Delete(ctx); err != nil {
+					meterHandle.Unlock()
+					logger.Errorw(ctx, "unable-to-delete-meter", log.Fields{"logical-device-id": ldID, "meterID": meterID})
+					continue
+				}
+				meterHandle.Unlock()
+			}
+		}
+	}
+	return nil
+}
+
 func (ldMgr *LogicalManager) setupUNILogicalPorts(ctx context.Context, childDevice *voltha.Device, childDevicePorts map[uint32]*voltha.Port) error {
 	logger.Debugw(ctx, "setup-uni-logical-ports", log.Fields{"child-device-id": childDevice.Id, "parent-device-id": childDevice.ParentId, "current-data": childDevice})
 	// Sanity check
diff --git a/rw_core/core/device/manager.go b/rw_core/core/device/manager.go
index 75435b1..869191e 100755
--- a/rw_core/core/device/manager.go
+++ b/rw_core/core/device/manager.go
@@ -19,11 +19,12 @@
 import (
 	"context"
 	"errors"
-	"github.com/opencord/voltha-go/rw_core/config"
-	"github.com/opencord/voltha-lib-go/v5/pkg/probe"
 	"sync"
 	"time"
 
+	"github.com/opencord/voltha-go/rw_core/config"
+	"github.com/opencord/voltha-lib-go/v5/pkg/probe"
+
 	"github.com/golang/protobuf/ptypes/empty"
 	"github.com/opencord/voltha-go/db/model"
 	"github.com/opencord/voltha-go/rw_core/core/adapter"
@@ -353,6 +354,13 @@
 // RunPostDeviceDelete removes any reference of this device
 func (dMgr *Manager) RunPostDeviceDelete(ctx context.Context, cDevice *voltha.Device) error {
 	logger.Infow(ctx, "run-post-device-delete", log.Fields{"device-id": cDevice.Id})
+	if agent := dMgr.getDeviceAgent(ctx, cDevice.Id); agent != nil {
+		logger.Debugw(ctx, "invoking-delete-device-ports", log.Fields{"device-id": cDevice.Id})
+		//delete ports
+		if err := agent.deleteAllPorts(ctx); err != nil {
+			logger.Warnw(ctx, "failure-delete-device-ports", log.Fields{"device-id": cDevice.Id, "error": err.Error()})
+		}
+	}
 	dMgr.stopManagingDevice(ctx, cDevice.Id)
 	return nil
 }
@@ -1147,9 +1155,7 @@
 // DeleteLogicalDevice deletes logical device from core
 func (dMgr *Manager) DeleteLogicalDevice(ctx context.Context, cDevice *voltha.Device) error {
 	logger.Info(ctx, "delete-logical-device")
-	var err error
-	if err = dMgr.logicalDeviceMgr.deleteLogicalDevice(ctx, cDevice); err != nil {
-		logger.Warnw(ctx, "delete-logical-device-error", log.Fields{"device-id": cDevice.Id})
+	if err := dMgr.logicalDeviceMgr.deleteLogicalDevice(ctx, cDevice); err != nil {
 		return err
 	}
 	// Remove the logical device Id from the parent device
@@ -2099,3 +2105,13 @@
 	}
 	return nil
 }
+
+func (dMgr *Manager) DeleteAllLogicalMeters(ctx context.Context, cDevice *voltha.Device) error {
+	logger.Debugw(ctx, "delete-all-logical-device-meters", log.Fields{"device-id": cDevice.Id})
+	if err := dMgr.logicalDeviceMgr.deleteAllLogicalMeters(ctx, cDevice.Id); err != nil {
+		// Just log the error.   The logical device or port may already have been deleted before this callback is invoked.
+		logger.Warnw(ctx, "delete-logical-ports-error", log.Fields{"device-id": cDevice.Id, "error": err})
+	}
+	return nil
+
+}
diff --git a/rw_core/core/device/state/transitions.go b/rw_core/core/device/state/transitions.go
index c3b98e7..7a28a83 100644
--- a/rw_core/core/device/state/transitions.go
+++ b/rw_core/core/device/state/transitions.go
@@ -18,10 +18,11 @@
 
 import (
 	"context"
-	"github.com/opencord/voltha-lib-go/v5/pkg/log"
-	"github.com/opencord/voltha-protos/v4/go/voltha"
 	"reflect"
 	"runtime"
+
+	"github.com/opencord/voltha-lib-go/v5/pkg/log"
+	"github.com/opencord/voltha-protos/v4/go/voltha"
 )
 
 // deviceType mentions type of device like parent, child
@@ -99,6 +100,7 @@
 	RunPostDeviceDelete(ctx context.Context, curr *voltha.Device) error
 	ChildDeviceLost(ctx context.Context, curr *voltha.Device) error
 	DeleteAllLogicalPorts(ctx context.Context, curr *voltha.Device) error
+	DeleteAllLogicalMeters(ctx context.Context, curr *voltha.Device) error
 	DeleteAllDeviceFlows(ctx context.Context, curr *voltha.Device) error
 	ReconcilingCleanup(ctx context.Context, curr *voltha.Device) error
 }
@@ -156,13 +158,13 @@
 			deviceType:    parent,
 			previousState: deviceState{Admin: voltha.AdminState_UNKNOWN, Connection: voltha.ConnectStatus_UNKNOWN, Operational: voltha.OperStatus_UNKNOWN, Transient: voltha.DeviceTransientState_ANY},
 			currentState:  deviceState{Admin: voltha.AdminState_UNKNOWN, Connection: voltha.ConnectStatus_UNKNOWN, Operational: voltha.OperStatus_UNKNOWN, Transient: voltha.DeviceTransientState_FORCE_DELETING},
-			handlers:      []transitionHandler{dMgr.DeleteAllLogicalPorts, dMgr.DeleteAllChildDevices, dMgr.DeleteLogicalDevice, dMgr.RunPostDeviceDelete}})
+			handlers:      []transitionHandler{dMgr.DeleteAllLogicalPorts, dMgr.DeleteAllLogicalMeters, dMgr.DeleteAllChildDevices, dMgr.DeleteLogicalDevice, dMgr.RunPostDeviceDelete}})
 	transitionMap.transitions = append(transitionMap.transitions,
 		transition{ //DELETE device after adapter response
 			deviceType:    parent,
 			previousState: deviceState{Admin: voltha.AdminState_UNKNOWN, Connection: voltha.ConnectStatus_UNKNOWN, Operational: voltha.OperStatus_UNKNOWN, Transient: voltha.DeviceTransientState_ANY},
 			currentState:  deviceState{Admin: voltha.AdminState_UNKNOWN, Connection: voltha.ConnectStatus_UNKNOWN, Operational: voltha.OperStatus_UNKNOWN, Transient: voltha.DeviceTransientState_DELETING_POST_ADAPTER_RESPONSE},
-			handlers:      []transitionHandler{dMgr.DeleteAllLogicalPorts, dMgr.DeleteAllChildDevices, dMgr.DeleteLogicalDevice, dMgr.RunPostDeviceDelete}})
+			handlers:      []transitionHandler{dMgr.DeleteAllLogicalPorts, dMgr.DeleteAllLogicalMeters, dMgr.DeleteAllChildDevices, dMgr.DeleteLogicalDevice, dMgr.RunPostDeviceDelete}})
 	transitionMap.transitions = append(transitionMap.transitions,
 		transition{ //DELETE no operation transition
 			deviceType:    parent,
@@ -174,13 +176,13 @@
 			deviceType:    parent,
 			previousState: deviceState{Admin: voltha.AdminState_ENABLED, Connection: voltha.ConnectStatus_REACHABLE, Operational: voltha.OperStatus_ACTIVE, Transient: voltha.DeviceTransientState_NONE},
 			currentState:  deviceState{Admin: voltha.AdminState_ENABLED, Connection: voltha.ConnectStatus_UNREACHABLE, Operational: voltha.OperStatus_UNKNOWN, Transient: voltha.DeviceTransientState_NONE},
-			handlers:      []transitionHandler{dMgr.DeleteAllLogicalPorts, dMgr.DeleteAllChildDevices, dMgr.DeleteLogicalDevice, dMgr.DeleteAllDeviceFlows}})
+			handlers:      []transitionHandler{dMgr.DeleteAllLogicalPorts, dMgr.DeleteAllLogicalMeters, dMgr.DeleteAllChildDevices, dMgr.DeleteLogicalDevice, dMgr.DeleteAllDeviceFlows}})
 	transitionMap.transitions = append(transitionMap.transitions,
 		transition{
 			deviceType:    parent,
 			previousState: deviceState{Admin: voltha.AdminState_DISABLED, Connection: voltha.ConnectStatus_REACHABLE, Operational: voltha.OperStatus_UNKNOWN, Transient: voltha.DeviceTransientState_NONE},
 			currentState:  deviceState{Admin: voltha.AdminState_DISABLED, Connection: voltha.ConnectStatus_UNREACHABLE, Operational: voltha.OperStatus_UNKNOWN, Transient: voltha.DeviceTransientState_NONE},
-			handlers:      []transitionHandler{dMgr.DeleteAllLogicalPorts, dMgr.DeleteAllChildDevices, dMgr.DeleteLogicalDevice, dMgr.DeleteAllDeviceFlows}})
+			handlers:      []transitionHandler{dMgr.DeleteAllLogicalPorts, dMgr.DeleteAllLogicalMeters, dMgr.DeleteAllChildDevices, dMgr.DeleteLogicalDevice, dMgr.DeleteAllDeviceFlows}})
 	transitionMap.transitions = append(transitionMap.transitions,
 		transition{
 			deviceType:    parent,
diff --git a/rw_core/core/device/state/transitions_test.go b/rw_core/core/device/state/transitions_test.go
index ad53526..69a7568 100644
--- a/rw_core/core/device/state/transitions_test.go
+++ b/rw_core/core/device/state/transitions_test.go
@@ -206,11 +206,12 @@
 	device = getDevice(true, voltha.AdminState_ENABLED, voltha.ConnectStatus_UNREACHABLE, voltha.OperStatus_UNKNOWN)
 	handlers = transitionMap.getTransitionHandler(ctx, device, previousDevice, voltha.DeviceTransientState_NONE,
 		voltha.DeviceTransientState_NONE)
-	assert.Equal(t, 4, len(handlers))
+	assert.Equal(t, 5, len(handlers))
 	assert.True(t, reflect.ValueOf(tdm.DeleteAllLogicalPorts).Pointer() == reflect.ValueOf(handlers[0]).Pointer())
-	assert.True(t, reflect.ValueOf(tdm.DeleteAllChildDevices).Pointer() == reflect.ValueOf(handlers[1]).Pointer())
-	assert.True(t, reflect.ValueOf(tdm.DeleteLogicalDevice).Pointer() == reflect.ValueOf(handlers[2]).Pointer())
-	assert.True(t, reflect.ValueOf(tdm.DeleteAllDeviceFlows).Pointer() == reflect.ValueOf(handlers[3]).Pointer())
+	assert.True(t, reflect.ValueOf(tdm.DeleteAllLogicalMeters).Pointer() == reflect.ValueOf(handlers[1]).Pointer())
+	assert.True(t, reflect.ValueOf(tdm.DeleteAllChildDevices).Pointer() == reflect.ValueOf(handlers[2]).Pointer())
+	assert.True(t, reflect.ValueOf(tdm.DeleteLogicalDevice).Pointer() == reflect.ValueOf(handlers[3]).Pointer())
+	assert.True(t, reflect.ValueOf(tdm.DeleteAllDeviceFlows).Pointer() == reflect.ValueOf(handlers[4]).Pointer())
 
 	previousDevice = getDevice(true, voltha.AdminState_ENABLED, voltha.ConnectStatus_UNREACHABLE, voltha.OperStatus_UNKNOWN)
 	device = getDevice(true, voltha.AdminState_ENABLED, voltha.ConnectStatus_REACHABLE, voltha.OperStatus_ACTIVE)
@@ -223,11 +224,12 @@
 	device = getDevice(true, voltha.AdminState_DISABLED, voltha.ConnectStatus_UNREACHABLE, voltha.OperStatus_UNKNOWN)
 	handlers = transitionMap.getTransitionHandler(ctx, device, previousDevice, voltha.DeviceTransientState_NONE,
 		voltha.DeviceTransientState_NONE)
-	assert.Equal(t, 4, len(handlers))
+	assert.Equal(t, 5, len(handlers))
 	assert.True(t, reflect.ValueOf(tdm.DeleteAllLogicalPorts).Pointer() == reflect.ValueOf(handlers[0]).Pointer())
-	assert.True(t, reflect.ValueOf(tdm.DeleteAllChildDevices).Pointer() == reflect.ValueOf(handlers[1]).Pointer())
-	assert.True(t, reflect.ValueOf(tdm.DeleteLogicalDevice).Pointer() == reflect.ValueOf(handlers[2]).Pointer())
-	assert.True(t, reflect.ValueOf(tdm.DeleteAllDeviceFlows).Pointer() == reflect.ValueOf(handlers[3]).Pointer())
+	assert.True(t, reflect.ValueOf(tdm.DeleteAllLogicalMeters).Pointer() == reflect.ValueOf(handlers[1]).Pointer())
+	assert.True(t, reflect.ValueOf(tdm.DeleteAllChildDevices).Pointer() == reflect.ValueOf(handlers[2]).Pointer())
+	assert.True(t, reflect.ValueOf(tdm.DeleteLogicalDevice).Pointer() == reflect.ValueOf(handlers[3]).Pointer())
+	assert.True(t, reflect.ValueOf(tdm.DeleteAllDeviceFlows).Pointer() == reflect.ValueOf(handlers[4]).Pointer())
 
 	previousDevice = getDevice(true, voltha.AdminState_DISABLED, voltha.ConnectStatus_UNREACHABLE, voltha.OperStatus_UNKNOWN)
 	device = getDevice(true, voltha.AdminState_DISABLED, voltha.ConnectStatus_REACHABLE, voltha.OperStatus_UNKNOWN)
@@ -328,6 +330,7 @@
 		},
 		expectedParentHandlers: []transitionHandler{
 			tdm.DeleteAllLogicalPorts,
+			tdm.DeleteAllLogicalMeters,
 			tdm.DeleteAllChildDevices,
 			tdm.DeleteLogicalDevice,
 			tdm.RunPostDeviceDelete,
@@ -346,7 +349,7 @@
 			t.Run(testName, func(t *testing.T) {
 				handlers = transitionMap.getTransitionHandler(ctx, device, previousDevice, voltha.DeviceTransientState_FORCE_DELETING,
 					voltha.DeviceTransientState_ANY)
-				assert.Equal(t, 4, len(handlers))
+				assert.Equal(t, 5, len(handlers))
 				for idx, expHandler := range deleteDeviceTest.expectedParentHandlers {
 					assert.True(t, reflect.ValueOf(expHandler).Pointer() == reflect.ValueOf(handlers[idx]).Pointer())
 				}
diff --git a/vendor/modules.txt b/vendor/modules.txt
index 68cf1c4..0cda7dd 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -93,7 +93,7 @@
 github.com/modern-go/concurrent
 # github.com/modern-go/reflect2 v1.0.1
 github.com/modern-go/reflect2
-# github.com/opencord/voltha-lib-go/v5 v5.0.5
+# github.com/opencord/voltha-lib-go/v5 v5.0.6
 ## explicit
 github.com/opencord/voltha-lib-go/v5/pkg/adapters
 github.com/opencord/voltha-lib-go/v5/pkg/adapters/adapterif