[VOL-4428] openonu-adapter-go - rw-core is using delete force when ONUS are in reconciling and delete OLT is issued

As suggested in Slack - all three related patches should be merged and then thoroughly tested.

Change-Id: I8fc6176211a0c0ba7be546950465cca86320b7bd
diff --git a/VERSION b/VERSION
index b6e9f71..181df10 100755
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.0.3-dev244
+2.0.3-dev245
diff --git a/internal/pkg/common/defines.go b/internal/pkg/common/defines.go
index 8f47cb0..e0e44db 100755
--- a/internal/pkg/common/defines.go
+++ b/internal/pkg/common/defines.go
@@ -332,3 +332,8 @@
 	OnugSerialNumberLen  = 8
 	OmciMacAddressLen    = 6
 )
+
+///////////////////////////////////////////////////////////
+
+// CBasePathOnuKVStore - kv store path of ONU specific data
+const CBasePathOnuKVStore = "%s/openonu"
diff --git a/internal/pkg/core/openonu.go b/internal/pkg/core/openonu.go
index 83ed30f..230b7c5 100755
--- a/internal/pkg/core/openonu.go
+++ b/internal/pkg/core/openonu.go
@@ -24,6 +24,7 @@
 	"sync"
 	"time"
 
+	"github.com/opencord/voltha-lib-go/v7/pkg/db"
 	vgrpc "github.com/opencord/voltha-lib-go/v7/pkg/grpc"
 
 	conf "github.com/opencord/voltha-lib-go/v7/pkg/config"
@@ -44,10 +45,13 @@
 
 	cmn "github.com/opencord/voltha-openonu-adapter-go/internal/pkg/common"
 	"github.com/opencord/voltha-openonu-adapter-go/internal/pkg/config"
+	pmmgr "github.com/opencord/voltha-openonu-adapter-go/internal/pkg/pmmgr"
 	"github.com/opencord/voltha-openonu-adapter-go/internal/pkg/swupg"
 	uniprt "github.com/opencord/voltha-openonu-adapter-go/internal/pkg/uniprt"
 )
 
+var onuKvStorePathPrefixes = []string{cmn.CBasePathOnuKVStore, pmmgr.CPmKvStorePrefixBase}
+
 //OpenONUAC structure holds the ONU core information
 type OpenONUAC struct {
 	deviceHandlers              map[string]*deviceHandler
@@ -314,27 +318,23 @@
 	if handler := oo.getDeviceHandler(ctx, device.Id, false); handler != nil {
 		var errorsList []error
 
+		handler.StopReconciling(ctx, false)
+
 		handler.mutexDeletionInProgressFlag.Lock()
 		handler.deletionInProgress = true
 		handler.mutexDeletionInProgressFlag.Unlock()
 
+		if err := handler.resetFsms(ctx, true); err != nil {
+			errorsList = append(errorsList, err)
+		}
 		if err := handler.deleteDevicePersistencyData(ctx); err != nil {
 			errorsList = append(errorsList, err)
 		}
-
-		// Stop PM, Alarm and Self Test event handler routines
-		if handler.GetCollectorIsRunning() {
-			handler.stopCollector <- true
-			logger.Debugw(ctx, "sent stop signal to metric collector routine", log.Fields{"device-id": device.Id})
-
-		}
-		if handler.GetAlarmManagerIsRunning(ctx) {
-			handler.stopAlarmManager <- true
-			logger.Debugw(ctx, "sent stop signal to alarm manager", log.Fields{"device-id": device.Id})
-		}
-		if handler.pSelfTestHdlr.GetSelfTestHandlerIsRunning() {
-			handler.pSelfTestHdlr.StopSelfTestModule <- true
-			logger.Debugw(ctx, "sent stop signal to self test handler module", log.Fields{"device-id": device.Id})
+		// Clear PM data on the KV store
+		if handler.pOnuMetricsMgr != nil {
+			if err := handler.pOnuMetricsMgr.ClearAllPmData(ctx); err != nil {
+				errorsList = append(errorsList, err)
+			}
 		}
 		for _, uni := range handler.uniEntityMap {
 			if handler.GetFlowMonitoringIsRunning(uni.UniID) {
@@ -342,24 +342,39 @@
 				logger.Debugw(ctx, "sent stop signal to self flow monitoring routine", log.Fields{"device-id": device.Id})
 			}
 		}
-
-		// Clear PM data on the KV store
-		if handler.pOnuMetricsMgr != nil {
-			if err := handler.pOnuMetricsMgr.ClearAllPmData(ctx); err != nil {
-				errorsList = append(errorsList, err)
-			}
-		}
-
 		//don't leave any garbage - even in error case
 		oo.deleteDeviceHandlerToMap(handler)
+
 		if len(errorsList) > 0 {
 			logger.Errorw(ctx, "one-or-more-error-during-device-delete", log.Fields{"device-id": device.Id})
 			return nil, fmt.Errorf("one-or-more-error-during-device-delete, errors:%v", errorsList)
 		}
 		return &empty.Empty{}, nil
 	}
-	logger.Warnw(ctx, "no handler found for device-deletion", log.Fields{"device-id": device.Id})
-	return nil, fmt.Errorf(fmt.Sprintf("handler-not-found-%s", device.Id))
+	logger.Infow(ctx, "no handler found for device-deletion - trying to delete remaining data in the kv-store ", log.Fields{"device-id": device.Id})
+
+	// delete ONU specific avcfg and pm data in kv store
+	for i := range onuKvStorePathPrefixes {
+		baseKvStorePath := fmt.Sprintf(onuKvStorePathPrefixes[i], oo.cm.Backend.PathPrefix)
+		logger.Debugw(ctx, "SetKVStoreBackend", log.Fields{"IpTarget": oo.KVStoreAddress, "BasePathKvStore": baseKvStorePath})
+		kvbackend := &db.Backend{
+			Client:     oo.kvClient,
+			StoreType:  oo.KVStoreType,
+			Address:    oo.KVStoreAddress,
+			Timeout:    oo.KVStoreTimeout,
+			PathPrefix: baseKvStorePath}
+
+		if kvbackend == nil {
+			logger.Errorw(ctx, "Can't access onuKVStore - no backend connection to service", log.Fields{"service": baseKvStorePath, "device-id": device.Id})
+			return nil, fmt.Errorf("can-not-access-onuKVStore-no-backend-connection-to-service")
+		}
+		err := kvbackend.Delete(ctx, device.Id)
+		if err != nil {
+			logger.Errorw(ctx, "unable to delete in KVstore", log.Fields{"service": baseKvStorePath, "device-id": device.Id, "err": err})
+			return nil, fmt.Errorf("unable-to-delete-in-KVstore")
+		}
+	}
+	return &empty.Empty{}, nil
 }
 
 //UpdateFlowsIncrementally updates (add/remove) the flows on a given device
diff --git a/internal/pkg/mib/onu_device_entry.go b/internal/pkg/mib/onu_device_entry.go
index c35377f..61e852d 100755
--- a/internal/pkg/mib/onu_device_entry.go
+++ b/internal/pkg/mib/onu_device_entry.go
@@ -116,7 +116,6 @@
 	// NOTE that this hardcoded to service/voltha as the MIB template is shared across stacks
 	cBasePathMibTemplateKvStore = "service/voltha/omci_mibs/go_templates"
 	cSuffixMibTemplateKvStore   = "%s/%s/%s"
-	cBasePathOnuKVStore         = "%s/openonu"
 )
 
 const cEmptyMacAddrString = "000000000000"
@@ -394,7 +393,7 @@
 	}
 
 	onuDeviceEntry.onuKVStorePath = onuDeviceEntry.deviceID
-	baseKvStorePath := fmt.Sprintf(cBasePathOnuKVStore, dh.GetBackendPathPrefix())
+	baseKvStorePath := fmt.Sprintf(cmn.CBasePathOnuKVStore, dh.GetBackendPathPrefix())
 	onuDeviceEntry.onuKVStore = onuDeviceEntry.baseDeviceHandler.SetBackend(ctx, baseKvStorePath)
 	if onuDeviceEntry.onuKVStore == nil {
 		logger.Errorw(ctx, "Can't access onuKVStore - no backend connection to service",
diff --git a/internal/pkg/pmmgr/onu_metrics_manager.go b/internal/pkg/pmmgr/onu_metrics_manager.go
index a71beed..f78e52b 100755
--- a/internal/pkg/pmmgr/onu_metrics_manager.go
+++ b/internal/pkg/pmmgr/onu_metrics_manager.go
@@ -255,9 +255,12 @@
 	GemPortHistoryFrequency = L2PmCollectionInterval
 )
 
+// CPmKvStorePrefixBase - kv store base path of ONU specific PM data
+const CPmKvStorePrefixBase = cmn.CBasePathOnuKVStore + "/pm-data" // <some-base-path>/openonu/pm-data
+
 // KV Store related constants
 const (
-	cPmKvStorePrefix    = "%s/openonu/pm-data/%s" // <some-base-path>/openonu/pm-data/<onu-device-id>
+	cPmKvStorePrefix    = CPmKvStorePrefixBase + "/%s" // <some-base-path>/openonu/pm-data/<onu-device-id>
 	cPmAdd              = "add"
 	cPmAdded            = "added"
 	cPmRemove           = "remove"