[VOL-4428] openonu-adapter-go - rw-core is using delete force when ONUS are in reconciling and delete OLT is issued

As suggested in Slack - all three related patches should be merged and then thoroughly tested.

Change-Id: I8fc6176211a0c0ba7be546950465cca86320b7bd
diff --git a/internal/pkg/core/openonu.go b/internal/pkg/core/openonu.go
index 83ed30f..230b7c5 100755
--- a/internal/pkg/core/openonu.go
+++ b/internal/pkg/core/openonu.go
@@ -24,6 +24,7 @@
 	"sync"
 	"time"
 
+	"github.com/opencord/voltha-lib-go/v7/pkg/db"
 	vgrpc "github.com/opencord/voltha-lib-go/v7/pkg/grpc"
 
 	conf "github.com/opencord/voltha-lib-go/v7/pkg/config"
@@ -44,10 +45,13 @@
 
 	cmn "github.com/opencord/voltha-openonu-adapter-go/internal/pkg/common"
 	"github.com/opencord/voltha-openonu-adapter-go/internal/pkg/config"
+	pmmgr "github.com/opencord/voltha-openonu-adapter-go/internal/pkg/pmmgr"
 	"github.com/opencord/voltha-openonu-adapter-go/internal/pkg/swupg"
 	uniprt "github.com/opencord/voltha-openonu-adapter-go/internal/pkg/uniprt"
 )
 
+var onuKvStorePathPrefixes = []string{cmn.CBasePathOnuKVStore, pmmgr.CPmKvStorePrefixBase}
+
 //OpenONUAC structure holds the ONU core information
 type OpenONUAC struct {
 	deviceHandlers              map[string]*deviceHandler
@@ -314,27 +318,23 @@
 	if handler := oo.getDeviceHandler(ctx, device.Id, false); handler != nil {
 		var errorsList []error
 
+		handler.StopReconciling(ctx, false)
+
 		handler.mutexDeletionInProgressFlag.Lock()
 		handler.deletionInProgress = true
 		handler.mutexDeletionInProgressFlag.Unlock()
 
+		if err := handler.resetFsms(ctx, true); err != nil {
+			errorsList = append(errorsList, err)
+		}
 		if err := handler.deleteDevicePersistencyData(ctx); err != nil {
 			errorsList = append(errorsList, err)
 		}
-
-		// Stop PM, Alarm and Self Test event handler routines
-		if handler.GetCollectorIsRunning() {
-			handler.stopCollector <- true
-			logger.Debugw(ctx, "sent stop signal to metric collector routine", log.Fields{"device-id": device.Id})
-
-		}
-		if handler.GetAlarmManagerIsRunning(ctx) {
-			handler.stopAlarmManager <- true
-			logger.Debugw(ctx, "sent stop signal to alarm manager", log.Fields{"device-id": device.Id})
-		}
-		if handler.pSelfTestHdlr.GetSelfTestHandlerIsRunning() {
-			handler.pSelfTestHdlr.StopSelfTestModule <- true
-			logger.Debugw(ctx, "sent stop signal to self test handler module", log.Fields{"device-id": device.Id})
+		// Clear PM data on the KV store
+		if handler.pOnuMetricsMgr != nil {
+			if err := handler.pOnuMetricsMgr.ClearAllPmData(ctx); err != nil {
+				errorsList = append(errorsList, err)
+			}
 		}
 		for _, uni := range handler.uniEntityMap {
 			if handler.GetFlowMonitoringIsRunning(uni.UniID) {
@@ -342,24 +342,39 @@
 				logger.Debugw(ctx, "sent stop signal to self flow monitoring routine", log.Fields{"device-id": device.Id})
 			}
 		}
-
-		// Clear PM data on the KV store
-		if handler.pOnuMetricsMgr != nil {
-			if err := handler.pOnuMetricsMgr.ClearAllPmData(ctx); err != nil {
-				errorsList = append(errorsList, err)
-			}
-		}
-
 		//don't leave any garbage - even in error case
 		oo.deleteDeviceHandlerToMap(handler)
+
 		if len(errorsList) > 0 {
 			logger.Errorw(ctx, "one-or-more-error-during-device-delete", log.Fields{"device-id": device.Id})
 			return nil, fmt.Errorf("one-or-more-error-during-device-delete, errors:%v", errorsList)
 		}
 		return &empty.Empty{}, nil
 	}
-	logger.Warnw(ctx, "no handler found for device-deletion", log.Fields{"device-id": device.Id})
-	return nil, fmt.Errorf(fmt.Sprintf("handler-not-found-%s", device.Id))
+	logger.Infow(ctx, "no handler found for device-deletion - trying to delete remaining data in the kv-store ", log.Fields{"device-id": device.Id})
+
+	// delete ONU specific avcfg and pm data in kv store
+	for i := range onuKvStorePathPrefixes {
+		baseKvStorePath := fmt.Sprintf(onuKvStorePathPrefixes[i], oo.cm.Backend.PathPrefix)
+		logger.Debugw(ctx, "SetKVStoreBackend", log.Fields{"IpTarget": oo.KVStoreAddress, "BasePathKvStore": baseKvStorePath})
+		kvbackend := &db.Backend{
+			Client:     oo.kvClient,
+			StoreType:  oo.KVStoreType,
+			Address:    oo.KVStoreAddress,
+			Timeout:    oo.KVStoreTimeout,
+			PathPrefix: baseKvStorePath}
+
+		if kvbackend == nil {
+			logger.Errorw(ctx, "Can't access onuKVStore - no backend connection to service", log.Fields{"service": baseKvStorePath, "device-id": device.Id})
+			return nil, fmt.Errorf("can-not-access-onuKVStore-no-backend-connection-to-service")
+		}
+		err := kvbackend.Delete(ctx, device.Id)
+		if err != nil {
+			logger.Errorw(ctx, "unable to delete in KVstore", log.Fields{"service": baseKvStorePath, "device-id": device.Id, "err": err})
+			return nil, fmt.Errorf("unable-to-delete-in-KVstore")
+		}
+	}
+	return &empty.Empty{}, nil
 }
 
 //UpdateFlowsIncrementally updates (add/remove) the flows on a given device