[VOL-4548] openonuAdapterGo - memory leak seen in long term tests

Change-Id: Ie3eb2048cebf8bf8b2f2a99ccd927d462000223a
diff --git a/internal/pkg/almgr/alarm_manager.go b/internal/pkg/almgr/alarm_manager.go
index fa4bc9f..69ae862 100755
--- a/internal/pkg/almgr/alarm_manager.go
+++ b/internal/pkg/almgr/alarm_manager.go
@@ -800,3 +800,10 @@
 func (am *OnuAlarmManager) GetAlarmMgrEventChannel() chan cmn.Message {
 	return am.eventChannel
 }
+
+// PrepareForGarbageCollection - remove references to prepare for garbage collection
+func (am *OnuAlarmManager) PrepareForGarbageCollection(ctx context.Context, aDeviceID string) {
+	logger.Debugw(ctx, "prepare for garbage collection", log.Fields{"device-id": aDeviceID})
+	am.pDeviceHandler = nil
+	am.pOnuDeviceEntry = nil
+}
diff --git a/internal/pkg/avcfg/omci_ani_config.go b/internal/pkg/avcfg/omci_ani_config.go
index 34ed185..cc023ba 100755
--- a/internal/pkg/avcfg/omci_ani_config.go
+++ b/internal/pkg/avcfg/omci_ani_config.go
@@ -1817,3 +1817,11 @@
 	oFsm.mutexChanSet.RUnlock()
 	return flagValue
 }
+
+// PrepareForGarbageCollection - remove references to prepare for garbage collection
+func (oFsm *UniPonAniConfigFsm) PrepareForGarbageCollection(ctx context.Context, aDeviceID string) {
+	logger.Debugw(ctx, "prepare for garbage collection", log.Fields{"device-id": oFsm.deviceID})
+	oFsm.pDeviceHandler = nil
+	oFsm.pOnuDeviceEntry = nil
+	oFsm.pOmciCC = nil
+}
diff --git a/internal/pkg/avcfg/omci_vlan_config.go b/internal/pkg/avcfg/omci_vlan_config.go
index 668d2c6..e97b0be 100755
--- a/internal/pkg/avcfg/omci_vlan_config.go
+++ b/internal/pkg/avcfg/omci_vlan_config.go
@@ -3229,3 +3229,11 @@
 		}
 	}
 }
+
+// PrepareForGarbageCollection - remove references to prepare for garbage collection
+func (oFsm *UniVlanConfigFsm) PrepareForGarbageCollection(ctx context.Context, aDeviceID string) {
+	logger.Debugw(ctx, "prepare for garbage collection", log.Fields{"device-id": aDeviceID})
+	oFsm.pDeviceHandler = nil
+	oFsm.pOnuDeviceEntry = nil
+	oFsm.pOmciCC = nil
+}
diff --git a/internal/pkg/avcfg/onu_uni_tp.go b/internal/pkg/avcfg/onu_uni_tp.go
index efb69be..c51c489 100755
--- a/internal/pkg/avcfg/onu_uni_tp.go
+++ b/internal/pkg/avcfg/onu_uni_tp.go
@@ -1009,3 +1009,14 @@
 	}
 	return false
 }
+
+// PrepareForGarbageCollection - remove references to prepare for garbage collection
+func (onuTP *OnuUniTechProf) PrepareForGarbageCollection(ctx context.Context, aDeviceID string) {
+	logger.Debugw(ctx, "prepare for garbage collection", log.Fields{"device-id": aDeviceID})
+	onuTP.baseDeviceHandler = nil
+	onuTP.onuDevice = nil
+	for k, v := range onuTP.PAniConfigFsm {
+		v.PrepareForGarbageCollection(ctx, aDeviceID)
+		delete(onuTP.PAniConfigFsm, k)
+	}
+}
diff --git a/internal/pkg/common/omci_cc.go b/internal/pkg/common/omci_cc.go
index 993a84e..57bc0ce 100755
--- a/internal/pkg/common/omci_cc.go
+++ b/internal/pkg/common/omci_cc.go
@@ -4674,3 +4674,11 @@
 		log.Fields{"Err": omciErr.GetError(), "device-id": oo.deviceID, "inst-id": strconv.FormatInt(int64(entityID), 16)})
 	return nil, omciErr.GetError()
 }
+
+// PrepareForGarbageCollection - remove references to prepare for garbage collection
+func (oo *OmciCC) PrepareForGarbageCollection(ctx context.Context, aDeviceID string) {
+	logger.Debugw(ctx, "prepare for garbage collection", log.Fields{"device-id": aDeviceID})
+	oo.pBaseDeviceHandler = nil
+	oo.pOnuDeviceEntry = nil
+	oo.pOnuAlarmManager = nil
+}
diff --git a/internal/pkg/core/device_handler.go b/internal/pkg/core/device_handler.go
index 58d42b1..332861b 100755
--- a/internal/pkg/core/device_handler.go
+++ b/internal/pkg/core/device_handler.go
@@ -4497,3 +4497,50 @@
 	}
 	return tpPathFound
 }
+
+// PrepareForGarbageCollection - remove references to prepare for garbage collection
+func (dh *deviceHandler) PrepareForGarbageCollection(ctx context.Context, aDeviceID string) {
+	logger.Debugw(ctx, "prepare for garbage collection", log.Fields{"device-id": aDeviceID})
+
+	// Note: This function must be called as a goroutine to prevent blocking of further processing!
+	// first let the objects rest for some time to give all asynchronously started
+	// cleanup routines a chance to come to an end
+	time.Sleep(5 * time.Second)
+
+	if dh.pOnuTP != nil {
+		dh.pOnuTP.PrepareForGarbageCollection(ctx, aDeviceID)
+	}
+	if dh.pOnuMetricsMgr != nil {
+		dh.pOnuMetricsMgr.PrepareForGarbageCollection(ctx, aDeviceID)
+	}
+	if dh.pAlarmMgr != nil {
+		dh.pAlarmMgr.PrepareForGarbageCollection(ctx, aDeviceID)
+	}
+	if dh.pSelfTestHdlr != nil {
+		dh.pSelfTestHdlr.PrepareForGarbageCollection(ctx, aDeviceID)
+	}
+	if dh.pLockStateFsm != nil {
+		dh.pLockStateFsm.PrepareForGarbageCollection(ctx, aDeviceID)
+	}
+	if dh.pUnlockStateFsm != nil {
+		dh.pUnlockStateFsm.PrepareForGarbageCollection(ctx, aDeviceID)
+	}
+	if dh.pOnuUpradeFsm != nil {
+		dh.pOnuUpradeFsm.PrepareForGarbageCollection(ctx, aDeviceID)
+	}
+	if dh.pOnuOmciDevice != nil {
+		dh.pOnuOmciDevice.PrepareForGarbageCollection(ctx, aDeviceID)
+	}
+	for k, v := range dh.UniVlanConfigFsmMap {
+		v.PrepareForGarbageCollection(ctx, aDeviceID)
+		delete(dh.UniVlanConfigFsmMap, k)
+	}
+	dh.pOnuOmciDevice = nil
+	dh.pOnuTP = nil
+	dh.pOnuMetricsMgr = nil
+	dh.pAlarmMgr = nil
+	dh.pSelfTestHdlr = nil
+	dh.pLockStateFsm = nil
+	dh.pUnlockStateFsm = nil
+	dh.pOnuUpradeFsm = nil
+}
diff --git a/internal/pkg/core/openonu.go b/internal/pkg/core/openonu.go
index f0da31b..44bfae0 100755
--- a/internal/pkg/core/openonu.go
+++ b/internal/pkg/core/openonu.go
@@ -344,34 +344,18 @@
 		if err := handler.resetFsms(ctx, true); err != nil {
 			errorsList = append(errorsList, err)
 		}
-		forceKvDelete := false
-
-		// Clear PM data on the KV store
-		if handler.pOnuMetricsMgr != nil {
-			if err := handler.pOnuMetricsMgr.ClearAllPmData(ctx); err != nil {
-				errorsList = append(errorsList, err)
-				forceKvDelete = true
-			}
-		} else {
-			forceKvDelete = true
-		}
-		if err := handler.deleteDevicePersistencyData(ctx); err != nil {
-			errorsList = append(errorsList, err)
-			forceKvDelete = true
-		}
 		for _, uni := range handler.uniEntityMap {
 			if handler.GetFlowMonitoringIsRunning(uni.UniID) {
 				handler.stopFlowMonitoringRoutine[uni.UniID] <- true
 				logger.Debugw(ctx, "sent stop signal to self flow monitoring routine", log.Fields{"device-id": device.Id})
 			}
 		}
-		//don't leave any garbage - even in error case
-		oo.deleteDeviceHandlerToMap(handler)
-		if forceKvDelete {
-			if err := oo.forceDeleteDeviceKvData(ctx, device.Id); err != nil {
-				errorsList = append(errorsList, err)
-			}
+		//don't leave any garbage in kv-store
+		if err := oo.forceDeleteDeviceKvData(ctx, device.Id); err != nil {
+			errorsList = append(errorsList, err)
 		}
+		oo.deleteDeviceHandlerToMap(handler)
+		go handler.PrepareForGarbageCollection(ctx, handler.DeviceID)
 
 		if len(errorsList) > 0 {
 			logger.Errorw(ctx, "one-or-more-error-during-device-delete", log.Fields{"device-id": device.Id})
diff --git a/internal/pkg/mib/mib_sync.go b/internal/pkg/mib/mib_sync.go
index d3f2712..5222b8b 100755
--- a/internal/pkg/mib/mib_sync.go
+++ b/internal/pkg/mib/mib_sync.go
@@ -488,6 +488,10 @@
 		switch message.Type {
 		case cmn.TestMsg:
 			msg, _ := message.Data.(cmn.TestMessage)
+			if msg.TestMessageVal == cmn.AbortMessageProcessing {
+				logger.Debugw(ctx, "MibSync Msg abort ProcessMsg", log.Fields{"for device-id": oo.deviceID})
+				break loop
+			}
 			oo.handleTestMsg(ctx, msg)
 		case cmn.OMCI:
 			msg, _ := message.Data.(cmn.OmciMessage)
@@ -1207,8 +1211,16 @@
 	}
 	//the MibSync FSM might be active all the ONU-active time,
 	// hence it must be stopped unconditionally
-	pMibUlFsm := oo.PMibUploadFsm.PFsm
+	pMibUlFsm := oo.PMibUploadFsm
 	if pMibUlFsm != nil {
-		_ = pMibUlFsm.Event(UlEvStop)
+		// abort running message processing
+		fsmAbortMsg := cmn.Message{
+			Type: cmn.TestMsg,
+			Data: cmn.TestMessage{
+				TestMessageVal: cmn.AbortMessageProcessing,
+			},
+		}
+		pMibUlFsm.CommChan <- fsmAbortMsg
+		_ = pMibUlFsm.PFsm.Event(UlEvStop)
 	}
 }
diff --git a/internal/pkg/mib/onu_device_entry.go b/internal/pkg/mib/onu_device_entry.go
index 26c346d..47c7b90 100755
--- a/internal/pkg/mib/onu_device_entry.go
+++ b/internal/pkg/mib/onu_device_entry.go
@@ -986,3 +986,14 @@
 	oo.mutexReconcilingFlowsFlag.RUnlock()
 	return value
 }
+
+// PrepareForGarbageCollection - remove references to prepare for garbage collection
+func (oo *OnuDeviceEntry) PrepareForGarbageCollection(ctx context.Context, aDeviceID string) {
+	logger.Debugw(ctx, "prepare for garbage collection", log.Fields{"device-id": aDeviceID})
+	oo.baseDeviceHandler = nil
+	oo.pOnuTP = nil
+	if oo.PDevOmciCC != nil {
+		oo.PDevOmciCC.PrepareForGarbageCollection(ctx, aDeviceID)
+	}
+	oo.PDevOmciCC = nil
+}
diff --git a/internal/pkg/omcitst/omci_self_test_handler.go b/internal/pkg/omcitst/omci_self_test_handler.go
index 93b32ce..6d7e26b 100755
--- a/internal/pkg/omcitst/omci_self_test_handler.go
+++ b/internal/pkg/omcitst/omci_self_test_handler.go
@@ -410,3 +410,10 @@
 	// If the return from here is NOT nil, the caller shall not wait for async response.
 	return selfTestCb.initiateNewSelfTestFsm(ctx, reqMsg, CommChan, meClassID, respChan)
 }
+
+// PrepareForGarbageCollection - remove references to prepare for garbage collection
+func (selfTestCb *SelfTestControlBlock) PrepareForGarbageCollection(ctx context.Context, aDeviceID string) {
+	logger.Debugw(ctx, "prepare for garbage collection", log.Fields{"device-id": aDeviceID})
+	selfTestCb.pDeviceHandler = nil
+	selfTestCb.pDevEntry = nil
+}
diff --git a/internal/pkg/pmmgr/onu_metrics_manager.go b/internal/pkg/pmmgr/onu_metrics_manager.go
index 5ed462a..2ce6174 100755
--- a/internal/pkg/pmmgr/onu_metrics_manager.go
+++ b/internal/pkg/pmmgr/onu_metrics_manager.go
@@ -3872,3 +3872,10 @@
 	controlBlock[7] = 0
 	return controlBlock
 }
+
+// PrepareForGarbageCollection - remove references to prepare for garbage collection
+func (mm *OnuMetricsManager) PrepareForGarbageCollection(ctx context.Context, aDeviceID string) {
+	logger.Debugw(ctx, "prepare for garbage collection", log.Fields{"device-id": aDeviceID})
+	mm.pDeviceHandler = nil
+	mm.pOnuDeviceEntry = nil
+}
diff --git a/internal/pkg/swupg/omci_onu_upgrade.go b/internal/pkg/swupg/omci_onu_upgrade.go
index 9f78044..f9b393e 100755
--- a/internal/pkg/swupg/omci_onu_upgrade.go
+++ b/internal/pkg/swupg/omci_onu_upgrade.go
@@ -1921,3 +1921,13 @@
 		oFsm.volthaImageState = voltha.ImageState_IMAGE_UNKNOWN //something like 'IMAGE_DOWNLOAD_ABORTED' would be better (proto)
 	}
 }
+
+// PrepareForGarbageCollection - remove references to prepare for garbage collection
+func (oFsm *OnuUpgradeFsm) PrepareForGarbageCollection(ctx context.Context, aDeviceID string) {
+	logger.Debugw(ctx, "prepare for garbage collection", log.Fields{"device-id": aDeviceID})
+	oFsm.pDeviceHandler = nil
+	oFsm.pDownloadManager = nil
+	oFsm.pFileManager = nil
+	oFsm.pDevEntry = nil
+	oFsm.pOmciCC = nil
+}
diff --git a/internal/pkg/uniprt/uniportadmin.go b/internal/pkg/uniprt/uniportadmin.go
index 0706fba..4ddf154 100755
--- a/internal/pkg/uniprt/uniportadmin.go
+++ b/internal/pkg/uniprt/uniportadmin.go
@@ -537,3 +537,11 @@
 		return fmt.Errorf("lockStateFsm uni-set responseError for device-id %s", oFsm.deviceID)
 	}
 }
+
+// PrepareForGarbageCollection - remove references to prepare for garbage collection
+func (oFsm *LockStateFsm) PrepareForGarbageCollection(ctx context.Context, aDeviceID string) {
+	logger.Debugw(ctx, "prepare for garbage collection", log.Fields{"device-id": aDeviceID})
+	oFsm.pDeviceHandler = nil
+	oFsm.pOnuDeviceEntry = nil
+	oFsm.pOmciCC = nil
+}