[VOL-4548] openonuAdapterGo - memory leak seen in long term tests
Change-Id: Ie3eb2048cebf8bf8b2f2a99ccd927d462000223a
diff --git a/VERSION b/VERSION
index 503bb24..5deb43e 100755
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.2.0-dev266
+2.2.0-dev267
diff --git a/internal/pkg/almgr/alarm_manager.go b/internal/pkg/almgr/alarm_manager.go
index fa4bc9f..69ae862 100755
--- a/internal/pkg/almgr/alarm_manager.go
+++ b/internal/pkg/almgr/alarm_manager.go
@@ -800,3 +800,10 @@
func (am *OnuAlarmManager) GetAlarmMgrEventChannel() chan cmn.Message {
return am.eventChannel
}
+
+// PrepareForGarbageCollection - remove references to prepare for garbage collection
+func (am *OnuAlarmManager) PrepareForGarbageCollection(ctx context.Context, aDeviceID string) {
+ logger.Debugw(ctx, "prepare for garbage collection", log.Fields{"device-id": aDeviceID})
+ am.pDeviceHandler = nil
+ am.pOnuDeviceEntry = nil
+}
diff --git a/internal/pkg/avcfg/omci_ani_config.go b/internal/pkg/avcfg/omci_ani_config.go
index 34ed185..cc023ba 100755
--- a/internal/pkg/avcfg/omci_ani_config.go
+++ b/internal/pkg/avcfg/omci_ani_config.go
@@ -1817,3 +1817,11 @@
oFsm.mutexChanSet.RUnlock()
return flagValue
}
+
+// PrepareForGarbageCollection - remove references to prepare for garbage collection
+func (oFsm *UniPonAniConfigFsm) PrepareForGarbageCollection(ctx context.Context, aDeviceID string) {
+ logger.Debugw(ctx, "prepare for garbage collection", log.Fields{"device-id": oFsm.deviceID})
+ oFsm.pDeviceHandler = nil
+ oFsm.pOnuDeviceEntry = nil
+ oFsm.pOmciCC = nil
+}
diff --git a/internal/pkg/avcfg/omci_vlan_config.go b/internal/pkg/avcfg/omci_vlan_config.go
index 668d2c6..e97b0be 100755
--- a/internal/pkg/avcfg/omci_vlan_config.go
+++ b/internal/pkg/avcfg/omci_vlan_config.go
@@ -3229,3 +3229,11 @@
}
}
}
+
+// PrepareForGarbageCollection - remove references to prepare for garbage collection
+func (oFsm *UniVlanConfigFsm) PrepareForGarbageCollection(ctx context.Context, aDeviceID string) {
+ logger.Debugw(ctx, "prepare for garbage collection", log.Fields{"device-id": aDeviceID})
+ oFsm.pDeviceHandler = nil
+ oFsm.pOnuDeviceEntry = nil
+ oFsm.pOmciCC = nil
+}
diff --git a/internal/pkg/avcfg/onu_uni_tp.go b/internal/pkg/avcfg/onu_uni_tp.go
index efb69be..c51c489 100755
--- a/internal/pkg/avcfg/onu_uni_tp.go
+++ b/internal/pkg/avcfg/onu_uni_tp.go
@@ -1009,3 +1009,14 @@
}
return false
}
+
+// PrepareForGarbageCollection - remove references to prepare for garbage collection
+func (onuTP *OnuUniTechProf) PrepareForGarbageCollection(ctx context.Context, aDeviceID string) {
+ logger.Debugw(ctx, "prepare for garbage collection", log.Fields{"device-id": aDeviceID})
+ onuTP.baseDeviceHandler = nil
+ onuTP.onuDevice = nil
+ for k, v := range onuTP.PAniConfigFsm {
+ v.PrepareForGarbageCollection(ctx, aDeviceID)
+ delete(onuTP.PAniConfigFsm, k)
+ }
+}
diff --git a/internal/pkg/common/omci_cc.go b/internal/pkg/common/omci_cc.go
index 993a84e..57bc0ce 100755
--- a/internal/pkg/common/omci_cc.go
+++ b/internal/pkg/common/omci_cc.go
@@ -4674,3 +4674,11 @@
log.Fields{"Err": omciErr.GetError(), "device-id": oo.deviceID, "inst-id": strconv.FormatInt(int64(entityID), 16)})
return nil, omciErr.GetError()
}
+
+// PrepareForGarbageCollection - remove references to prepare for garbage collection
+func (oo *OmciCC) PrepareForGarbageCollection(ctx context.Context, aDeviceID string) {
+ logger.Debugw(ctx, "prepare for garbage collection", log.Fields{"device-id": aDeviceID})
+ oo.pBaseDeviceHandler = nil
+ oo.pOnuDeviceEntry = nil
+ oo.pOnuAlarmManager = nil
+}
diff --git a/internal/pkg/core/device_handler.go b/internal/pkg/core/device_handler.go
index 58d42b1..332861b 100755
--- a/internal/pkg/core/device_handler.go
+++ b/internal/pkg/core/device_handler.go
@@ -4497,3 +4497,50 @@
}
return tpPathFound
}
+
+// PrepareForGarbageCollection - remove references to prepare for garbage collection
+func (dh *deviceHandler) PrepareForGarbageCollection(ctx context.Context, aDeviceID string) {
+ logger.Debugw(ctx, "prepare for garbage collection", log.Fields{"device-id": aDeviceID})
+
+ // Note: This function must be called as a goroutine to prevent blocking of further processing!
+ // first let the objects rest for some time to give all asynchronously started
+ // cleanup routines a chance to come to an end
+ time.Sleep(5 * time.Second)
+
+ if dh.pOnuTP != nil {
+ dh.pOnuTP.PrepareForGarbageCollection(ctx, aDeviceID)
+ }
+ if dh.pOnuMetricsMgr != nil {
+ dh.pOnuMetricsMgr.PrepareForGarbageCollection(ctx, aDeviceID)
+ }
+ if dh.pAlarmMgr != nil {
+ dh.pAlarmMgr.PrepareForGarbageCollection(ctx, aDeviceID)
+ }
+ if dh.pSelfTestHdlr != nil {
+ dh.pSelfTestHdlr.PrepareForGarbageCollection(ctx, aDeviceID)
+ }
+ if dh.pLockStateFsm != nil {
+ dh.pLockStateFsm.PrepareForGarbageCollection(ctx, aDeviceID)
+ }
+ if dh.pUnlockStateFsm != nil {
+ dh.pUnlockStateFsm.PrepareForGarbageCollection(ctx, aDeviceID)
+ }
+ if dh.pOnuUpradeFsm != nil {
+ dh.pOnuUpradeFsm.PrepareForGarbageCollection(ctx, aDeviceID)
+ }
+ if dh.pOnuOmciDevice != nil {
+ dh.pOnuOmciDevice.PrepareForGarbageCollection(ctx, aDeviceID)
+ }
+ for k, v := range dh.UniVlanConfigFsmMap {
+ v.PrepareForGarbageCollection(ctx, aDeviceID)
+ delete(dh.UniVlanConfigFsmMap, k)
+ }
+ dh.pOnuOmciDevice = nil
+ dh.pOnuTP = nil
+ dh.pOnuMetricsMgr = nil
+ dh.pAlarmMgr = nil
+ dh.pSelfTestHdlr = nil
+ dh.pLockStateFsm = nil
+ dh.pUnlockStateFsm = nil
+ dh.pOnuUpradeFsm = nil
+}
diff --git a/internal/pkg/core/openonu.go b/internal/pkg/core/openonu.go
index f0da31b..44bfae0 100755
--- a/internal/pkg/core/openonu.go
+++ b/internal/pkg/core/openonu.go
@@ -344,34 +344,18 @@
if err := handler.resetFsms(ctx, true); err != nil {
errorsList = append(errorsList, err)
}
- forceKvDelete := false
-
- // Clear PM data on the KV store
- if handler.pOnuMetricsMgr != nil {
- if err := handler.pOnuMetricsMgr.ClearAllPmData(ctx); err != nil {
- errorsList = append(errorsList, err)
- forceKvDelete = true
- }
- } else {
- forceKvDelete = true
- }
- if err := handler.deleteDevicePersistencyData(ctx); err != nil {
- errorsList = append(errorsList, err)
- forceKvDelete = true
- }
for _, uni := range handler.uniEntityMap {
if handler.GetFlowMonitoringIsRunning(uni.UniID) {
handler.stopFlowMonitoringRoutine[uni.UniID] <- true
logger.Debugw(ctx, "sent stop signal to self flow monitoring routine", log.Fields{"device-id": device.Id})
}
}
- //don't leave any garbage - even in error case
- oo.deleteDeviceHandlerToMap(handler)
- if forceKvDelete {
- if err := oo.forceDeleteDeviceKvData(ctx, device.Id); err != nil {
- errorsList = append(errorsList, err)
- }
+ //don't leave any garbage in kv-store
+ if err := oo.forceDeleteDeviceKvData(ctx, device.Id); err != nil {
+ errorsList = append(errorsList, err)
}
+ oo.deleteDeviceHandlerToMap(handler)
+ go handler.PrepareForGarbageCollection(ctx, handler.DeviceID)
if len(errorsList) > 0 {
logger.Errorw(ctx, "one-or-more-error-during-device-delete", log.Fields{"device-id": device.Id})
diff --git a/internal/pkg/mib/mib_sync.go b/internal/pkg/mib/mib_sync.go
index d3f2712..5222b8b 100755
--- a/internal/pkg/mib/mib_sync.go
+++ b/internal/pkg/mib/mib_sync.go
@@ -488,6 +488,10 @@
switch message.Type {
case cmn.TestMsg:
msg, _ := message.Data.(cmn.TestMessage)
+ if msg.TestMessageVal == cmn.AbortMessageProcessing {
+ logger.Debugw(ctx, "MibSync Msg abort ProcessMsg", log.Fields{"for device-id": oo.deviceID})
+ break loop
+ }
oo.handleTestMsg(ctx, msg)
case cmn.OMCI:
msg, _ := message.Data.(cmn.OmciMessage)
@@ -1207,8 +1211,16 @@
}
//the MibSync FSM might be active all the ONU-active time,
// hence it must be stopped unconditionally
- pMibUlFsm := oo.PMibUploadFsm.PFsm
+ pMibUlFsm := oo.PMibUploadFsm
if pMibUlFsm != nil {
- _ = pMibUlFsm.Event(UlEvStop)
+ // abort running message processing
+ fsmAbortMsg := cmn.Message{
+ Type: cmn.TestMsg,
+ Data: cmn.TestMessage{
+ TestMessageVal: cmn.AbortMessageProcessing,
+ },
+ }
+ pMibUlFsm.CommChan <- fsmAbortMsg
+ _ = pMibUlFsm.PFsm.Event(UlEvStop)
}
}
diff --git a/internal/pkg/mib/onu_device_entry.go b/internal/pkg/mib/onu_device_entry.go
index 26c346d..47c7b90 100755
--- a/internal/pkg/mib/onu_device_entry.go
+++ b/internal/pkg/mib/onu_device_entry.go
@@ -986,3 +986,14 @@
oo.mutexReconcilingFlowsFlag.RUnlock()
return value
}
+
+// PrepareForGarbageCollection - remove references to prepare for garbage collection
+func (oo *OnuDeviceEntry) PrepareForGarbageCollection(ctx context.Context, aDeviceID string) {
+ logger.Debugw(ctx, "prepare for garbage collection", log.Fields{"device-id": aDeviceID})
+ oo.baseDeviceHandler = nil
+ oo.pOnuTP = nil
+ if oo.PDevOmciCC != nil {
+ oo.PDevOmciCC.PrepareForGarbageCollection(ctx, aDeviceID)
+ }
+ oo.PDevOmciCC = nil
+}
diff --git a/internal/pkg/omcitst/omci_self_test_handler.go b/internal/pkg/omcitst/omci_self_test_handler.go
index 93b32ce..6d7e26b 100755
--- a/internal/pkg/omcitst/omci_self_test_handler.go
+++ b/internal/pkg/omcitst/omci_self_test_handler.go
@@ -410,3 +410,10 @@
// If the return from here is NOT nil, the caller shall not wait for async response.
return selfTestCb.initiateNewSelfTestFsm(ctx, reqMsg, CommChan, meClassID, respChan)
}
+
+// PrepareForGarbageCollection - remove references to prepare for garbage collection
+func (selfTestCb *SelfTestControlBlock) PrepareForGarbageCollection(ctx context.Context, aDeviceID string) {
+ logger.Debugw(ctx, "prepare for garbage collection", log.Fields{"device-id": aDeviceID})
+ selfTestCb.pDeviceHandler = nil
+ selfTestCb.pDevEntry = nil
+}
diff --git a/internal/pkg/pmmgr/onu_metrics_manager.go b/internal/pkg/pmmgr/onu_metrics_manager.go
index 5ed462a..2ce6174 100755
--- a/internal/pkg/pmmgr/onu_metrics_manager.go
+++ b/internal/pkg/pmmgr/onu_metrics_manager.go
@@ -3872,3 +3872,10 @@
controlBlock[7] = 0
return controlBlock
}
+
+// PrepareForGarbageCollection - remove references to prepare for garbage collection
+func (mm *OnuMetricsManager) PrepareForGarbageCollection(ctx context.Context, aDeviceID string) {
+ logger.Debugw(ctx, "prepare for garbage collection", log.Fields{"device-id": aDeviceID})
+ mm.pDeviceHandler = nil
+ mm.pOnuDeviceEntry = nil
+}
diff --git a/internal/pkg/swupg/omci_onu_upgrade.go b/internal/pkg/swupg/omci_onu_upgrade.go
index 9f78044..f9b393e 100755
--- a/internal/pkg/swupg/omci_onu_upgrade.go
+++ b/internal/pkg/swupg/omci_onu_upgrade.go
@@ -1921,3 +1921,13 @@
oFsm.volthaImageState = voltha.ImageState_IMAGE_UNKNOWN //something like 'IMAGE_DOWNLOAD_ABORTED' would be better (proto)
}
}
+
+// PrepareForGarbageCollection - remove references to prepare for garbage collection
+func (oFsm *OnuUpgradeFsm) PrepareForGarbageCollection(ctx context.Context, aDeviceID string) {
+ logger.Debugw(ctx, "prepare for garbage collection", log.Fields{"device-id": aDeviceID})
+ oFsm.pDeviceHandler = nil
+ oFsm.pDownloadManager = nil
+ oFsm.pFileManager = nil
+ oFsm.pDevEntry = nil
+ oFsm.pOmciCC = nil
+}
diff --git a/internal/pkg/uniprt/uniportadmin.go b/internal/pkg/uniprt/uniportadmin.go
index 0706fba..4ddf154 100755
--- a/internal/pkg/uniprt/uniportadmin.go
+++ b/internal/pkg/uniprt/uniportadmin.go
@@ -537,3 +537,11 @@
return fmt.Errorf("lockStateFsm uni-set responseError for device-id %s", oFsm.deviceID)
}
}
+
+// PrepareForGarbageCollection - remove references to prepare for garbage collection
+func (oFsm *LockStateFsm) PrepareForGarbageCollection(ctx context.Context, aDeviceID string) {
+ logger.Debugw(ctx, "prepare for garbage collection", log.Fields{"device-id": aDeviceID})
+ oFsm.pDeviceHandler = nil
+ oFsm.pOnuDeviceEntry = nil
+ oFsm.pOmciCC = nil
+}