[VOL-4548] openonuAdapterGo - memory leak seen in long term tests
Change-Id: I306e3b8d9529ba1ffb43f93cd70524931d7c5e3c
diff --git a/VERSION b/VERSION
index 98390b6..4d9579a 100755
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-1.3.24
+1.3.25
diff --git a/internal/pkg/onuadaptercore/alarm_manager.go b/internal/pkg/onuadaptercore/alarm_manager.go
index 0dbc5b8..2c8690d 100644
--- a/internal/pkg/onuadaptercore/alarm_manager.go
+++ b/internal/pkg/onuadaptercore/alarm_manager.go
@@ -786,3 +786,9 @@
am.onuAlarmManagerLock.RUnlock()
return value
}
+
+// PrepareForGarbageCollection - remove references to prepare for garbage collection
+func (am *onuAlarmManager) PrepareForGarbageCollection(ctx context.Context, aDeviceID string) {
+ logger.Debugw(ctx, "prepare for garbage collection", log.Fields{"device-id": aDeviceID})
+ am.pDeviceHandler = nil
+}
diff --git a/internal/pkg/onuadaptercore/device_handler.go b/internal/pkg/onuadaptercore/device_handler.go
index 4a831eb..478df56 100644
--- a/internal/pkg/onuadaptercore/device_handler.go
+++ b/internal/pkg/onuadaptercore/device_handler.go
@@ -3942,3 +3942,50 @@
dh.lockVlanConfig.RUnlock()
return value
}
+
+// PrepareForGarbageCollection - remove references to prepare for garbage collection
+func (dh *deviceHandler) PrepareForGarbageCollection(ctx context.Context, aDeviceID string) {
+ logger.Debugw(ctx, "prepare for garbage collection", log.Fields{"device-id": aDeviceID})
+
+ // Note: This function must be called as a goroutine to prevent blocking of further processing!
+ // first let the objects rest for some time to give all asynchronously started
+ // cleanup routines a chance to come to an end
+ time.Sleep(5 * time.Second)
+
+ if dh.pOnuTP != nil {
+ dh.pOnuTP.PrepareForGarbageCollection(ctx, aDeviceID)
+ }
+ if dh.pOnuMetricsMgr != nil {
+ dh.pOnuMetricsMgr.PrepareForGarbageCollection(ctx, aDeviceID)
+ }
+ if dh.pAlarmMgr != nil {
+ dh.pAlarmMgr.PrepareForGarbageCollection(ctx, aDeviceID)
+ }
+ if dh.pSelfTestHdlr != nil {
+ dh.pSelfTestHdlr.PrepareForGarbageCollection(ctx, aDeviceID)
+ }
+ if dh.pLockStateFsm != nil {
+ dh.pLockStateFsm.PrepareForGarbageCollection(ctx, aDeviceID)
+ }
+ if dh.pUnlockStateFsm != nil {
+ dh.pUnlockStateFsm.PrepareForGarbageCollection(ctx, aDeviceID)
+ }
+ if dh.pOnuUpradeFsm != nil {
+ dh.pOnuUpradeFsm.PrepareForGarbageCollection(ctx, aDeviceID)
+ }
+ if dh.pOnuOmciDevice != nil {
+ dh.pOnuOmciDevice.PrepareForGarbageCollection(ctx, aDeviceID)
+ }
+ for k, v := range dh.UniVlanConfigFsmMap {
+ v.PrepareForGarbageCollection(ctx, aDeviceID)
+ delete(dh.UniVlanConfigFsmMap, k)
+ }
+ dh.pOnuOmciDevice = nil
+ dh.pOnuTP = nil
+ dh.pOnuMetricsMgr = nil
+ dh.pAlarmMgr = nil
+ dh.pSelfTestHdlr = nil
+ dh.pLockStateFsm = nil
+ dh.pUnlockStateFsm = nil
+ dh.pOnuUpradeFsm = nil
+}
diff --git a/internal/pkg/onuadaptercore/mib_sync.go b/internal/pkg/onuadaptercore/mib_sync.go
index 208d8cd..3439375 100644
--- a/internal/pkg/onuadaptercore/mib_sync.go
+++ b/internal/pkg/onuadaptercore/mib_sync.go
@@ -418,6 +418,10 @@
switch message.Type {
case TestMsg:
msg, _ := message.Data.(TestMessage)
+ if msg.TestMessageVal == AbortMessageProcessing {
+ logger.Debugw(ctx, "MibSync Msg abort ProcessMsg", log.Fields{"for device-id": oo.deviceID})
+ break loop
+ }
oo.handleTestMsg(ctx, msg)
case OMCI:
msg, _ := message.Data.(OmciMessage)
@@ -1103,8 +1107,16 @@
}
//the MibSync FSM might be active all the ONU-active time,
// hence it must be stopped unconditionally
- pMibUlFsm := oo.pMibUploadFsm.pFsm
+ pMibUlFsm := oo.pMibUploadFsm
if pMibUlFsm != nil {
- _ = pMibUlFsm.Event(ulEvStop)
+ // abort running message processing
+ fsmAbortMsg := Message{
+ Type: TestMsg,
+ Data: TestMessage{
+ TestMessageVal: AbortMessageProcessing,
+ },
+ }
+ pMibUlFsm.commChan <- fsmAbortMsg
+ _ = pMibUlFsm.pFsm.Event(ulEvStop)
}
}
diff --git a/internal/pkg/onuadaptercore/omci_ani_config.go b/internal/pkg/onuadaptercore/omci_ani_config.go
index e6f8680..31087be 100644
--- a/internal/pkg/onuadaptercore/omci_ani_config.go
+++ b/internal/pkg/onuadaptercore/omci_ani_config.go
@@ -1760,3 +1760,10 @@
oFsm.mutexChanSet.RUnlock()
return flagValue
}
+
+// PrepareForGarbageCollection - remove references to prepare for garbage collection
+func (oFsm *uniPonAniConfigFsm) PrepareForGarbageCollection(ctx context.Context, aDeviceID string) {
+ logger.Debugw(ctx, "prepare for garbage collection", log.Fields{"device-id": oFsm.deviceID})
+ oFsm.pDeviceHandler = nil
+ oFsm.pOmciCC = nil
+}
diff --git a/internal/pkg/onuadaptercore/omci_cc.go b/internal/pkg/onuadaptercore/omci_cc.go
index 4006283..5d28f59 100644
--- a/internal/pkg/onuadaptercore/omci_cc.go
+++ b/internal/pkg/onuadaptercore/omci_cc.go
@@ -3237,3 +3237,10 @@
log.Fields{"Err": omciErr.GetError(), "device-id": oo.deviceID, "upstream": upstream, "create": create, "inst-id": strconv.FormatInt(int64(entityID), 16)})
return nil, omciErr.GetError()
}
+
+// PrepareForGarbageCollection - remove references to prepare for garbage collection
+func (oo *omciCC) PrepareForGarbageCollection(ctx context.Context, aDeviceID string) {
+ logger.Debugw(ctx, "prepare for garbage collection", log.Fields{"device-id": aDeviceID})
+ oo.pBaseDeviceHandler = nil
+ oo.pOnuDeviceEntry = nil
+}
diff --git a/internal/pkg/onuadaptercore/omci_onu_upgrade.go b/internal/pkg/onuadaptercore/omci_onu_upgrade.go
index dbdad2b..b9b98af 100644
--- a/internal/pkg/onuadaptercore/omci_onu_upgrade.go
+++ b/internal/pkg/onuadaptercore/omci_onu_upgrade.go
@@ -1910,3 +1910,13 @@
oFsm.volthaImageState = voltha.ImageState_IMAGE_UNKNOWN //something like 'IMAGE_DOWNLOAD_ABORTED' would be better (proto)
}
}
+
+// PrepareForGarbageCollection - remove references to prepare for garbage collection
+func (oFsm *OnuUpgradeFsm) PrepareForGarbageCollection(ctx context.Context, aDeviceID string) {
+ logger.Debugw(ctx, "prepare for garbage collection", log.Fields{"device-id": aDeviceID})
+ oFsm.pDeviceHandler = nil
+ oFsm.pDownloadManager = nil
+ oFsm.pFileManager = nil
+ oFsm.pDevEntry = nil
+ oFsm.pOmciCC = nil
+}
diff --git a/internal/pkg/onuadaptercore/omci_self_test_handler.go b/internal/pkg/onuadaptercore/omci_self_test_handler.go
index 2e968fb..2e14b6d 100644
--- a/internal/pkg/onuadaptercore/omci_self_test_handler.go
+++ b/internal/pkg/onuadaptercore/omci_self_test_handler.go
@@ -20,13 +20,14 @@
import (
"context"
"fmt"
+ "sync"
+ "time"
+
"github.com/looplab/fsm"
"github.com/opencord/omci-lib-go"
"github.com/opencord/omci-lib-go/generated"
"github.com/opencord/voltha-lib-go/v5/pkg/log"
"github.com/opencord/voltha-protos/v4/go/extension"
- "sync"
- "time"
)
const (
@@ -379,3 +380,9 @@
// If the return from here is NOT nil, the caller shall not wait for async response.
return selfTestCb.initiateNewSelfTestFsm(ctx, reqMsg, commChan, meClassID, respChan)
}
+
+// PrepareForGarbageCollection - remove references to prepare for garbage collection
+func (selfTestCb *selfTestControlBlock) PrepareForGarbageCollection(ctx context.Context, aDeviceID string) {
+ logger.Debugw(ctx, "prepare for garbage collection", log.Fields{"device-id": aDeviceID})
+ selfTestCb.pDeviceHandler = nil
+}
diff --git a/internal/pkg/onuadaptercore/omci_vlan_config.go b/internal/pkg/onuadaptercore/omci_vlan_config.go
index daa4473..5311952 100644
--- a/internal/pkg/onuadaptercore/omci_vlan_config.go
+++ b/internal/pkg/onuadaptercore/omci_vlan_config.go
@@ -3156,3 +3156,10 @@
oFsm.numVlanFilterEntries++
}
}
+
+// PrepareForGarbageCollection - remove references to prepare for garbage collection
+func (oFsm *UniVlanConfigFsm) PrepareForGarbageCollection(ctx context.Context, aDeviceID string) {
+ logger.Debugw(ctx, "prepare for garbage collection", log.Fields{"device-id": aDeviceID})
+ oFsm.pDeviceHandler = nil
+ oFsm.pOmciCC = nil
+}
diff --git a/internal/pkg/onuadaptercore/onu_device_entry.go b/internal/pkg/onuadaptercore/onu_device_entry.go
index e7fc201..5de76bc 100644
--- a/internal/pkg/onuadaptercore/onu_device_entry.go
+++ b/internal/pkg/onuadaptercore/onu_device_entry.go
@@ -980,3 +980,13 @@
defer oo.mutexPersOnuConfig.Unlock()
delete(oo.sOnuPersistentData.PersTcontMap, allocID)
}
+
+// PrepareForGarbageCollection - remove references to prepare for garbage collection
+func (oo *OnuDeviceEntry) PrepareForGarbageCollection(ctx context.Context, aDeviceID string) {
+ logger.Debugw(ctx, "prepare for garbage collection", log.Fields{"device-id": aDeviceID})
+ oo.baseDeviceHandler = nil
+ if oo.PDevOmciCC != nil {
+ oo.PDevOmciCC.PrepareForGarbageCollection(ctx, aDeviceID)
+ }
+ oo.PDevOmciCC = nil
+}
diff --git a/internal/pkg/onuadaptercore/onu_metrics_manager.go b/internal/pkg/onuadaptercore/onu_metrics_manager.go
index ab3000c..baa783b 100644
--- a/internal/pkg/onuadaptercore/onu_metrics_manager.go
+++ b/internal/pkg/onuadaptercore/onu_metrics_manager.go
@@ -3670,3 +3670,9 @@
controlBlock[7] = 0
return controlBlock
}
+
+// PrepareForGarbageCollection - remove references to prepare for garbage collection
+func (mm *onuMetricsManager) PrepareForGarbageCollection(ctx context.Context, aDeviceID string) {
+ logger.Debugw(ctx, "prepare for garbage collection", log.Fields{"device-id": aDeviceID})
+ mm.pDeviceHandler = nil
+}
diff --git a/internal/pkg/onuadaptercore/onu_uni_tp.go b/internal/pkg/onuadaptercore/onu_uni_tp.go
index 6e365ab..94e8598 100644
--- a/internal/pkg/onuadaptercore/onu_uni_tp.go
+++ b/internal/pkg/onuadaptercore/onu_uni_tp.go
@@ -1003,3 +1003,13 @@
}
return false
}
+
+// PrepareForGarbageCollection - remove references to prepare for garbage collection
+func (onuTP *onuUniTechProf) PrepareForGarbageCollection(ctx context.Context, aDeviceID string) {
+ logger.Debugw(ctx, "prepare for garbage collection", log.Fields{"device-id": aDeviceID})
+ onuTP.baseDeviceHandler = nil
+ for k, v := range onuTP.pAniConfigFsm {
+ v.PrepareForGarbageCollection(ctx, aDeviceID)
+ delete(onuTP.pAniConfigFsm, k)
+ }
+}
diff --git a/internal/pkg/onuadaptercore/openonu.go b/internal/pkg/onuadaptercore/openonu.go
index f31b04f..4859493 100644
--- a/internal/pkg/onuadaptercore/openonu.go
+++ b/internal/pkg/onuadaptercore/openonu.go
@@ -409,20 +409,12 @@
handler.deletionInProgress = true
handler.mutexDeletionInProgressFlag.Unlock()
- if err := handler.deleteDevicePersistencyData(ctx); err != nil {
+ if err := handler.resetFsms(ctx, true); err != nil {
errorsList = append(errorsList, err)
}
- select {
- case handler.stopCollector <- true: // stop the metric collector routine
- logger.Debugw(ctx, "sent stop signal to metric collector routine", log.Fields{"device-id": device.Id})
- default:
- logger.Warnw(ctx, "metric collector routine not waiting on stop signal", log.Fields{"device-id": device.Id})
- }
- select {
- case handler.stopAlarmManager <- true: //stop the alarm manager.
- logger.Debugw(ctx, "sent stop signal to alarm manager", log.Fields{"device-id": device.Id})
- default:
- logger.Warnw(ctx, "alarm manager not waiting on stop signal", log.Fields{"device-id": device.Id})
+
+ if err := handler.deleteDevicePersistencyData(ctx); err != nil {
+ errorsList = append(errorsList, err)
}
if handler.pOnuMetricsMgr != nil {
if err := handler.pOnuMetricsMgr.clearAllPmData(ctx); err != nil {
@@ -437,6 +429,7 @@
}
//don't leave any garbage - even in error case
oo.deleteDeviceHandlerToMap(handler)
+ go handler.PrepareForGarbageCollection(ctx, device.Id)
if len(errorsList) > 0 {
logger.Errorw(ctx, "one-or-more-error-during-device-delete", log.Fields{"device-id": device.Id})
return fmt.Errorf("one-or-more-error-during-device-delete, errors:%v", errorsList)
diff --git a/internal/pkg/onuadaptercore/uniportadmin.go b/internal/pkg/onuadaptercore/uniportadmin.go
index 4b829d5..090e2fe 100644
--- a/internal/pkg/onuadaptercore/uniportadmin.go
+++ b/internal/pkg/onuadaptercore/uniportadmin.go
@@ -531,3 +531,10 @@
return fmt.Errorf("lockStateFsm uni-set responseError for device-id %s", oFsm.deviceID)
}
}
+
+// PrepareForGarbageCollection - remove references to prepare for garbage collection
+func (oFsm *lockStateFsm) PrepareForGarbageCollection(ctx context.Context, aDeviceID string) {
+ logger.Debugw(ctx, "prepare for garbage collection", log.Fields{"device-id": aDeviceID})
+ oFsm.pDeviceHandler = nil
+ oFsm.pOmciCC = nil
+}