VOL-5223:Alarm Mgr has to handle unconfigured MEs responses from ONT during Alarm Audit

Change-Id: I9d15232ab63c839d9d6e16ce342f9b8bbe0a3b21
diff --git a/VERSION b/VERSION
index 22ac5d0..bea85d2 100755
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.11.9
+2.11.10
diff --git a/internal/pkg/almgr/alarm_manager.go b/internal/pkg/almgr/alarm_manager.go
index cc6c898..cb9019b 100755
--- a/internal/pkg/almgr/alarm_manager.go
+++ b/internal/pkg/almgr/alarm_manager.go
@@ -33,6 +33,8 @@
 	cmn "github.com/opencord/voltha-openonu-adapter-go/internal/pkg/common"
 	"github.com/opencord/voltha-protos/v5/go/extension"
 	"github.com/opencord/voltha-protos/v5/go/voltha"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 )
 
 const (
@@ -250,13 +252,8 @@
 }
 func (am *OnuAlarmManager) asFsmLeaveAuditing(ctx context.Context, e *fsm.Event) {
 	logger.Debugw(ctx, "alarm-sync-fsm-leave-auditing state", log.Fields{"state": e.FSM.Current(), "device-id": am.deviceID})
-	if am.isAsyncAlarmRequest {
-		logger.Errorw(ctx, "alarm-sync-fsm-leave-auditing state process the updated ONU alarms ", log.Fields{"state": e.FSM.Current(), "device-id": am.deviceID})
-		am.AsyncAlarmsCommChan <- struct{}{}
-		am.isAsyncAlarmRequest = false
-
-	}
 }
+
 func (am *OnuAlarmManager) asFsmResynchronizing(ctx context.Context, e *fsm.Event) {
 	logger.Debugw(ctx, "alarm-sync-fsm", log.Fields{"state": e.FSM.Current(), "device-id": am.deviceID})
 	failureTransition := func() {
@@ -264,80 +261,27 @@
 			logger.Debugw(ctx, "alarm-sync-fsm-cannot-go-to-state-failure", log.Fields{"device-id": am.deviceID, "err": err})
 		}
 	}
-	// See if there is any onu only diff, meaning the class and entity is only in onu DB
-	for alarm := range am.onuDBCopy {
-		if _, exists := am.oltDbCopy[meAlarmKey{
-			classID:    alarm.classID,
-			instanceID: alarm.instanceID,
-		}]; !exists {
-			// We need to raise all such alarms as OLT wont have received notification for these alarms
-			omciAlarmMessage := &omci.AlarmNotificationMsg{
-				MeBasePacket: omci.MeBasePacket{
-					EntityClass:    alarm.classID,
-					EntityInstance: alarm.instanceID,
-				},
-				AlarmBitmap: am.onuDBCopy[alarm],
-			}
-			if err := am.processAlarmData(ctx, omciAlarmMessage); err != nil {
-				logger.Errorw(ctx, "unable-to-process-alarm-notification", log.Fields{"device-id": am.deviceID})
-				// Transition to failure.
-				go failureTransition()
-				return
-			}
-		}
+
+	// Process onu only differences
+	if err := am.processOnuOnlyDifferences(ctx, failureTransition); err != nil {
+		return
 	}
-	// See if there is any olt only diff, meaning the class and entity is only in olt DB
-	for alarm := range am.oltDbCopy {
-		if _, exists := am.onuDBCopy[meAlarmKey{
-			classID:    alarm.classID,
-			instanceID: alarm.instanceID,
-		}]; !exists {
-			// We need to clear all such alarms as OLT might have stale data and the alarms are already cleared.
-			omciAlarmMessage := &omci.AlarmNotificationMsg{
-				MeBasePacket: omci.MeBasePacket{
-					EntityClass:    alarm.classID,
-					EntityInstance: alarm.instanceID,
-				},
-				AlarmBitmap: am.oltDbCopy[alarm],
-			}
-			if err := am.processAlarmData(ctx, omciAlarmMessage); err != nil {
-				logger.Errorw(ctx, "unable-to-process-alarm-notification", log.Fields{"device-id": am.deviceID})
-				// Transition to failure
-				go failureTransition()
-				return
-			}
-		}
+
+	// Process olt only differences
+	if err := am.processOltOnlyDifferences(ctx, failureTransition); err != nil {
+		return
 	}
-	// See if there is any attribute difference
-	for alarm := range am.onuDBCopy {
-		if _, exists := am.oltDbCopy[alarm]; exists {
-			if am.onuDBCopy[alarm] != am.oltDbCopy[alarm] {
-				omciAlarmMessage := &omci.AlarmNotificationMsg{
-					MeBasePacket: omci.MeBasePacket{
-						EntityClass:    alarm.classID,
-						EntityInstance: alarm.instanceID,
-					},
-					AlarmBitmap: am.onuDBCopy[alarm],
-				}
-				// We will assume that onudb is correct always in this case and process the changed bitmap.
-				if err := am.processAlarmData(ctx, omciAlarmMessage); err != nil {
-					logger.Errorw(ctx, "unable-to-process-alarm-notification", log.Fields{"device-id": am.deviceID})
-					// Transition to failure
-					go failureTransition()
-					return
-				}
-			}
-		}
+
+	// Process attribute differences
+	if err := am.processAttributeDifferences(ctx, failureTransition); err != nil {
+		return
 	}
-	// Send the buffered notifications if no failure.
-	for _, notif := range am.bufferedNotifications {
-		logger.Debugw(ctx, "processing-buffered-alarm-notification", log.Fields{"device-id": am.deviceID,
-			"notification": notif})
-		if err := am.processAlarmData(ctx, notif); err != nil {
-			logger.Errorw(ctx, "unable-to-process-alarm-notification", log.Fields{"device-id": am.deviceID})
-			go failureTransition()
-		}
+
+	// Process buffered notifications
+	if err := am.processBufferedNotifications(ctx, failureTransition); err != nil {
+		return
 	}
+
 	go func() {
 		if err := am.AlarmSyncFsm.PFsm.Event(AsEvSuccess); err != nil {
 			logger.Debugw(ctx, "alarm-sync-fsm-cannot-go-to-state-sync", log.Fields{"device-id": am.deviceID, "err": err})
@@ -347,6 +291,14 @@
 
 func (am *OnuAlarmManager) asFsmInSync(ctx context.Context, e *fsm.Event) {
 	logger.Debugw(ctx, "alarm-sync-fsm", log.Fields{"state": e.FSM.Current(), "device-id": am.deviceID})
+
+	if am.isAsyncAlarmRequest {
+		logger.Debugw(ctx, "alarm-sync-fsm-before entering the sync state process the updated ONU alarms ", log.Fields{"state": e.FSM.Current(), "device-id": am.deviceID})
+		am.AsyncAlarmsCommChan <- struct{}{}
+		am.isAsyncAlarmRequest = false
+
+	}
+
 	if am.pDeviceHandler.GetAlarmAuditInterval() > 0 {
 		select {
 		case <-time.After(am.pDeviceHandler.GetAlarmAuditInterval()):
@@ -370,6 +322,16 @@
 			}()
 
 		}
+	} else {
+		<-am.AsyncAlarmsCommChan
+		go func() {
+			logger.Debugw(ctx, "On demand Auditing the ONU for Alarms  ", log.Fields{"device-id": am.deviceID})
+			if err := am.AlarmSyncFsm.PFsm.Event(AsEvAudit); err != nil {
+				logger.Errorw(ctx, "alarm-sync-fsm-cannot-go-to-state-auditing, use current snapshot of alarms", log.Fields{"device-id": am.deviceID, "err": err})
+				am.isAsyncAlarmRequest = false
+				am.AsyncAlarmsCommChan <- struct{}{}
+			}
+		}()
 	}
 }
 
@@ -632,12 +594,12 @@
 	if !am.processMessage {
 		logger.Warnw(ctx, "ignoring-alarm-notification-received-for-me-as-channel-for-processing-is-closed",
 			log.Fields{"device-id": am.deviceID})
-		return fmt.Errorf("alarm-manager-is-in-stopped-state")
+		return status.Error(codes.Unavailable, "alarm-manager-is-in-stopped-state")
 	}
 	if _, present := am.pOnuDeviceEntry.GetOnuDB().MeDb[classID][meInstance]; !present {
 		logger.Errorw(ctx, "me-class-instance-not-present",
 			log.Fields{"class-id": classID, "instance-id": meInstance, "device-id": am.deviceID})
-		return fmt.Errorf("me-class-%d-instance-%d-not-present", classID, meInstance)
+		return status.Error(codes.NotFound, "me-class-instance-not-present")
 	}
 	if sequenceNo > 0 {
 		if am.AlarmSyncFsm.PFsm.Is(asStAuditing) || am.AlarmSyncFsm.PFsm.Is(asStResynchronizing) {
@@ -661,12 +623,12 @@
 	if omciErr.StatusCode() != me.Success {
 		//log error and return
 		logger.Error(ctx, "unable-to-get-managed-entity", log.Fields{"class-id": classID, "instance-id": meInstance})
-		return fmt.Errorf("unable-to-get-managed-entity-class-%d-instance-%d", classID, meInstance)
+		return status.Error(codes.NotFound, "unable-to-get-managed-entity")
 	}
 	meAlarmMap := entity.GetAlarmMap()
 	if meAlarmMap == nil {
 		logger.Error(ctx, "unable-to-get-managed-entity-alarm-map", log.Fields{"class-id": classID, "instance-id": meInstance})
-		return fmt.Errorf("unable-to-get-managed-entity-alarm-map-%d-instance-%d", classID, meInstance)
+		return status.Error(codes.NotFound, "unable-to-get-managed-entity-alarm-map")
 	}
 
 	am.alarmBitMapDB[meAlarmKey{
@@ -910,3 +872,84 @@
 	am.pDeviceHandler = nil
 	am.pOnuDeviceEntry = nil
 }
+
+func (am *OnuAlarmManager) processOnuOnlyDifferences(ctx context.Context, failureTransition func()) error {
+	for alarm := range am.onuDBCopy {
+		if _, exists := am.oltDbCopy[meAlarmKey{classID: alarm.classID, instanceID: alarm.instanceID}]; !exists {
+			omciAlarmMessage := createOmciAlarmMessage(alarm, am.onuDBCopy[alarm])
+			if err := am.processAlarm(ctx, omciAlarmMessage, failureTransition); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (am *OnuAlarmManager) processOltOnlyDifferences(ctx context.Context, failureTransition func()) error {
+	for alarm := range am.oltDbCopy {
+		if _, exists := am.onuDBCopy[meAlarmKey{classID: alarm.classID, instanceID: alarm.instanceID}]; !exists {
+			omciAlarmMessage := createOmciAlarmMessage(alarm, am.oltDbCopy[alarm])
+			if err := am.processAlarm(ctx, omciAlarmMessage, failureTransition); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (am *OnuAlarmManager) processAttributeDifferences(ctx context.Context, failureTransition func()) error {
+	for alarm := range am.onuDBCopy {
+		if _, exists := am.oltDbCopy[alarm]; exists && am.onuDBCopy[alarm] != am.oltDbCopy[alarm] {
+			omciAlarmMessage := createOmciAlarmMessage(alarm, am.onuDBCopy[alarm])
+			if err := am.processAlarm(ctx, omciAlarmMessage, failureTransition); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (am *OnuAlarmManager) processBufferedNotifications(ctx context.Context, failureTransition func()) error {
+	for _, notif := range am.bufferedNotifications {
+		logger.Debugw(ctx, "processing-buffered-alarm-notification", log.Fields{"device-id": am.deviceID, "notification": notif})
+		if err := am.processAlarm(ctx, notif, failureTransition); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (am *OnuAlarmManager) processAlarm(ctx context.Context, omciAlarmMessage *omci.AlarmNotificationMsg, failureTransition func()) error {
+	// [https://jira.opencord.org/browse/VOL-5223]
+	//The following test scenarios cause AlarmMgr to get into a loop state.
+	//Test Scenario:
+	// - Unconfigured MEs being reported by vendor ONTs.
+	// - Undefined Alarm Bit Map (ONU-G ME for Example.)
+	// - MEs created by OLT as per G984.4 standard are not part of ONU DB.
+	if err := am.processAlarmData(ctx, omciAlarmMessage); err != nil {
+		if statusErr, ok := status.FromError(err); ok {
+			switch statusErr.Code() {
+			case codes.NotFound:
+				logger.Warnw(ctx, "ME Instance or ME Alarm Map not found in ONUDB", log.Fields{"device-id": am.deviceID, "Error": err})
+			case codes.Unavailable:
+				logger.Warnw(ctx, "Alarm Mgr is stopped, stop further processing", log.Fields{"device-id": am.deviceID, "Error": err})
+				return err
+			default:
+				logger.Errorw(ctx, "Unexpected error", log.Fields{"device-id": am.deviceID, "Error": err})
+				go failureTransition()
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func createOmciAlarmMessage(alarm meAlarmKey, alarmBitmap [alarmBitMapSizeBytes]byte) *omci.AlarmNotificationMsg {
+	return &omci.AlarmNotificationMsg{
+		MeBasePacket: omci.MeBasePacket{
+			EntityClass:    alarm.classID,
+			EntityInstance: alarm.instanceID,
+		},
+		AlarmBitmap: alarmBitmap,
+	}
+}
diff --git a/internal/pkg/avcfg/omci_ani_config.go b/internal/pkg/avcfg/omci_ani_config.go
index 2e73349..8539c71 100755
--- a/internal/pkg/avcfg/omci_ani_config.go
+++ b/internal/pkg/avcfg/omci_ani_config.go
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-//Package avcfg provides anig and vlan configuration functionality
+// Package avcfg provides anig and vlan configuration functionality
 package avcfg
 
 import (
@@ -125,7 +125,7 @@
 	dynamicACL     string
 }
 
-//UniPonAniConfigFsm defines the structure for the state machine to config the PON ANI ports of ONU UNI ports via OMCI
+// UniPonAniConfigFsm defines the structure for the state machine to config the PON ANI ports of ONU UNI ports via OMCI
 type UniPonAniConfigFsm struct {
 	deviceID                 string
 	pDeviceHandler           cmn.IdeviceHandler
@@ -160,7 +160,7 @@
 	tcontSetBefore           bool
 }
 
-//NewUniPonAniConfigFsm is the 'constructor' for the state machine to config the PON ANI ports of ONU UNI ports via OMCI
+// NewUniPonAniConfigFsm is the 'constructor' for the state machine to config the PON ANI ports of ONU UNI ports via OMCI
 func NewUniPonAniConfigFsm(ctx context.Context, apDevOmciCC *cmn.OmciCC, apUniPort *cmn.OnuUniPort, apUniTechProf *OnuUniTechProf,
 	apOnuDB *devdb.OnuDeviceDB, aTechProfileID uint8, aTechProfileType string, aRequestEvent cmn.OnuDeviceEvent, aName string,
 	apDeviceHandler cmn.IdeviceHandler, apOnuDeviceEntry cmn.IonuDeviceEntry, aCommChannel chan cmn.Message) *UniPonAniConfigFsm {
@@ -271,14 +271,14 @@
 	return instFsm
 }
 
-//setFsmCompleteChannel sets the requested channel and channel result for transfer on success
+// setFsmCompleteChannel sets the requested channel and channel result for transfer on success
 func (oFsm *UniPonAniConfigFsm) setFsmCompleteChannel(aChSuccess chan<- uint8, aProcStep uint8) {
 	oFsm.chSuccess = aChSuccess
 	oFsm.procStep = aProcStep
 	oFsm.setChanSet(true)
 }
 
-//CancelProcessing ensures that suspended processing at waiting on some response is aborted and reset of FSM
+// CancelProcessing ensures that suspended processing at waiting on some response is aborted and reset of FSM
 func (oFsm *UniPonAniConfigFsm) CancelProcessing(ctx context.Context) {
 	logger.Info(ctx, "CancelProcessing entered", log.Fields{"device-id": oFsm.deviceID})
 	//early indication about started reset processing
@@ -325,8 +325,8 @@
 	oFsm.pUniTechProf.clearAniSideConfig(ctx, oFsm.pOnuUniPort.UniID, oFsm.techProfileID)
 }
 
-//nolint: gocyclo
-//TODO:visit here for refactoring for gocyclo
+// nolint: gocyclo
+// TODO:visit here for refactoring for gocyclo
 func (oFsm *UniPonAniConfigFsm) prepareAndEnterConfigState(ctx context.Context, aPAFsm *cmn.AdapterFsm) {
 	if aPAFsm != nil && aPAFsm.PFsm != nil {
 		var err error
@@ -459,6 +459,24 @@
 			}
 
 			oFsm.gemPortAttribsSlice = append(oFsm.gemPortAttribsSlice, loGemPortAttribs)
+			if oFsm.pDeviceHandler.IsSkipOnuConfigReconciling() {
+				meParams := me.ParamData{
+					EntityID: loGemPortAttribs.gemPortID,
+					Attributes: me.AttributeValueMap{
+						me.GemPortNetworkCtp_PortId:                              loGemPortAttribs.gemPortID,
+						me.GemPortNetworkCtp_TContPointer:                        oFsm.tcont0ID,
+						me.GemPortNetworkCtp_Direction:                           loGemPortAttribs.direction,
+						me.GemPortNetworkCtp_TrafficManagementPointerForUpstream: loGemPortAttribs.upQueueID,
+						me.GemPortNetworkCtp_PriorityQueuePointerForDownStream:   loGemPortAttribs.downQueueID,
+					},
+				}
+				oFsm.pOnuDB.PutMe(ctx, me.GemPortNetworkCtpClassID, loGemPortAttribs.gemPortID, meParams.Attributes)
+				var meAttributes me.AttributeValueMap //dummy , anyways we are not going to use the values.
+				oFsm.pOnuDB.PutMe(ctx, me.GemPortNetworkCtpPerformanceMonitoringHistoryDataClassID, loGemPortAttribs.gemPortID, meAttributes)
+
+				oFsm.pOnuDB.PutMe(ctx, me.GemInterworkingTerminationPointClassID, loGemPortAttribs.gemPortID, meAttributes)
+
+			}
 		}
 		if !oFsm.pDeviceHandler.IsSkipOnuConfigReconciling() {
 			_ = aPAFsm.PFsm.Event(aniEvStartConfig)
@@ -853,6 +871,8 @@
 	}
 	oFsm.pLastTxMeInstance = meInstance
 	oFsm.mutexPLastTxMeInstance.Unlock()
+	logger.Infow(ctx, "Deleting  GemIWTP at  the ONU DB ", log.Fields{"device-id": oFsm.deviceID, "GEMID": loGemPortID})
+	oFsm.pOnuDB.DeleteMe(me.GemInterworkingTerminationPointClassID, loGemPortID)
 }
 
 func (oFsm *UniPonAniConfigFsm) enterWaitingFlowRem(ctx context.Context, e *fsm.Event) {
@@ -953,7 +973,7 @@
 	}
 	oFsm.pLastTxMeInstance = meInstance
 	oFsm.mutexPLastTxMeInstance.Unlock()
-
+	oFsm.pOnuDB.DeleteMe(me.GemPortNetworkCtpClassID, loGemPortID)
 	// Mark the gem port to be removed for Performance History monitoring
 	OnuMetricsManager := oFsm.pDeviceHandler.GetOnuMetricsManager()
 	if OnuMetricsManager != nil {
@@ -1509,6 +1529,7 @@
 			_ = oFsm.PAdaptFsm.PFsm.Event(aniEvReset)
 			return
 		}
+		oFsm.pOnuDB.PutMe(ctx, me.GemPortNetworkCtpClassID, gemPortAttribs.gemPortID, meParams.Attributes)
 		// Mark the gem port to be added for Performance History monitoring
 		OnuMetricsManager := oFsm.pDeviceHandler.GetOnuMetricsManager()
 		if OnuMetricsManager != nil {
@@ -1538,10 +1559,11 @@
 			"SPPtr":     strconv.FormatInt(int64(oFsm.mapperSP0ID), 16),
 			"device-id": oFsm.deviceID})
 
+		var meParams me.ParamData
 		//TODO if the port has only downstream direction the isMulticast flag can be removed.
 		if gemPortAttribs.isMulticast {
 
-			meParams := me.ParamData{
+			meParams = me.ParamData{
 				EntityID: gemPortAttribs.multicastGemID,
 				Attributes: me.AttributeValueMap{
 					me.MulticastGemInterworkingTerminationPoint_GemPortNetworkCtpConnectivityPointer: gemPortAttribs.multicastGemID,
@@ -1605,7 +1627,7 @@
 			oFsm.mutexPLastTxMeInstance.Unlock()
 
 		} else {
-			meParams := me.ParamData{
+			meParams = me.ParamData{
 				EntityID: gemPortAttribs.gemPortID,
 				Attributes: me.AttributeValueMap{
 					me.GemInterworkingTerminationPoint_GemPortNetworkCtpConnectivityPointer: gemPortAttribs.gemPortID, //same as EntityID, see above
@@ -1638,6 +1660,9 @@
 			_ = oFsm.PAdaptFsm.PFsm.Event(aniEvReset)
 			return
 		}
+		logger.Infow(ctx, "Adding GemIWTP to the ONU DB ", log.Fields{"device-id": oFsm.deviceID, "GEMID": gemPortAttribs.gemPortID})
+		oFsm.pOnuDB.PutMe(ctx, me.GemInterworkingTerminationPointClassID, gemPortAttribs.gemPortID, meParams.Attributes)
+
 	} //for all GemPort's of this T-Cont
 
 	// if Config has been done for all GemPort instances let the FSM proceed
diff --git a/internal/pkg/avcfg/omci_vlan_config.go b/internal/pkg/avcfg/omci_vlan_config.go
index d4bf25a..54be738 100755
--- a/internal/pkg/avcfg/omci_vlan_config.go
+++ b/internal/pkg/avcfg/omci_vlan_config.go
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-//Package avcfg provides anig and vlan configuration functionality
+// Package avcfg provides anig and vlan configuration functionality
 package avcfg
 
 import (
@@ -143,8 +143,9 @@
 	respChan         *chan error
 }
 
-//UniVlanConfigFsm defines the structure for the state machine for configuration of the VLAN related setting via OMCI
-//  builds upon 'VLAN rules' that are derived from multiple flows
+// UniVlanConfigFsm defines the structure for the state machine for configuration of the VLAN related setting via OMCI
+//
+//	builds upon 'VLAN rules' that are derived from multiple flows
 type UniVlanConfigFsm struct {
 	pDeviceHandler              cmn.IdeviceHandler
 	pOnuDeviceEntry             cmn.IonuDeviceEntry
@@ -184,8 +185,9 @@
 	lastFlowToReconcile bool
 }
 
-//NewUniVlanConfigFsm is the 'constructor' for the state machine to config the PON ANI ports
-//  of ONU UNI ports via OMCI
+// NewUniVlanConfigFsm is the 'constructor' for the state machine to config the PON ANI ports
+//
+//	of ONU UNI ports via OMCI
 func NewUniVlanConfigFsm(ctx context.Context, apDeviceHandler cmn.IdeviceHandler, apOnuDeviceEntry cmn.IonuDeviceEntry, apDevOmciCC *cmn.OmciCC, apUniPort *cmn.OnuUniPort,
 	apUniTechProf *OnuUniTechProf, apOnuDB *devdb.OnuDeviceDB, aTechProfileID uint8,
 	aRequestEvent cmn.OnuDeviceEvent, aName string, aCommChannel chan cmn.Message, aAcceptIncrementalEvto bool,
@@ -279,7 +281,7 @@
 	return instFsm
 }
 
-//initUniFlowParams is a simplified form of SetUniFlowParams() used for first flow parameters configuration
+// initUniFlowParams is a simplified form of SetUniFlowParams() used for first flow parameters configuration
 func (oFsm *UniVlanConfigFsm) initUniFlowParams(ctx context.Context, aTpID uint8, aCookieSlice []uint64,
 	aMatchVlan uint16, aMatchPcp uint8, aSetVlan uint16, aSetPcp uint8, innerCvlan uint16, aMeter *of.OfpMeterConfig, respChan *chan error) error {
 	loRuleParams := cmn.UniVlanRuleParams{
@@ -352,13 +354,13 @@
 	return nil
 }
 
-//CancelProcessing ensures that suspended processing at waiting on some response is aborted and reset of FSM
+// CancelProcessing ensures that suspended processing at waiting on some response is aborted and reset of FSM
 func (oFsm *UniVlanConfigFsm) CancelProcessing(ctx context.Context) {
-	logger.Debugw(ctx, "CancelProcessing entered", log.Fields{"device-id": oFsm.deviceID})
 	if oFsm == nil {
 		logger.Error(ctx, "no valid UniVlanConfigFsm!")
 		return
 	}
+	logger.Debugw(ctx, "CancelProcessing entered", log.Fields{"device-id": oFsm.deviceID})
 	//mutex protection is required for possible concurrent access to FSM members
 	oFsm.mutexIsAwaitingResponse.Lock()
 	oFsm.isCanceled = true
@@ -382,7 +384,7 @@
 	}
 }
 
-//GetWaitingTpID returns the TpId that the FSM might be waiting for continuation (0 if none)
+// GetWaitingTpID returns the TpId that the FSM might be waiting for continuation (0 if none)
 func (oFsm *UniVlanConfigFsm) GetWaitingTpID(ctx context.Context) uint8 {
 	if oFsm == nil {
 		logger.Error(ctx, "no valid UniVlanConfigFsm!")
@@ -394,7 +396,7 @@
 	return oFsm.TpIDWaitingFor
 }
 
-//SetUniFlowParams verifies on existence of flow parameters to be configured,
+// SetUniFlowParams verifies on existence of flow parameters to be configured,
 // optionally udates the cookie list or appends a new flow if there is space
 // if possible the FSM is trigggerd to start with the processing
 // ignore complexity by now
@@ -784,7 +786,7 @@
 	return delayedCookie, deleteSuccess
 }
 
-//returns flowModified, RuleAppendRequest
+// returns flowModified, RuleAppendRequest
 func (oFsm *UniVlanConfigFsm) reviseFlowConstellation(ctx context.Context, aCookie uint64, aUniVlanRuleParams cmn.UniVlanRuleParams) (bool, bool) {
 	flowEntryMatch := false
 	oFsm.mutexFlowParams.Lock()
@@ -822,7 +824,7 @@
 
 // VOL-3828 flow config sequence workaround ###########  end ##########
 
-//RemoveUniFlowParams verifies on existence of flow cookie,
+// RemoveUniFlowParams verifies on existence of flow cookie,
 // if found removes cookie from flow cookie list and if this is empty
 // initiates removal of the flow related configuration from the ONU (via OMCI)
 func (oFsm *UniVlanConfigFsm) RemoveUniFlowParams(ctx context.Context, aCookie uint64, respChan *chan error) error {
@@ -994,10 +996,12 @@
 	return true
 }
 
-//removeFlowFromParamsSlice removes a flow from stored  uniVlanFlowParamsSlice based on the cookie
-//  it assumes that adding cookies for this flow (including the actual one to delete) was prevented
-//  from the start of the deletion request to avoid to much interference
-//  so when called, there can only be one cookie active for this flow
+// removeFlowFromParamsSlice removes a flow from stored  uniVlanFlowParamsSlice based on the cookie
+//
+//	it assumes that adding cookies for this flow (including the actual one to delete) was prevented
+//	from the start of the deletion request to avoid to much interference
+//	so when called, there can only be one cookie active for this flow
+//
 // requires mutexFlowParams to be locked at call
 func (oFsm *UniVlanConfigFsm) removeFlowFromParamsSlice(ctx context.Context, aCookie uint64, aWasConfigured bool) error {
 	logger.Debugw(ctx, "UniVlanConfigFsm flow removal from ParamsSlice", log.Fields{
diff --git a/internal/pkg/devdb/onu_device_db.go b/internal/pkg/devdb/onu_device_db.go
index 99a775b..9bf62e1 100755
--- a/internal/pkg/devdb/onu_device_db.go
+++ b/internal/pkg/devdb/onu_device_db.go
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-//Package devdb provides access to internal device ME DB
+// Package devdb provides access to internal device ME DB
 package devdb
 
 import (
@@ -50,7 +50,7 @@
 }
 type unknownMeAndAttribDbMap map[UnknownMeOrAttribName]map[me.ClassID]map[uint16]unknownAttribs
 
-//OnuDeviceDB structure holds information about ME's
+// OnuDeviceDB structure holds information about ME's
 type OnuDeviceDB struct {
 	ctx                  context.Context
 	deviceID             string
@@ -59,7 +59,7 @@
 	UnknownMeAndAttribDb unknownMeAndAttribDbMap
 }
 
-//NewOnuDeviceDB returns a new instance for a specific ONU_Device_Entry
+// NewOnuDeviceDB returns a new instance for a specific ONU_Device_Entry
 func NewOnuDeviceDB(ctx context.Context, aDeviceID string) *OnuDeviceDB {
 	logger.Debugw(ctx, "Init OnuDeviceDB for:", log.Fields{"device-id": aDeviceID})
 	var OnuDeviceDB OnuDeviceDB
@@ -71,7 +71,7 @@
 	return &OnuDeviceDB
 }
 
-//PutMe puts an ME instance into internal ONU DB
+// PutMe puts an ME instance into internal ONU DB
 func (OnuDeviceDB *OnuDeviceDB) PutMe(ctx context.Context, meClassID me.ClassID, meEntityID uint16, meAttributes me.AttributeValueMap) {
 	OnuDeviceDB.meDbLock.Lock()
 	defer OnuDeviceDB.meDbLock.Unlock()
@@ -99,7 +99,7 @@
 	}
 }
 
-//GetMe returns an ME instance from internal ONU DB
+// GetMe returns an ME instance from internal ONU DB
 func (OnuDeviceDB *OnuDeviceDB) GetMe(meClassID me.ClassID, meEntityID uint16) me.AttributeValueMap {
 	OnuDeviceDB.meDbLock.RLock()
 	defer OnuDeviceDB.meDbLock.RUnlock()
@@ -113,7 +113,7 @@
 	return nil
 }
 
-//GetUint32Attrib converts JSON numbers to uint32
+// GetUint32Attrib converts JSON numbers to uint32
 func (OnuDeviceDB *OnuDeviceDB) GetUint32Attrib(meAttribute interface{}) (uint32, error) {
 
 	switch reflect.TypeOf(meAttribute).Kind() {
@@ -127,7 +127,7 @@
 	}
 }
 
-//GetUint16Attrib converts JSON numbers to uint16
+// GetUint16Attrib converts JSON numbers to uint16
 func (OnuDeviceDB *OnuDeviceDB) GetUint16Attrib(meAttribute interface{}) (uint16, error) {
 
 	switch reflect.TypeOf(meAttribute).Kind() {
@@ -141,7 +141,7 @@
 	}
 }
 
-//GetSortedInstKeys returns a sorted list of all instances of an ME
+// GetSortedInstKeys returns a sorted list of all instances of an ME
 func (OnuDeviceDB *OnuDeviceDB) GetSortedInstKeys(ctx context.Context, meClassID me.ClassID) []uint16 {
 
 	var meInstKeys []uint16
@@ -158,14 +158,14 @@
 	return meInstKeys
 }
 
-//GetNumberOfInst returns the number of instances of an ME
+// GetNumberOfInst returns the number of instances of an ME
 func (OnuDeviceDB *OnuDeviceDB) GetNumberOfInst(meClassID me.ClassID) int {
 	OnuDeviceDB.meDbLock.RLock()
 	defer OnuDeviceDB.meDbLock.RUnlock()
 	return len(OnuDeviceDB.MeDb[meClassID])
 }
 
-//LogMeDb logs content of internal ONU DB
+// LogMeDb logs content of internal ONU DB
 func (OnuDeviceDB *OnuDeviceDB) LogMeDb(ctx context.Context) {
 	logger.Debugw(ctx, "ME instances stored for :", log.Fields{"device-id": OnuDeviceDB.deviceID})
 	for meClassID, meInstMap := range OnuDeviceDB.MeDb {
@@ -176,7 +176,7 @@
 	}
 }
 
-//PutUnknownMeOrAttrib puts an instance with unknown ME or attributes into internal ONU DB
+// PutUnknownMeOrAttrib puts an instance with unknown ME or attributes into internal ONU DB
 func (OnuDeviceDB *OnuDeviceDB) PutUnknownMeOrAttrib(ctx context.Context, aMeName UnknownMeOrAttribName, aMeClassID me.ClassID, aMeEntityID uint16,
 	aMeAttributeMask uint16, aMePayload []byte) {
 
@@ -201,3 +201,10 @@
 		}
 	}
 }
+
+// DeleteMe deletes an ME instance from internal ONU DB
+func (OnuDeviceDB *OnuDeviceDB) DeleteMe(meClassID me.ClassID, meEntityID uint16) {
+	OnuDeviceDB.meDbLock.Lock()
+	defer OnuDeviceDB.meDbLock.Unlock()
+	delete(OnuDeviceDB.MeDb[meClassID], meEntityID)
+}
diff --git a/internal/pkg/pmmgr/onu_metrics_manager.go b/internal/pkg/pmmgr/onu_metrics_manager.go
index 03f7af7..f38fd54 100755
--- a/internal/pkg/pmmgr/onu_metrics_manager.go
+++ b/internal/pkg/pmmgr/onu_metrics_manager.go
@@ -2824,6 +2824,7 @@
 func (mm *OnuMetricsManager) AddGemPortForPerfMonitoring(ctx context.Context, gemPortNTPInstID uint16) {
 	mm.OnuMetricsManagerLock.Lock()
 	defer mm.OnuMetricsManagerLock.Unlock()
+	var meAttributes me.AttributeValueMap
 	logger.Debugw(ctx, "add gemport for perf monitoring - start", log.Fields{"device-id": mm.deviceID, "gemPortID": gemPortNTPInstID})
 	// mark the instance for addition
 	mm.GroupMetricMap[GemPortHistoryName].pmMEData.InstancesToAdd = mm.appendIfMissingUnt16(mm.GroupMetricMap[GemPortHistoryName].pmMEData.InstancesToAdd, gemPortNTPInstID)
@@ -2845,6 +2846,7 @@
 			logger.Warnw(ctx, "error calling event", log.Fields{"device-id": mm.deviceID, "err": err})
 		}
 	}()
+	mm.pOnuDeviceEntry.GetOnuDB().PutMe(ctx, me.GemPortNetworkCtpPerformanceMonitoringHistoryDataClassID, gemPortNTPInstID, meAttributes)
 }
 
 // RemoveGemPortForPerfMonitoring - TODO: add comment
@@ -2871,6 +2873,7 @@
 			logger.Warnw(ctx, "error calling event", log.Fields{"device-id": mm.deviceID, "err": err})
 		}
 	}()
+	mm.pOnuDeviceEntry.GetOnuDB().DeleteMe(me.GemPortNetworkCtpPerformanceMonitoringHistoryDataClassID, gemPortNTPInstID)
 }
 
 func (mm *OnuMetricsManager) updateGemPortNTPInstanceToAddForPerfMonitoring(ctx context.Context) {