[VOL-4148] openonuAdapterGo - crash in flow processing -> vlan config FSM revised

Signed-off-by: mpagenko <michael.pagenkopf@adtran.com>
Change-Id: I8baddcd37ce0f3acb78ba7d5e914455ac866192d
diff --git a/VERSION b/VERSION
index 259bb26..1413d09 100755
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-1.3.2-dev
+1.3.2-dev208
diff --git a/internal/pkg/onuadaptercore/omci_vlan_config.go b/internal/pkg/onuadaptercore/omci_vlan_config.go
index 028245d..dcc0f05 100644
--- a/internal/pkg/onuadaptercore/omci_vlan_config.go
+++ b/internal/pkg/onuadaptercore/omci_vlan_config.go
@@ -87,6 +87,7 @@
 const (
 	// events of config UNI port VLAN FSM
 	vlanEvStart                   = "vlanEvStart"
+	vlanEvPrepareDone             = "vlanEvPrepareDone"
 	vlanEvWaitTechProf            = "vlanEvWaitTechProf"
 	vlanEvCancelOutstandingConfig = "vlanEvCancelOutstandingConfig"
 	vlanEvContinueConfig          = "vlanEvContinueConfig"
@@ -110,6 +111,7 @@
 const (
 	// states of config UNI port VLAN FSM
 	vlanStDisabled        = "vlanStDisabled"
+	vlanStPreparing       = "vlanStPreparing"
 	vlanStStarting        = "vlanStStarting"
 	vlanStWaitingTechProf = "vlanStWaitingTechProf"
 	vlanStConfigVtfd      = "vlanStConfigVtfd"
@@ -139,8 +141,10 @@
 }
 
 type uniRemoveVlanFlowParams struct {
-	cookie         uint64 //just the last cookie valid for removal
-	vlanRuleParams uniVlanRuleParams
+	isSuspendedOnAdd bool
+	removeChannel    chan bool
+	cookie           uint64 //just the last cookie valid for removal
+	vlanRuleParams   uniVlanRuleParams
 }
 
 //UniVlanConfigFsm defines the structure for the state machine for configuration of the VLAN related setting via OMCI
@@ -215,7 +219,8 @@
 	instFsm.pAdaptFsm.pFsm = fsm.NewFSM(
 		vlanStDisabled,
 		fsm.Events{
-			{Name: vlanEvStart, Src: []string{vlanStDisabled}, Dst: vlanStStarting},
+			{Name: vlanEvStart, Src: []string{vlanStDisabled}, Dst: vlanStPreparing},
+			{Name: vlanEvPrepareDone, Src: []string{vlanStPreparing}, Dst: vlanStStarting},
 			{Name: vlanEvWaitTechProf, Src: []string{vlanStStarting}, Dst: vlanStWaitingTechProf},
 			{Name: vlanEvCancelOutstandingConfig, Src: []string{vlanStWaitingTechProf}, Dst: vlanStConfigDone},
 			{Name: vlanEvContinueConfig, Src: []string{vlanStWaitingTechProf}, Dst: vlanStConfigVtfd},
@@ -244,12 +249,13 @@
 			// the only way to get to resource-cleared disabled state again is via "resseting"
 			{Name: vlanEvRestart, Src: []string{vlanStResetting}, Dst: vlanStDisabled},
 			// transitions for reconcile handling according to VOL-3834
-			{Name: vlanEvSkipOmciConfig, Src: []string{vlanStStarting}, Dst: vlanStConfigDone},
+			{Name: vlanEvSkipOmciConfig, Src: []string{vlanStPreparing}, Dst: vlanStConfigDone},
 			{Name: vlanEvSkipOmciConfig, Src: []string{vlanStConfigDone}, Dst: vlanStConfigIncrFlow},
 			{Name: vlanEvSkipIncFlowConfig, Src: []string{vlanStConfigIncrFlow}, Dst: vlanStConfigDone},
 		},
 		fsm.Callbacks{
 			"enter_state":                   func(e *fsm.Event) { instFsm.pAdaptFsm.logFsmStateChange(ctx, e) },
+			"enter_" + vlanStPreparing:      func(e *fsm.Event) { instFsm.enterPreparing(ctx, e) },
 			"enter_" + vlanStStarting:       func(e *fsm.Event) { instFsm.enterConfigStarting(ctx, e) },
 			"enter_" + vlanStConfigVtfd:     func(e *fsm.Event) { instFsm.enterConfigVtfd(ctx, e) },
 			"enter_" + vlanStConfigEvtocd:   func(e *fsm.Event) { instFsm.enterConfigEvtocd(ctx, e) },
@@ -419,6 +425,29 @@
 		}
 	}
 
+	//check if there is some ongoing delete-request running for this flow. If so, block here until this is finished.
+	//  might be accordingly rwCore processing runs into timeout in specific situations - needs to be observed ...
+	//  this is to protect uniVlanFlowParams from adding new or re-writing the same cookie to the rule currently under deletion
+	oFsm.mutexFlowParams.RLock()
+	if len(oFsm.uniRemoveFlowsSlice) > 0 {
+		for flow, removeUniFlowParams := range oFsm.uniRemoveFlowsSlice {
+			if removeUniFlowParams.vlanRuleParams == loRuleParams {
+				// the flow to add is the same as the one already in progress of deleting
+				logger.Infow(ctx, "UniVlanConfigFsm flow setting - suspending rule-add due to ongoing removal", log.Fields{
+					"device-id": oFsm.deviceID, "cookie": removeUniFlowParams.cookie})
+				pRemoveParams := &oFsm.uniRemoveFlowsSlice[flow] //wants to modify the uniRemoveFlowsSlice element directly!
+				oFsm.mutexFlowParams.RUnlock()
+				if err := oFsm.suspendAddRule(ctx, pRemoveParams); err != nil {
+					logger.Errorw(ctx, "UniVlanConfigFsm suspension on add aborted - abort complete add-request", log.Fields{
+						"device-id": oFsm.deviceID, "cookie": removeUniFlowParams.cookie})
+					return fmt.Errorf("abort UniVlanConfigFsm suspension on add %s", oFsm.deviceID)
+				}
+				oFsm.mutexFlowParams.RLock()
+			}
+		}
+	}
+	oFsm.mutexFlowParams.RUnlock()
+
 	flowEntryMatch := false
 	flowCookieModify := false
 	requestAppendRule := false
@@ -527,6 +556,14 @@
 				} else {
 					//some further flows are to be configured
 					//store the actual rule that shall be worked upon in the following transient states
+					if len(oFsm.uniVlanFlowParamsSlice) < int(oFsm.configuredUniFlow) {
+						//check introduced after having observed some panic here
+						logger.Errorw(ctx, "error in FsmEvent handling UniVlanConfigFsm - inconsistent counter",
+							log.Fields{"configuredUniFlow": oFsm.configuredUniFlow,
+								"sliceLen": len(oFsm.uniVlanFlowParamsSlice), "device-id": oFsm.deviceID})
+						oFsm.mutexFlowParams.Unlock()
+						return fmt.Errorf("abort UniVlanConfigFsm on add due to internal counter mismatch %s", oFsm.deviceID)
+					}
 					oFsm.actualUniVlanConfigRule = oFsm.uniVlanFlowParamsSlice[oFsm.configuredUniFlow].VlanRuleParams
 					//tpId of the next rule to be configured
 					tpID := oFsm.actualUniVlanConfigRule.TpID
@@ -602,12 +639,40 @@
 	return nil
 }
 
+func (oFsm *UniVlanConfigFsm) suspendAddRule(ctx context.Context, apRemoveFlowParams *uniRemoveVlanFlowParams) error {
+	oFsm.mutexFlowParams.Lock()
+	deleteChannel := apRemoveFlowParams.removeChannel
+	apRemoveFlowParams.isSuspendedOnAdd = true
+	oFsm.mutexFlowParams.Unlock()
+
+	// isSuspendedOnAdd is not reset here-after as the assumption is, that after
+	select {
+	case success := <-deleteChannel:
+		//no need to reset isSuspendedOnAdd as in this case the removeElement will be deleted completely
+		if success {
+			logger.Infow(ctx, "resume adding this rule after having completed deletion", log.Fields{
+				"device-id": oFsm.deviceID})
+			return nil
+		}
+		return fmt.Errorf("suspend aborted, also aborting add-activity: %s", oFsm.deviceID)
+	case <-time.After(oFsm.pOmciCC.GetMaxOmciTimeoutWithRetries() * time.Second):
+		oFsm.mutexFlowParams.Lock()
+		if apRemoveFlowParams != nil {
+			apRemoveFlowParams.isSuspendedOnAdd = false
+		}
+		oFsm.mutexFlowParams.Unlock()
+		logger.Errorw(ctx, "timeout waiting for deletion of rule, just try to continue", log.Fields{
+			"device-id": oFsm.deviceID})
+	}
+	return nil
+}
+
 // VOL-3828 flow config sequence workaround ###########  start ##########
 func (oFsm *UniVlanConfigFsm) delayNewRuleForCookie(ctx context.Context, aCookieSlice []uint64) uint64 {
 	//assumes mutexFlowParams.Lock() protection from caller!
 	if oFsm.delayNewRuleCookie == 0 && len(aCookieSlice) == 1 {
 		// if not already waiting, limitation for this workaround is to just have one overlapping cookie/rule
-		// suspend check is done only of there is only one cookie in the request
+		// suspend check is done only if there is only one cookie in the request
 		//  background: more elements only expected in reconcile use case, where no conflicting sequence is to be expected
 		newCookie := aCookieSlice[0]
 		for _, storedUniFlowParams := range oFsm.uniVlanFlowParamsSlice {
@@ -719,7 +784,7 @@
 					//  so we have to check if we have to abort the outstanding AddRequest and regard the current DelRequest as done
 					//  if the Fsm is in some other transient (config) state, we will reach the DelRequest later and correctly process it then
 					if pConfigVlanStateBaseFsm.Is(vlanStWaitingTechProf) {
-						logger.Debugw(ctx, "UniVlanConfigFsm was waiting for TechProf config with this rule, aborting the outstanding config",
+						logger.Debugw(ctx, "UniVlanConfigFsm was waiting for TechProf config with add-request, just aborting the outstanding add",
 							log.Fields{"device-id": oFsm.deviceID})
 						cancelPendingConfig = true
 					} else {
@@ -728,44 +793,32 @@
 							vlanRuleParams: storedUniFlowParams.VlanRuleParams,
 							cookie:         aCookie,
 						}
+						loRemoveParams.removeChannel = make(chan bool)
 						oFsm.uniRemoveFlowsSlice = append(oFsm.uniRemoveFlowsSlice, loRemoveParams)
 					}
 
-					//and remove the actual element from the addVlanFlow slice
-					// oFsm.uniVlanFlowParamsSlice[flow].CookieSlice = nil //automatically done by garbage collector
+					usedTpID := storedUniFlowParams.VlanRuleParams.TpID
 					if len(oFsm.uniVlanFlowParamsSlice) <= 1 {
-						oFsm.numUniFlows = 0              //no more flows
-						oFsm.configuredUniFlow = 0        //no more flows configured
-						oFsm.uniVlanFlowParamsSlice = nil //reset the slice
-						//at this point it is evident that no flow anymore refers to a still possibly active Techprofile
+						//at this point it is evident that no flow anymore will refer to a still possibly active Techprofile
 						//request that this profile gets deleted before a new flow add is allowed (except for some aborted add)
 						if !cancelPendingConfig {
-							oFsm.pUniTechProf.setProfileToDelete(oFsm.pOnuUniPort.uniID, loRemoveParams.vlanRuleParams.TpID, true)
+							logger.Debugw(ctx, "UniVlanConfigFsm flow removal requested - set TechProfile to-delete", log.Fields{
+								"device-id": oFsm.deviceID})
+							if oFsm.pUniTechProf != nil {
+								oFsm.pUniTechProf.setProfileToDelete(oFsm.pOnuUniPort.uniID, usedTpID, true)
+							}
 						}
-						logger.Debugw(ctx, "UniVlanConfigFsm flow removal - no more flows", log.Fields{
-							"device-id": oFsm.deviceID})
 					} else {
-						oFsm.numUniFlows--
-						if oFsm.configuredUniFlow > 0 {
-							oFsm.configuredUniFlow--
-							//TODO!! might be needed to consider still outstanding configure requests ..
-							//  so a flow at removal might still not be configured !?!
-						}
-						usedTpID := storedUniFlowParams.VlanRuleParams.TpID
-						//cut off the requested flow by slicing out this element
-						oFsm.uniVlanFlowParamsSlice = append(
-							oFsm.uniVlanFlowParamsSlice[:flow], oFsm.uniVlanFlowParamsSlice[flow+1:]...)
-						//here we have to check, if there are still other flows referencing to the actual ProfileId
-						//  before we can request that this profile gets deleted before a new flow add is allowed
-						//  (needed to extract to function due to lint complexity)
 						if !cancelPendingConfig {
 							oFsm.updateTechProfileToDelete(ctx, usedTpID)
 						}
-						logger.Debugw(ctx, "UniVlanConfigFsm flow removal - specific flow removed from data", log.Fields{
-							"device-id": oFsm.deviceID})
 					}
 					//trigger the FSM to remove the relevant rule
 					if cancelPendingConfig {
+						//as the uniFlow parameters are already stored (for add) but no explicit removal is done anymore
+						//  the paramSlice has to be updated with rule-removal, which also then updates numUniFlows
+						oFsm.removeFlowFromParamsSlice(ctx, aCookie, false) //call from 'non-configured' state of the rule
+
 						oFsm.requestEventOffset = uint8(cDeviceEventOffsetRemoveWithKvStore) //offset for last flow-remove activity (with kvStore request)
 						//attention: take care to release and re-take the mutexFlowParams when calling the FSM directly -
 						//  synchronous FSM 'event/state' functions may rely on this mutex
@@ -844,6 +897,53 @@
 	return nil
 }
 
+//removeFlowFromParamsSlice removes a flow from stored  uniVlanFlowParamsSlice based on the cookie
+//  it assumes that adding cookies for this flow (including the actual one to delete) was prevented
+//  from the start of the deletion request to avoid to much interference
+//  so when called, there can only be one cookie active for this flow
+// requires mutexFlowParams to be locked at call
+func (oFsm *UniVlanConfigFsm) removeFlowFromParamsSlice(ctx context.Context, aCookie uint64, aWasConfigured bool) {
+	logger.Debugw(ctx, "UniVlanConfigFsm flow removal from ParamsSlice", log.Fields{
+		"device-id": oFsm.deviceID, "cookie": aCookie})
+removeFromSlice_loop:
+	for flow, storedUniFlowParams := range oFsm.uniVlanFlowParamsSlice {
+		// if UniFlowParams exists, cookieSlice is assumed to have always at least one element
+		if storedUniFlowParams.CookieSlice[0] == aCookie {
+			if len(storedUniFlowParams.CookieSlice) != 1 {
+				errStr := "UniVlanConfigFsm flow removal from ParamsSlice abort - unexpected cookie slice length"
+				logger.Errorw(ctx, errStr, log.Fields{
+					"cookieSliceLen": len(oFsm.uniVlanFlowParamsSlice), "device-id": oFsm.deviceID})
+				return
+			}
+			logger.Debugw(ctx, "UniVlanConfigFsm flow removal from ParamsSlice - cookie found", log.Fields{
+				"device-id": oFsm.deviceID, "cookie": aCookie})
+			//remove the actual element from the addVlanFlow slice
+			// oFsm.uniVlanFlowParamsSlice[flow].CookieSlice = nil //automatically done by garbage collector
+			if len(oFsm.uniVlanFlowParamsSlice) <= 1 {
+				oFsm.numUniFlows = 0              //no more flows
+				oFsm.configuredUniFlow = 0        //no more flows configured
+				oFsm.uniVlanFlowParamsSlice = nil //reset the slice
+				//at this point it is evident that no flow anymore refers to a still possibly active Techprofile
+				//request that this profile gets deleted before a new flow add is allowed
+				logger.Debugw(ctx, "UniVlanConfigFsm flow removal from ParamsSlice - no more flows", log.Fields{
+					"device-id": oFsm.deviceID})
+			} else {
+				oFsm.numUniFlows--
+				if aWasConfigured && oFsm.configuredUniFlow > 0 {
+					oFsm.configuredUniFlow--
+				}
+				//cut off the requested flow by slicing out this element
+				oFsm.uniVlanFlowParamsSlice = append(
+					oFsm.uniVlanFlowParamsSlice[:flow], oFsm.uniVlanFlowParamsSlice[flow+1:]...)
+				logger.Debugw(ctx, "UniVlanConfigFsm flow removal - specific flow removed from data", log.Fields{
+					"device-id": oFsm.deviceID})
+			}
+			break removeFromSlice_loop //found the cookie - no further search for this requested cookie
+		}
+	} //search all flows
+}
+
+// requires mutexFlowParams to be locked at call
 func (oFsm *UniVlanConfigFsm) updateTechProfileToDelete(ctx context.Context, usedTpID uint8) {
 	//here we have to check, if there are still other flows referencing to the actual ProfileId
 	//  before we can request that this profile gets deleted before a new flow add is allowed
@@ -858,16 +958,17 @@
 		logger.Debugw(ctx, "UniVlanConfigFsm tp-id used in deleted flow is still used in other flows", log.Fields{
 			"device-id": oFsm.deviceID, "tp-id": usedTpID})
 	} else {
-		logger.Debugw(ctx, "UniVlanConfigFsm tp-id used in deleted flow is not used anymore", log.Fields{
+		logger.Debugw(ctx, "UniVlanConfigFsm tp-id used in deleted flow is not used anymore - set TechProfile to-delete", log.Fields{
 			"device-id": oFsm.deviceID, "tp-id": usedTpID})
-		//request that this profile gets deleted before a new flow add is allowed
-		oFsm.pUniTechProf.setProfileToDelete(oFsm.pOnuUniPort.uniID, usedTpID, true)
+		if oFsm.pUniTechProf != nil {
+			//request that this profile gets deleted before a new flow add is allowed
+			oFsm.pUniTechProf.setProfileToDelete(oFsm.pOnuUniPort.uniID, usedTpID, true)
+		}
 	}
 }
 
-func (oFsm *UniVlanConfigFsm) enterConfigStarting(ctx context.Context, e *fsm.Event) {
-	logger.Debugw(ctx, "UniVlanConfigFsm start", log.Fields{"in state": e.FSM.Current(),
-		"device-id": oFsm.deviceID})
+func (oFsm *UniVlanConfigFsm) enterPreparing(ctx context.Context, e *fsm.Event) {
+	logger.Debugw(ctx, "UniVlanConfigFsm preparing", log.Fields{"device-id": oFsm.deviceID})
 
 	// this FSM is not intended for re-start, needs always new creation for a new run
 	// (self-destroying - compare enterDisabled())
@@ -878,15 +979,30 @@
 	//let the state machine run forward from here directly
 	pConfigVlanStateAFsm := oFsm.pAdaptFsm
 	if pConfigVlanStateAFsm != nil {
-
 		if oFsm.pDeviceHandler.isSkipOnuConfigReconciling() {
 			logger.Debugw(ctx, "reconciling - skip omci-config of vlan rule",
 				log.Fields{"fsmState": oFsm.pAdaptFsm.pFsm.Current(), "device-id": oFsm.deviceID})
+			// Can't call FSM Event directly, decoupling it
 			go func(a_pAFsm *AdapterFsm) {
 				_ = a_pAFsm.pFsm.Event(vlanEvSkipOmciConfig)
 			}(pConfigVlanStateAFsm)
 			return
 		}
+		// Can't call FSM Event directly, decoupling it
+		go func(a_pAFsm *AdapterFsm) {
+			_ = a_pAFsm.pFsm.Event(vlanEvPrepareDone)
+		}(pConfigVlanStateAFsm)
+		return
+	}
+	logger.Errorw(ctx, "UniVlanConfigFsm abort: invalid FSM pointer", log.Fields{
+		"in state": e.FSM.Current(), "device-id": oFsm.deviceID})
+	//should never happen, else: recovery would be needed from outside the FSM
+}
+
+func (oFsm *UniVlanConfigFsm) enterConfigStarting(ctx context.Context, e *fsm.Event) {
+	logger.Debugw(ctx, "UniVlanConfigFsm start vlan configuration", log.Fields{"device-id": oFsm.deviceID})
+	pConfigVlanStateAFsm := oFsm.pAdaptFsm
+	if pConfigVlanStateAFsm != nil {
 		oFsm.mutexFlowParams.Lock()
 		//possibly the entry is not valid anymore based on intermediate delete requests
 		//just a basic protection ...
@@ -998,18 +1114,20 @@
 
 func (oFsm *UniVlanConfigFsm) enterConfigEvtocd(ctx context.Context, e *fsm.Event) {
 	logger.Debugw(ctx, "UniVlanConfigFsm - start config EVTOCD loop", log.Fields{
-		"in state": e.FSM.Current(), "device-id": oFsm.deviceID})
+		"device-id": oFsm.deviceID})
 	oFsm.requestEventOffset = uint8(cDeviceEventOffsetAddWithKvStore) //0 offset for last flow-add activity
 	go func() {
 		//using the first element in the slice because it's the first flow per definition here
 		errEvto := oFsm.performConfigEvtocdEntries(ctx, 0)
 		//This is correct passing scenario
 		if errEvto == nil {
+			oFsm.mutexFlowParams.RLock()
 			tpID := oFsm.actualUniVlanConfigRule.TpID
 			vlanID := oFsm.actualUniVlanConfigRule.SetVid
 			for _, gemPort := range oFsm.pUniTechProf.getMulticastGemPorts(ctx, oFsm.pOnuUniPort.uniID, uint8(tpID)) {
 				logger.Infow(ctx, "Setting multicast MEs, with first flow", log.Fields{"deviceID": oFsm.deviceID,
 					"techProfile": tpID, "gemPort": gemPort, "vlanID": vlanID, "configuredUniFlow": oFsm.configuredUniFlow})
+				oFsm.mutexFlowParams.RUnlock()
 				errCreateAllMulticastME := oFsm.performSettingMulticastME(ctx, tpID, gemPort,
 					vlanID)
 				if errCreateAllMulticastME != nil {
@@ -1017,7 +1135,9 @@
 						log.Fields{"device-id": oFsm.deviceID})
 					_ = oFsm.pAdaptFsm.pFsm.Event(vlanEvReset)
 				}
+				oFsm.mutexFlowParams.RLock()
 			}
+			oFsm.mutexFlowParams.RUnlock()
 			//TODO Possibly insert new state for multicast --> possibly another jira/later time.
 			_ = oFsm.pAdaptFsm.pFsm.Event(vlanEvRxConfigEvtocd)
 		}
@@ -1026,14 +1146,14 @@
 
 func (oFsm *UniVlanConfigFsm) enterVlanConfigDone(ctx context.Context, e *fsm.Event) {
 
-	oFsm.mutexFlowParams.RLock()
-	defer oFsm.mutexFlowParams.RUnlock()
+	oFsm.mutexFlowParams.Lock()
 
 	logger.Infow(ctx, "UniVlanConfigFsm config done - checking on more flows", log.Fields{
-		"in state": e.FSM.Current(), "device-id": oFsm.deviceID,
+		"device-id":         oFsm.deviceID,
 		"overall-uni-rules": oFsm.numUniFlows, "configured-uni-rules": oFsm.configuredUniFlow})
 	pConfigVlanStateAFsm := oFsm.pAdaptFsm
 	if pConfigVlanStateAFsm == nil {
+		oFsm.mutexFlowParams.Unlock()
 		logger.Errorw(ctx, "UniVlanConfigFsm abort: invalid FSM pointer", log.Fields{
 			"in state": e.FSM.Current(), "device-id": oFsm.deviceID})
 		//should never happen, else: recovery would be needed from outside the FSM
@@ -1046,6 +1166,7 @@
 			"device-id": oFsm.deviceID, "uni-id": oFsm.pOnuUniPort.uniID,
 			"tp-id":    oFsm.uniRemoveFlowsSlice[0].vlanRuleParams.TpID,
 			"set-Vlan": oFsm.uniRemoveFlowsSlice[0].vlanRuleParams.SetVid})
+		oFsm.mutexFlowParams.Unlock()
 		// Can't call FSM Event directly, decoupling it
 		go func(a_pBaseFsm *fsm.FSM) {
 			_ = a_pBaseFsm.Event(vlanEvRemFlowConfig)
@@ -1061,10 +1182,12 @@
 		}
 		logger.Debugw(ctx, "reconciling - skip enterVlanConfigDone processing",
 			log.Fields{"numUniFlows": oFsm.numUniFlows, "configuredUniFlow": oFsm.configuredUniFlow, "device-id": oFsm.deviceID})
+		oFsm.mutexFlowParams.Unlock()
 		return
 	}
 	if oFsm.numUniFlows > oFsm.configuredUniFlow {
 		if oFsm.configuredUniFlow == 0 {
+			oFsm.mutexFlowParams.Unlock()
 			// this is a restart with a complete new flow, we can re-use the initial flow config control
 			// including the check, if the related techProfile is (still) available (probably also removed in between)
 			// Can't call FSM Event directly, decoupling it
@@ -1076,6 +1199,17 @@
 
 		//some further flows are to be configured
 		//store the actual rule that shall be worked upon in the following transient states
+		if len(oFsm.uniVlanFlowParamsSlice) < int(oFsm.configuredUniFlow) {
+			//check introduced after having observed some panic in this processing
+			logger.Errorw(ctx, "error in FsmEvent handling UniVlanConfigFsm in ConfigDone - inconsistent counter",
+				log.Fields{"configuredUniFlow": oFsm.configuredUniFlow,
+					"sliceLen": len(oFsm.uniVlanFlowParamsSlice), "device-id": oFsm.deviceID})
+			oFsm.mutexFlowParams.Unlock()
+			go func(a_pAFsm *AdapterFsm) {
+				_ = a_pAFsm.pFsm.Event(vlanEvReset)
+			}(pConfigVlanStateAFsm)
+			return
+		}
 		oFsm.actualUniVlanConfigRule = oFsm.uniVlanFlowParamsSlice[oFsm.configuredUniFlow].VlanRuleParams
 		//tpId of the next rule to be configured
 		tpID := oFsm.actualUniVlanConfigRule.TpID
@@ -1084,6 +1218,7 @@
 		logger.Debugw(ctx, "UniVlanConfigFsm - incremental config request", log.Fields{
 			"device-id": oFsm.deviceID, "uni-id": oFsm.pOnuUniPort.uniID,
 			"set-Vlan": oFsm.actualUniVlanConfigRule.SetVid, "tp-id": tpID, "ProfDone": loTechProfDone})
+		oFsm.mutexFlowParams.Unlock()
 		// Can't call FSM Event directly, decoupling it
 		go func(aPBaseFsm *fsm.FSM, aTechProfDone bool) {
 			if aTechProfDone {
@@ -1096,6 +1231,7 @@
 		}(pConfigVlanStateBaseFsm, loTechProfDone)
 		return
 	}
+	oFsm.mutexFlowParams.Unlock()
 	logger.Debugw(ctx, "UniVlanConfigFsm - VLAN config done: send dh event notification", log.Fields{
 		"device-id": oFsm.deviceID})
 	// it might appear that some flows are requested also after 'flowPushed' event has been generated ...
@@ -1119,13 +1255,12 @@
 	}
 	oFsm.mutexFlowParams.Lock()
 	logger.Debugw(ctx, "UniVlanConfigFsm - start config further incremental flow", log.Fields{
-		"in state": e.FSM.Current(), "recent flow-number": oFsm.configuredUniFlow,
-		"device-id": oFsm.deviceID})
+		"recent flow-number": oFsm.configuredUniFlow,
+		"device-id":          oFsm.deviceID})
 	oFsm.TpIDWaitingFor = 0 //reset indication to avoid misinterpretation
 
 	if oFsm.actualUniVlanConfigRule.SetVid == uint32(of.OfpVlanId_OFPVID_PRESENT) {
 		// meaning transparent setup - no specific VTFD setting required
-		oFsm.mutexFlowParams.Unlock()
 		logger.Debugw(ctx, "UniVlanConfigFsm: no VTFD config required", log.Fields{
 			"in state": e.FSM.Current(), "device-id": oFsm.deviceID})
 	} else {
@@ -1140,8 +1275,8 @@
 			//no VTFD yet created
 			logger.Debugw(ctx, "UniVlanConfigFsm create VTFD", log.Fields{
 				"EntitytId": strconv.FormatInt(int64(vtfdID), 16),
-				"in state":  e.FSM.Current(), "device-id": oFsm.deviceID,
-				"macBpNo": oFsm.pOnuUniPort.macBpNo, "TpID": oFsm.actualUniVlanConfigRule.TpID})
+				"device-id": oFsm.deviceID,
+				"macBpNo":   oFsm.pOnuUniPort.macBpNo, "TpID": oFsm.actualUniVlanConfigRule.TpID})
 			// 'SetVid' below is assumed to be masked already by the caller to 12 bit
 			oFsm.vlanFilterList[0] = uint16(oFsm.actualUniVlanConfigRule.SetVid)
 
@@ -1161,6 +1296,7 @@
 				oFsm.pAdaptFsm.commChan, meParams)
 			if err != nil {
 				oFsm.mutexPLastTxMeInstance.Unlock()
+				oFsm.mutexFlowParams.Unlock()
 				logger.Errorw(ctx, "VTFD create failed, aborting UniVlanConfig FSM!",
 					log.Fields{"device-id": oFsm.deviceID})
 				pConfigVlanStateAFsm := oFsm.pAdaptFsm
@@ -1185,8 +1321,8 @@
 
 			logger.Debugw(ctx, "UniVlanConfigFsm set VTFD", log.Fields{
 				"EntitytId": strconv.FormatInt(int64(vtfdID), 16),
-				"in state":  e.FSM.Current(), "device-id": oFsm.deviceID,
-				"macBpNo": oFsm.pOnuUniPort.macBpNo, "TpID": oFsm.actualUniVlanConfigRule.TpID})
+				"device-id": oFsm.deviceID,
+				"macBpNo":   oFsm.pOnuUniPort.macBpNo, "TpID": oFsm.actualUniVlanConfigRule.TpID})
 			// setVid is assumed to be masked already by the caller to 12 bit
 			oFsm.vlanFilterList[oFsm.numVlanFilterEntries] =
 				uint16(oFsm.actualUniVlanConfigRule.SetVid)
@@ -1211,6 +1347,7 @@
 				oFsm.pAdaptFsm.commChan, meParams)
 			if err != nil {
 				oFsm.mutexPLastTxMeInstance.Unlock()
+				oFsm.mutexFlowParams.Unlock()
 				logger.Errorw(ctx, "UniVlanFsm create Vlan Tagging Filter ME result error",
 					log.Fields{"device-id": oFsm.deviceID, "Error": err})
 				_ = oFsm.pAdaptFsm.pFsm.Event(vlanEvReset)
@@ -1224,10 +1361,10 @@
 			oFsm.pLastTxMeInstance = meInstance
 			oFsm.mutexPLastTxMeInstance.Unlock()
 		}
-		oFsm.mutexFlowParams.Unlock()
 		//verify response
 		err := oFsm.waitforOmciResponse(ctx)
 		if err != nil {
+			oFsm.mutexFlowParams.Unlock()
 			logger.Errorw(ctx, "VTFD create/set failed, aborting VlanConfig FSM!",
 				log.Fields{"device-id": oFsm.deviceID})
 			pConfigVlanStateBaseFsm := oFsm.pAdaptFsm.pFsm
@@ -1238,7 +1375,7 @@
 			return
 		}
 	}
-	oFsm.mutexFlowParams.Lock()
+
 	oFsm.requestEventOffset = uint8(cDeviceEventOffsetAddWithKvStore) //0 offset for last flow-add activity
 	oFsm.mutexFlowParams.Unlock()
 	go func() {
@@ -1255,7 +1392,7 @@
 				vlanID := oFsm.actualUniVlanConfigRule.SetVid
 				logger.Infow(ctx, "Setting multicast MEs for additional flows", log.Fields{"deviceID": oFsm.deviceID,
 					"techProfile": tpID, "gemPort": gemPort,
-					"vlanID": vlanID, "configuredUniFlow": oFsm.configuredUniFlow})
+					"vlanID": vlanID, "configuredUniFlow": configuredUniFlow})
 				oFsm.mutexFlowParams.RUnlock()
 				errCreateAllMulticastME := oFsm.performSettingMulticastME(ctx, tpID, gemPort, vlanID)
 				if errCreateAllMulticastME != nil {
@@ -1272,8 +1409,8 @@
 func (oFsm *UniVlanConfigFsm) enterRemoveFlow(ctx context.Context, e *fsm.Event) {
 	oFsm.mutexFlowParams.RLock()
 	logger.Debugw(ctx, "UniVlanConfigFsm - start removing the top remove-flow", log.Fields{
-		"in state": e.FSM.Current(), "with last cookie": oFsm.uniRemoveFlowsSlice[0].cookie,
-		"device-id": oFsm.deviceID})
+		"with last cookie": oFsm.uniRemoveFlowsSlice[0].cookie,
+		"device-id":        oFsm.deviceID})
 
 	pConfigVlanStateBaseFsm := oFsm.pAdaptFsm.pFsm
 	loAllowSpecificOmciConfig := oFsm.pDeviceHandler.isReadyForOmciConfig()
@@ -1299,8 +1436,8 @@
 			//  so we can just delete the VTFD entry
 			logger.Debugw(ctx, "UniVlanConfigFsm: VTFD delete (no more vlan filters)",
 				log.Fields{"current vlan list": oFsm.vlanFilterList, "EntitytId": strconv.FormatInt(int64(vtfdID), 16),
-					"in state": e.FSM.Current(), "device-id": oFsm.deviceID,
-					"macBpNo": oFsm.pOnuUniPort.macBpNo, "TpID": loRuleParams.TpID})
+					"device-id": oFsm.deviceID,
+					"macBpNo":   oFsm.pOnuUniPort.macBpNo, "TpID": loRuleParams.TpID})
 			loVlanEntryClear = 1           //full VlanFilter clear request
 			if loAllowSpecificOmciConfig { //specific OMCI config is expected to work acc. to the device state
 				oFsm.mutexPLastTxMeInstance.Lock()
@@ -1427,6 +1564,17 @@
 	}
 	oFsm.mutexFlowParams.Lock()
 	deletedCookie := oFsm.uniRemoveFlowsSlice[0].cookie
+
+	// here we need o finally remove the removed data also from uniVlanFlowParamsSlice and possibly have to
+	//  stop the suspension of a add-activity waiting for the end of removal
+	oFsm.removeFlowFromParamsSlice(ctx, deletedCookie, true) //call from 'configured' state of the rule
+	if oFsm.uniRemoveFlowsSlice[0].isSuspendedOnAdd {
+		removeChannel := oFsm.uniRemoveFlowsSlice[0].removeChannel
+		oFsm.mutexFlowParams.Unlock()
+		removeChannel <- true
+		oFsm.mutexFlowParams.Lock()
+	}
+
 	logger.Debugw(ctx, "UniVlanConfigFsm - removing the removal data", log.Fields{
 		"in state": e.FSM.Current(), "device-id": oFsm.deviceID,
 		"removed cookie": deletedCookie, "waitForDeleteCookie": oFsm.delayNewRuleCookie})
@@ -1509,12 +1657,32 @@
 	oFsm.mutexPLastTxMeInstance.Lock()
 	oFsm.pLastTxMeInstance = nil
 	oFsm.mutexPLastTxMeInstance.Unlock()
+
+	oFsm.mutexFlowParams.RLock()
+	if oFsm.delayNewRuleCookie != 0 {
+		// looks like the waiting AddFlow is stuck
+		oFsm.mutexFlowParams.RUnlock()
+		oFsm.chCookieDeleted <- true // let the waiting AddFlow thread continue/terminate
+		oFsm.mutexFlowParams.RLock()
+	}
+	if len(oFsm.uniRemoveFlowsSlice) > 0 {
+		for _, removeUniFlowParams := range oFsm.uniRemoveFlowsSlice {
+			if removeUniFlowParams.isSuspendedOnAdd {
+				removeChannel := removeUniFlowParams.removeChannel
+				logger.Debugw(ctx, "UniVlanConfigFsm flow clear-up - abort suspended rule-add", log.Fields{
+					"device-id": oFsm.deviceID, "cookie": removeUniFlowParams.cookie})
+				oFsm.mutexFlowParams.RUnlock()
+				removeChannel <- false
+				oFsm.mutexFlowParams.RLock()
+			}
+		}
+	}
+
 	if oFsm.pDeviceHandler != nil {
 		//TODO: to clarify with improved error treatment for VlanConfigFsm (timeout,reception) errors
 		//  current code removes the complete FSM including all flow/rule configuration done so far
 		//  this might be a bit to much, it would require fully new flow config from rwCore (at least on OnuDown/up)
 		//  maybe a more sophisticated approach is possible without clearing the data
-		oFsm.mutexFlowParams.RLock()
 		if oFsm.clearPersistency {
 			//permanently remove possibly stored persistent data
 			if len(oFsm.uniVlanFlowParamsSlice) > 0 {
@@ -1524,14 +1692,12 @@
 		} else {
 			logger.Debugw(ctx, "UniVlanConfigFsm persistency data not cleared", log.Fields{"device-id": oFsm.deviceID})
 		}
-		if oFsm.delayNewRuleCookie != 0 {
-			// looks like the waiting AddFlow is stuck
-			oFsm.chCookieDeleted <- true // let the waiting AddFlow thread continue/treminate
-		}
 		oFsm.mutexFlowParams.RUnlock()
 		//request removal of 'reference' in the Handler (completely clear the FSM and its data)
 		go oFsm.pDeviceHandler.RemoveVlanFilterFsm(ctx, oFsm.pOnuUniPort)
+		return
 	}
+	oFsm.mutexFlowParams.RUnlock()
 }
 
 func (oFsm *UniVlanConfigFsm) processOmciVlanMessages(ctx context.Context) { //ctx context.Context?
@@ -1746,12 +1912,16 @@
 }
 
 func (oFsm *UniVlanConfigFsm) performConfigEvtocdEntries(ctx context.Context, aFlowEntryNo uint8) error {
+	oFsm.mutexFlowParams.RLock()
+	evtocdID := oFsm.evtocdID
+	oFsm.mutexFlowParams.RUnlock()
+
 	if aFlowEntryNo == 0 {
 		// EthType set only at first flow element
 		// EVTOCD ME is expected to exist at this point already from MIB-Download (with AssociationType/Pointer)
 		// we need to extend the configuration by EthType definition and, to be sure, downstream 'inverse' mode
 		logger.Debugw(ctx, "UniVlanConfigFsm Tx Create::EVTOCD", log.Fields{
-			"EntitytId":  strconv.FormatInt(int64(oFsm.evtocdID), 16),
+			"EntitytId":  strconv.FormatInt(int64(evtocdID), 16),
 			"i/oEthType": strconv.FormatInt(int64(cDefaultTpid), 16),
 			"device-id":  oFsm.deviceID})
 		associationType := 2 // default to uniPPTP
@@ -1760,7 +1930,7 @@
 		}
 		// Create the EVTOCD ME
 		meParams := me.ParamData{
-			EntityID: oFsm.evtocdID,
+			EntityID: evtocdID,
 			Attributes: me.AttributeValueMap{
 				"AssociationType":     uint8(associationType),
 				"AssociatedMePointer": oFsm.pOnuUniPort.entityID,
@@ -1792,7 +1962,7 @@
 
 		// Set the EVTOCD ME default params
 		meParams = me.ParamData{
-			EntityID: oFsm.evtocdID,
+			EntityID: evtocdID,
 			Attributes: me.AttributeValueMap{
 				"InputTpid":      uint16(cDefaultTpid), //could be possibly retrieved from flow config one day, by now just like py-code base
 				"OutputTpid":     uint16(cDefaultTpid), //could be possibly retrieved from flow config one day, by now just like py-code base
@@ -1856,7 +2026,7 @@
 				cSetOutputTpidCopyDei<<cTreatTpidOffset) // Set TPID = 0x8100
 
 		meParams := me.ParamData{
-			EntityID: oFsm.evtocdID,
+			EntityID: evtocdID,
 			Attributes: me.AttributeValueMap{
 				"ReceivedFrameVlanTaggingOperationTable": sliceEvtocdRule,
 			},
@@ -1922,7 +2092,7 @@
 			oFsm.mutexFlowParams.RUnlock()
 
 			meParams := me.ParamData{
-				EntityID: oFsm.evtocdID,
+				EntityID: evtocdID,
 				Attributes: me.AttributeValueMap{
 					"ReceivedFrameVlanTaggingOperationTable": sliceEvtocdRule,
 				},
@@ -1984,7 +2154,7 @@
 
 				oFsm.mutexFlowParams.RUnlock()
 				meParams := me.ParamData{
-					EntityID: oFsm.evtocdID,
+					EntityID: evtocdID,
 					Attributes: me.AttributeValueMap{
 						"ReceivedFrameVlanTaggingOperationTable": sliceEvtocdRule,
 					},
@@ -2047,7 +2217,7 @@
 				oFsm.mutexFlowParams.RUnlock()
 
 				meParams := me.ParamData{
-					EntityID: oFsm.evtocdID,
+					EntityID: evtocdID,
 					Attributes: me.AttributeValueMap{
 						"ReceivedFrameVlanTaggingOperationTable": sliceEvtocdRule,
 					},
@@ -2083,11 +2253,17 @@
 
 	// if Config has been done for all EVTOCD entries let the FSM proceed
 	logger.Debugw(ctx, "EVTOCD set loop finished", log.Fields{"device-id": oFsm.deviceID})
+	oFsm.mutexFlowParams.Lock()
 	oFsm.configuredUniFlow++ // one (more) flow configured
+	oFsm.mutexFlowParams.Unlock()
 	return nil
 }
 
 func (oFsm *UniVlanConfigFsm) removeEvtocdEntries(ctx context.Context, aRuleParams uniVlanRuleParams) {
+	oFsm.mutexFlowParams.RLock()
+	evtocdID := oFsm.evtocdID
+	oFsm.mutexFlowParams.RUnlock()
+
 	// configured Input/Output TPID is not modified again - no influence if no filter is applied
 	if aRuleParams.SetVid == uint32(of.OfpVlanId_OFPVID_PRESENT) {
 		//transparent transmission was set
@@ -2120,7 +2296,7 @@
 				cDontCareTpid<<cTreatTpidOffset) // copy TPID and DEI
 
 		meParams := me.ParamData{
-			EntityID: oFsm.evtocdID,
+			EntityID: evtocdID,
 			Attributes: me.AttributeValueMap{
 				"ReceivedFrameVlanTaggingOperationTable": sliceEvtocdRule,
 			},
@@ -2151,7 +2327,9 @@
 		}
 	} else {
 		// according to py-code acceptIncrementalEvto program option decides upon stacking or translation scenario
+		oFsm.mutexFlowParams.RLock()
 		if oFsm.acceptIncrementalEvtoOption {
+			oFsm.mutexFlowParams.RUnlock()
 			// this defines VID translation scenario: singletagged->singletagged (if not transparent)
 			logger.Debugw(ctx, "UniVlanConfigFsm Tx Set::EVTOCD clear single tagged translation rule", log.Fields{
 				"device-id": oFsm.deviceID, "match-vlan": aRuleParams.MatchVid})
@@ -2173,7 +2351,7 @@
 			binary.BigEndian.PutUint32(sliceEvtocdRule[cTreatInnerOffset:], 0xFFFFFFFF)
 
 			meParams := me.ParamData{
-				EntityID: oFsm.evtocdID,
+				EntityID: evtocdID,
 				Attributes: me.AttributeValueMap{
 					"ReceivedFrameVlanTaggingOperationTable": sliceEvtocdRule,
 				},
@@ -2213,12 +2391,13 @@
 			// there is only one service vlan (oFsm.acceptIncrementalEvtoOption is false in this case).
 			// Interstingly this problem has not been observed in multi-tcont (or multi-service) scenario (in
 			// which case the oFsm.acceptIncrementalEvtoOption is set to true).
-			if oFsm.configuredUniFlow == 0 && !oFsm.acceptIncrementalEvtoOption {
+			if oFsm.configuredUniFlow == 1 && !oFsm.acceptIncrementalEvtoOption {
+				oFsm.mutexFlowParams.RUnlock()
 				logger.Debugw(ctx, "UniVlanConfigFsm Tx Remove::EVTOCD", log.Fields{"device-id": oFsm.deviceID})
 				// When there are no more EVTOCD vlan configurations on the ONU and acceptIncrementalEvtoOption
 				// is not enabled, delete the EVTOCD ME.
 				meParams := me.ParamData{
-					EntityID: oFsm.evtocdID,
+					EntityID: evtocdID,
 				}
 				oFsm.mutexPLastTxMeInstance.Lock()
 				meInstance, err := oFsm.pOmciCC.sendDeleteEvtocd(log.WithSpanFromContext(context.TODO(), ctx),
@@ -2249,6 +2428,7 @@
 				// This is true for only ATT/DT workflow
 				logger.Debugw(ctx, "UniVlanConfigFsm: Remove EVTOCD set operation",
 					log.Fields{"configured-flow": oFsm.configuredUniFlow, "incremental-evto": oFsm.acceptIncrementalEvtoOption})
+				oFsm.mutexFlowParams.RUnlock()
 				//not transparent and not acceptIncrementalEvtoOption: untagged/priotagged->singletagged
 				{ // just for local var's
 					// this defines stacking scenario: untagged->singletagged
@@ -2283,7 +2463,7 @@
 							cDontCareTpid<<cTreatTpidOffset) // copy TPID and DEI
 
 					meParams := me.ParamData{
-						EntityID: oFsm.evtocdID,
+						EntityID: evtocdID,
 						Attributes: me.AttributeValueMap{
 							"ReceivedFrameVlanTaggingOperationTable": sliceEvtocdRule,
 						},
@@ -2332,7 +2512,7 @@
 					binary.BigEndian.PutUint32(sliceEvtocdRule[cTreatInnerOffset:], 0xFFFFFFFF)
 
 					meParams := me.ParamData{
-						EntityID: oFsm.evtocdID,
+						EntityID: evtocdID,
 						Attributes: me.AttributeValueMap{
 							"ReceivedFrameVlanTaggingOperationTable": sliceEvtocdRule,
 						},
@@ -2365,7 +2545,6 @@
 			} //just for local var's
 		}
 	}
-
 	// if Config has been done for all EVTOCD entries let the FSM proceed
 	logger.Debugw(ctx, "EVTOCD filter remove loop finished", log.Fields{"device-id": oFsm.deviceID})
 	_ = oFsm.pAdaptFsm.pFsm.Event(vlanEvRemFlowDone, aRuleParams.TpID)