[VOL-4126] Fixing race condition between vlanConfig FSM and reconcilingFLows flag during reconcile of the flows upon adapter restart

Change-Id: Icf9f4f3f715a7ac95aec95d71fcea6cc2bec486b
diff --git a/internal/pkg/onuadaptercore/device_handler.go b/internal/pkg/onuadaptercore/device_handler.go
index 73a3d48..df3c477 100644
--- a/internal/pkg/onuadaptercore/device_handler.go
+++ b/internal/pkg/onuadaptercore/device_handler.go
@@ -934,16 +934,22 @@
 			return
 		}
 		flowsFound = true
+		lastFlowToReconcile := false
 		flowsProcessed := 0
 		dh.setReconcilingFlows(true)
 		for _, flowData := range uniData.PersFlowParams {
 			logger.Debugw(ctx, "reconciling - add flow with cookie slice", log.Fields{"device-id": dh.deviceID, "cookies": flowData.CookieSlice})
+			// If this is the last flow for the device we need to announce it the waiting
+			// chReconcilingFlowsFinished channel
+			if flowsProcessed == len(uniData.PersFlowParams)-1 {
+				lastFlowToReconcile = true
+			}
 			//the slice can be passed 'by value' here, - which internally passes its reference copy
 			dh.lockVlanConfig.RLock()
 			if _, exist = dh.UniVlanConfigFsmMap[uniData.PersUniID]; exist {
 				if err := dh.UniVlanConfigFsmMap[uniData.PersUniID].SetUniFlowParams(ctx, flowData.VlanRuleParams.TpID,
 					flowData.CookieSlice, uint16(flowData.VlanRuleParams.MatchVid), uint16(flowData.VlanRuleParams.SetVid),
-					uint8(flowData.VlanRuleParams.SetPcp)); err != nil {
+					uint8(flowData.VlanRuleParams.SetPcp), lastFlowToReconcile); err != nil {
 					logger.Errorw(ctx, err.Error(), log.Fields{"device-id": dh.deviceID})
 				}
 				dh.lockVlanConfig.RUnlock()
@@ -951,7 +957,7 @@
 				dh.lockVlanConfig.RUnlock()
 				if err := dh.createVlanFilterFsm(ctx, uniPort, flowData.VlanRuleParams.TpID, flowData.CookieSlice,
 					uint16(flowData.VlanRuleParams.MatchVid), uint16(flowData.VlanRuleParams.SetVid),
-					uint8(flowData.VlanRuleParams.SetPcp), OmciVlanFilterAddDone); err != nil {
+					uint8(flowData.VlanRuleParams.SetPcp), OmciVlanFilterAddDone, lastFlowToReconcile); err != nil {
 					logger.Errorw(ctx, err.Error(), log.Fields{"device-id": dh.deviceID})
 				}
 			}
@@ -960,7 +966,10 @@
 		logger.Debugw(ctx, "reconciling - flows processed", log.Fields{"device-id": dh.deviceID, "flowsProcessed": flowsProcessed,
 			"numUniFlows":       dh.UniVlanConfigFsmMap[uniData.PersUniID].numUniFlows,
 			"configuredUniFlow": dh.UniVlanConfigFsmMap[uniData.PersUniID].configuredUniFlow})
-		dh.setReconcilingFlows(false)
+		// this can't be used as global finished reconciling flag because
+		// assumes is getting called before the state machines for the last flow is completed,
+		// while this is not guaranteed.
+		//dh.setReconcilingFlows(false)
 	}
 	if !flowsFound {
 		logger.Debugw(ctx, "reconciling - no flows have been stored before adapter restart - terminate reconcilement",
@@ -2749,13 +2758,13 @@
 	logger.Debugw(ctx, "flow-add got lock", log.Fields{"device-id": dh.deviceID})
 	if _, exist := dh.UniVlanConfigFsmMap[apUniPort.uniID]; exist {
 		err := dh.UniVlanConfigFsmMap[apUniPort.uniID].SetUniFlowParams(ctx, loTpID, loCookieSlice,
-			loMatchVlan, loSetVlan, loSetPcp)
+			loMatchVlan, loSetVlan, loSetPcp, false)
 		dh.lockVlanConfig.RUnlock()
 		return err
 	}
 	dh.lockVlanConfig.RUnlock()
 	return dh.createVlanFilterFsm(ctx, apUniPort, loTpID, loCookieSlice,
-		loMatchVlan, loSetVlan, loSetPcp, OmciVlanFilterAddDone)
+		loMatchVlan, loSetVlan, loSetPcp, OmciVlanFilterAddDone, false)
 }
 
 //removeFlowItemFromUniPort parses the actual flow item to remove it from the UniPort
@@ -2805,7 +2814,7 @@
 // createVlanFilterFsm initializes and runs the VlanFilter FSM to transfer OMCI related VLAN config
 // if this function is called from possibly concurrent processes it must be mutex-protected from the caller!
 func (dh *deviceHandler) createVlanFilterFsm(ctx context.Context, apUniPort *onuUniPort, aTpID uint8, aCookieSlice []uint64,
-	aMatchVlan uint16, aSetVlan uint16, aSetPcp uint8, aDevEvent OnuDeviceEvent) error {
+	aMatchVlan uint16, aSetVlan uint16, aSetPcp uint8, aDevEvent OnuDeviceEvent, lastFlowToReconcile bool) error {
 	chVlanFilterFsm := make(chan Message, 2048)
 
 	pDevEntry := dh.getOnuDeviceEntry(ctx, true)
@@ -2816,7 +2825,7 @@
 
 	pVlanFilterFsm := NewUniVlanConfigFsm(ctx, dh, pDevEntry.PDevOmciCC, apUniPort, dh.pOnuTP,
 		pDevEntry.pOnuDB, aTpID, aDevEvent, "UniVlanConfigFsm", chVlanFilterFsm,
-		dh.pOpenOnuAc.AcceptIncrementalEvto, aCookieSlice, aMatchVlan, aSetVlan, aSetPcp)
+		dh.pOpenOnuAc.AcceptIncrementalEvto, aCookieSlice, aMatchVlan, aSetVlan, aSetPcp, lastFlowToReconcile)
 	if pVlanFilterFsm != nil {
 		dh.lockVlanConfig.Lock()
 		//ensure the mutex is locked throughout the state transition to 'starting' to prevent unintended (ignored) events to be sent there
diff --git a/internal/pkg/onuadaptercore/omci_vlan_config.go b/internal/pkg/onuadaptercore/omci_vlan_config.go
index 7e06cae..028245d 100644
--- a/internal/pkg/onuadaptercore/omci_vlan_config.go
+++ b/internal/pkg/onuadaptercore/omci_vlan_config.go
@@ -179,6 +179,9 @@
 	flowDeleteChannel           chan<- bool
 	//cookie value that indicates that a rule to add is delayed by waiting for deletion of some other existing rule with the same cookie
 	delayNewRuleCookie uint64
+	// Used to indicate if the FSM is for a reconciling flow and if it's the last flow to be reconciled
+	// thus notification needs to be sent on chan.
+	lastFlowToReconcile bool
 }
 
 //NewUniVlanConfigFsm is the 'constructor' for the state machine to config the PON ANI ports
@@ -186,7 +189,7 @@
 func NewUniVlanConfigFsm(ctx context.Context, apDeviceHandler *deviceHandler, apDevOmciCC *omciCC, apUniPort *onuUniPort,
 	apUniTechProf *onuUniTechProf, apOnuDB *onuDeviceDB, aTechProfileID uint8,
 	aRequestEvent OnuDeviceEvent, aName string, aCommChannel chan Message, aAcceptIncrementalEvto bool,
-	aCookieSlice []uint64, aMatchVlan uint16, aSetVlan uint16, aSetPcp uint8) *UniVlanConfigFsm {
+	aCookieSlice []uint64, aMatchVlan uint16, aSetVlan uint16, aSetPcp uint8, lastFlowToRec bool) *UniVlanConfigFsm {
 	instFsm := &UniVlanConfigFsm{
 		pDeviceHandler:              apDeviceHandler,
 		deviceID:                    apDeviceHandler.deviceID,
@@ -200,6 +203,7 @@
 		configuredUniFlow:           0,
 		numRemoveFlows:              0,
 		clearPersistency:            true,
+		lastFlowToReconcile:         lastFlowToRec,
 	}
 
 	instFsm.pAdaptFsm = NewAdapterFsm(aName, instFsm.deviceID, aCommChannel)
@@ -380,7 +384,7 @@
 // ignore complexity by now
 // nolint: gocyclo
 func (oFsm *UniVlanConfigFsm) SetUniFlowParams(ctx context.Context, aTpID uint8, aCookieSlice []uint64,
-	aMatchVlan uint16, aSetVlan uint16, aSetPcp uint8) error {
+	aMatchVlan uint16, aSetVlan uint16, aSetPcp uint8, lastFlowToReconcile bool) error {
 	loRuleParams := uniVlanRuleParams{
 		TpID:     aTpID,
 		MatchVid: uint32(aMatchVlan),
@@ -418,6 +422,7 @@
 	flowEntryMatch := false
 	flowCookieModify := false
 	requestAppendRule := false
+	oFsm.lastFlowToReconcile = lastFlowToReconcile
 	//mutex protection is required for possible concurrent access to FSM members
 	oFsm.mutexFlowParams.Lock()
 	for flow, storedUniFlowParams := range oFsm.uniVlanFlowParamsSlice {
@@ -1049,8 +1054,9 @@
 	}
 	if oFsm.pDeviceHandler.isSkipOnuConfigReconciling() {
 		oFsm.configuredUniFlow = oFsm.numUniFlows
-		if !oFsm.pDeviceHandler.isReconcilingFlows() {
+		if oFsm.lastFlowToReconcile {
 			logger.Debugw(ctx, "reconciling - flow processing finished", log.Fields{"device-id": oFsm.deviceID})
+			oFsm.pDeviceHandler.setReconcilingFlows(false)
 			oFsm.pDeviceHandler.chReconcilingFlowsFinished <- true
 		}
 		logger.Debugw(ctx, "reconciling - skip enterVlanConfigDone processing",