[VOL-3959] Reconciling on MDS match: Replace timer approach by IPC

Change-Id: I550b212288e6853b6fd1189cf1977dd43a230af0
diff --git a/VERSION b/VERSION
index e67ee18..853d583 100755
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-1.3.0-dev191
+1.3.0-dev192
diff --git a/internal/pkg/onuadaptercore/device_handler.go b/internal/pkg/onuadaptercore/device_handler.go
index b9b8504..2d4d67d 100644
--- a/internal/pkg/onuadaptercore/device_handler.go
+++ b/internal/pkg/onuadaptercore/device_handler.go
@@ -216,6 +216,9 @@
 	reconciling                 uint8
 	mutexReconcilingFlag        sync.RWMutex
 	chReconcilingFinished       chan bool //channel to indicate that reconciling has been finished
+	reconcilingFlows            bool
+	mutexReconcilingFlowsFlag   sync.RWMutex
+	chReconcilingFlowsFinished  chan bool //channel to indicate that reconciling of flows has been finished
 	ReadyForSpecificOmciConfig  bool
 	deletionInProgress          bool
 	mutexDeletionInProgressFlag sync.RWMutex
@@ -249,6 +252,8 @@
 	dh.UniVlanConfigFsmMap = make(map[uint8]*UniVlanConfigFsm)
 	dh.reconciling = cNoReconciling
 	dh.chReconcilingFinished = make(chan bool)
+	dh.reconcilingFlows = false
+	dh.chReconcilingFlowsFinished = make(chan bool)
 	dh.ReadyForSpecificOmciConfig = false
 	dh.deletionInProgress = false
 
@@ -926,6 +931,7 @@
 		}
 		flowsFound = true
 		flowsProcessed := 0
+		dh.setReconcilingFlows(true)
 		for _, flowData := range uniData.PersFlowParams {
 			logger.Debugw(ctx, "reconciling - add flow with cookie slice", log.Fields{"device-id": dh.deviceID, "cookies": flowData.CookieSlice})
 			//the slice can be passed 'by value' here, - which internally passes its reference copy
@@ -950,6 +956,7 @@
 		logger.Debugw(ctx, "reconciling - flows processed", log.Fields{"device-id": dh.deviceID, "flowsProcessed": flowsProcessed,
 			"numUniFlows":       dh.UniVlanConfigFsmMap[uniData.PersUniID].numUniFlows,
 			"configuredUniFlow": dh.UniVlanConfigFsmMap[uniData.PersUniID].configuredUniFlow})
+		dh.setReconcilingFlows(false)
 	}
 	if !flowsFound {
 		logger.Debugw(ctx, "reconciling - no flows have been stored before adapter restart - terminate reconcilement",
@@ -1663,12 +1670,7 @@
 		return fmt.Errorf("no valid OnuDevice: %s", dh.deviceID)
 	}
 	if includingMibSyncFsm {
-		//the MibSync FSM might be active all the ONU-active time,
-		// hence it must be stopped unconditionally
-		pMibUlFsm := pDevEntry.pMibUploadFsm.pFsm
-		if pMibUlFsm != nil {
-			_ = pMibUlFsm.Event(ulEvStop) //TODO!! verify if MibSyncFsm stop-processing is sufficient (to allow it again afterwards)
-		}
+		pDevEntry.CancelProcessing(ctx)
 	}
 	//MibDownload may run
 	pMibDlFsm := pDevEntry.pMibDownloadFsm.pFsm
@@ -3220,9 +3222,14 @@
 			logger.Debugw(ctx, "wait for channel signal or timeout",
 				log.Fields{"timeout": dh.pOpenOnuAc.maxTimeoutReconciling, "device-id": dh.deviceID})
 			select {
-			case <-dh.chReconcilingFinished:
-				logger.Debugw(ctx, "reconciling has been finished in time",
-					log.Fields{"device-id": dh.deviceID})
+			case success := <-dh.chReconcilingFinished:
+				if success {
+					logger.Debugw(ctx, "reconciling has been finished in time",
+						log.Fields{"device-id": dh.deviceID})
+				} else {
+					logger.Debugw(ctx, "wait for reconciling aborted",
+						log.Fields{"device-id": dh.deviceID})
+				}
 			case <-time.After(dh.pOpenOnuAc.maxTimeoutReconciling):
 				logger.Errorw(ctx, "timeout waiting for reconciling to be finished!",
 					log.Fields{"device-id": dh.deviceID})
@@ -3278,3 +3285,16 @@
 func (dh *deviceHandler) getDeviceReasonString() string {
 	return deviceReasonMap[dh.getDeviceReason()]
 }
+
+func (dh *deviceHandler) setReconcilingFlows(value bool) {
+	dh.mutexReconcilingFlowsFlag.Lock()
+	dh.reconcilingFlows = value
+	dh.mutexReconcilingFlowsFlag.Unlock()
+}
+
+func (dh *deviceHandler) isReconcilingFlows() bool {
+	dh.mutexReconcilingFlowsFlag.RLock()
+	value := dh.reconcilingFlows
+	dh.mutexReconcilingFlowsFlag.RUnlock()
+	return value
+}
diff --git a/internal/pkg/onuadaptercore/mib_sync.go b/internal/pkg/onuadaptercore/mib_sync.go
index 8aabeb7..15868fc 100644
--- a/internal/pkg/onuadaptercore/mib_sync.go
+++ b/internal/pkg/onuadaptercore/mib_sync.go
@@ -244,16 +244,28 @@
 			oo.baseDeviceHandler.enableUniPortStateUpdate(ctx)
 		}
 		go func() {
-			// Stopping reconcilement has to be delayed as in multi-ONU/multi-flow environment
-			// the parallel processing to rebuild the adapter internal flow data could still be
-			// running here. It will take only a few milliseconds until the corresponding threads
-			// will be finished as no OMCI-config is done in this use case.
-			// TODO: The timer approach should be replaced by a more sophisticated solution using
-			// a real interaction between this routine and the threads configuring the flow data
-			// after imminent release VOLTHA v2.7
-			time.Sleep(100 * time.Millisecond)
+			// In multi-ONU/multi-flow environment stopping reconcilement has to be delayed until
+			// we get a signal that the processing of the last step to rebuild the adapter internal
+			// flow data is finished.
+			select {
+			case success := <-oo.baseDeviceHandler.chReconcilingFlowsFinished:
+				if success {
+					logger.Debugw(ctx, "reconciling flows has been finished in time",
+						log.Fields{"device-id": oo.deviceID})
+					_ = oo.pMibUploadFsm.pFsm.Event(ulEvSuccess)
+				} else {
+					logger.Debugw(ctx, "wait for reconciling flows aborted",
+						log.Fields{"device-id": oo.deviceID})
+					oo.baseDeviceHandler.setReconcilingFlows(false)
+					return
+				}
+			case <-time.After(100 * time.Millisecond):
+				logger.Errorw(ctx, "timeout waiting for reconciling flows to be finished!",
+					log.Fields{"device-id": oo.deviceID})
+				oo.baseDeviceHandler.setReconcilingFlows(false)
+				_ = oo.pMibUploadFsm.pFsm.Event(ulEvMismatch)
+			}
 			oo.baseDeviceHandler.stopReconciling(ctx)
-			_ = oo.pMibUploadFsm.pFsm.Event(ulEvSuccess)
 		}()
 
 	} else {
@@ -904,3 +916,20 @@
 	}
 	return restoredFromMibTemplate
 }
+
+//CancelProcessing terminates potentially running reconciling processes and stops the FSM
+func (oo *OnuDeviceEntry) CancelProcessing(ctx context.Context) {
+
+	if oo.baseDeviceHandler.isReconcilingFlows() {
+		oo.baseDeviceHandler.chReconcilingFlowsFinished <- false
+	}
+	if oo.baseDeviceHandler.isReconciling() {
+		oo.baseDeviceHandler.chReconcilingFinished <- false
+	}
+	//the MibSync FSM might be active all the ONU-active time,
+	// hence it must be stopped unconditionally
+	pMibUlFsm := oo.pMibUploadFsm.pFsm
+	if pMibUlFsm != nil {
+		_ = pMibUlFsm.Event(ulEvStop)
+	}
+}
diff --git a/internal/pkg/onuadaptercore/omci_vlan_config.go b/internal/pkg/onuadaptercore/omci_vlan_config.go
index 9af8e23..45e0944 100644
--- a/internal/pkg/onuadaptercore/omci_vlan_config.go
+++ b/internal/pkg/onuadaptercore/omci_vlan_config.go
@@ -341,12 +341,9 @@
 	// in any case (even if it might be automatically requested by above cancellation of waiting) ensure resetting the FSM
 	pAdaptFsm := oFsm.pAdaptFsm
 	if pAdaptFsm != nil {
-		// obviously calling some FSM event here directly does not work - so trying to decouple it ...
-		go func(aPAFsm *AdapterFsm) {
-			if aPAFsm.pFsm != nil {
-				_ = oFsm.pAdaptFsm.pFsm.Event(vlanEvReset)
-			}
-		}(pAdaptFsm)
+		if pAdaptFsm.pFsm != nil {
+			_ = pAdaptFsm.pFsm.Event(vlanEvReset)
+		}
 	}
 }
 
@@ -485,9 +482,7 @@
 					log.Fields{"fsmState": oFsm.pAdaptFsm.pFsm.Current(), "device-id": oFsm.deviceID})
 				oFsm.mutexFlowParams.Unlock()
 				if pConfigVlanStateBaseFsm.Is(vlanStConfigDone) {
-					go func(a_pBaseFsm *fsm.FSM) {
-						_ = a_pBaseFsm.Event(vlanEvSkipOmciConfig)
-					}(pConfigVlanStateBaseFsm)
+					_ = pConfigVlanStateBaseFsm.Event(vlanEvSkipOmciConfig)
 				}
 				return nil
 			}
@@ -501,7 +496,6 @@
 				if oFsm.configuredUniFlow == 0 {
 					// this is a restart with a complete new flow, we can re-use the initial flow config control
 					// including the check, if the related techProfile is (still) available (probably also removed in between)
-					// Can't call FSM Event directly, decoupling it
 					go func(a_pBaseFsm *fsm.FSM) {
 						_ = a_pBaseFsm.Event(vlanEvRenew)
 					}(pConfigVlanStateBaseFsm)
@@ -516,7 +510,6 @@
 					logger.Debugw(ctx, "UniVlanConfigFsm - incremental config request (on setConfig)", log.Fields{
 						"device-id": oFsm.deviceID, "uni-id": oFsm.pOnuUniPort.uniID,
 						"set-Vlan": oFsm.actualUniVlanConfigRule.SetVid, "tp-id": tpID, "ProfDone": loTechProfDone})
-
 					go func(aPBaseFsm *fsm.FSM, aTechProfDone bool) {
 						if aTechProfDone {
 							// let the vlan processing continue with next rule
@@ -1003,6 +996,10 @@
 	}
 	if oFsm.pDeviceHandler.isSkipOnuConfigReconciling() {
 		oFsm.configuredUniFlow = oFsm.numUniFlows
+		if !oFsm.pDeviceHandler.isReconcilingFlows() {
+			logger.Debugw(ctx, "reconciling - flow processing finished", log.Fields{"device-id": oFsm.deviceID})
+			oFsm.pDeviceHandler.chReconcilingFlowsFinished <- true
+		}
 		logger.Debugw(ctx, "reconciling - skip enterVlanConfigDone processing",
 			log.Fields{"numUniFlows": oFsm.numUniFlows, "configuredUniFlow": oFsm.configuredUniFlow, "device-id": oFsm.deviceID})
 		return