[VOL-3748] Support periodical audit of mib data sync (MDS) counter

Change-Id: Ifb61d98dca1285de70ac8fdcda4bb673374c6e07
diff --git a/VERSION b/VERSION
index e03597d..7c48a4c 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-1.2.1-dev160
+1.2.1-dev161
diff --git a/internal/pkg/onuadaptercore/device_handler.go b/internal/pkg/onuadaptercore/device_handler.go
index f690e38..c014ae4 100644
--- a/internal/pkg/onuadaptercore/device_handler.go
+++ b/internal/pkg/onuadaptercore/device_handler.go
@@ -92,6 +92,31 @@
 	cOnuActivatedEvent = "ONU_ACTIVATED"
 )
 
+type usedOmciConfigFsms int
+
+const (
+	cUploadFsm usedOmciConfigFsms = iota
+	cDownloadFsm
+	cUniLockFsm
+	cUniUnLockFsm
+	cAniConfigFsm
+	cUniVlanConfigFsm
+)
+
+type idleCheckStruct struct {
+	idleCheckFunc func(*deviceHandler, context.Context, string) bool
+	idleState     string
+}
+
+var fsmIdleStateFuncMap = map[usedOmciConfigFsms]idleCheckStruct{
+	cUploadFsm:        {(*deviceHandler).mibUploadFsmInIdleState, cMibUlFsmIdleState},
+	cDownloadFsm:      {(*deviceHandler).mibDownloadFsmInIdleState, cMibDlFsmIdleState},
+	cUniLockFsm:       {(*deviceHandler).devUniLockFsmInIdleState, cUniFsmIdleState},
+	cUniUnLockFsm:     {(*deviceHandler).devUniUnlockFsmInIdleState, cUniFsmIdleState},
+	cAniConfigFsm:     {(*deviceHandler).devAniConfigFsmInIdleState, cAniFsmIdleState},
+	cUniVlanConfigFsm: {(*deviceHandler).devUniVlanConfigFsmInIdleState, cVlanFsmIdleState},
+}
+
 const (
 	// device reasons
 	drUnset                            = 0
@@ -164,6 +189,8 @@
 	//discOnus sync.Map
 	//onus     sync.Map
 	//portStats          *OpenOltStatisticsMgr
+	collectorIsRunning         bool
+	mutexCollectorFlag         sync.RWMutex
 	stopCollector              chan bool
 	stopHeartbeatCheck         chan bool
 	uniEntityMap               map[uint32]*onuUniPort
@@ -188,6 +215,7 @@
 	dh.exitChannel = make(chan int, 1)
 	dh.lockDevice = sync.RWMutex{}
 	dh.deviceEntrySet = make(chan bool, 1)
+	dh.collectorIsRunning = false
 	dh.stopCollector = make(chan bool, 2)
 	dh.stopHeartbeatCheck = make(chan bool, 2)
 	//dh.metrics = pmmetrics.NewPmMetrics(cloned.Id, pmmetrics.Frequency(150), pmmetrics.FrequencyOverride(false), pmmetrics.Grouped(false), pmmetrics.Metrics(pmNames))
@@ -1423,9 +1451,10 @@
 		return fmt.Errorf("can't execute MibSync: %s", dh.deviceID)
 	}
 
-	// Start PM collector routine
-	go dh.startCollector(ctx)
-
+	if !dh.getCollectorIsRunning() {
+		// Start PM collector routine
+		go dh.startCollector(ctx)
+	}
 	return nil
 }
 
@@ -1438,7 +1467,7 @@
 		//stop all running FSM processing - make use of the DH-state as mirrored in the deviceReason
 		//here no conflict with aborted FSM's should arise as a complete OMCI initialization is assumed on ONU-Up
 		//but that might change with some simple MDS check on ONU-Up treatment -> attention!!!
-		if err := dh.resetFsms(ctx); err != nil {
+		if err := dh.resetFsms(ctx, true); err != nil {
 			logger.Errorw(ctx, "error-updateInterface at FSM stop",
 				log.Fields{"device-id": dh.deviceID, "error": err})
 			// abort: system behavior is just unstable ...
@@ -1491,7 +1520,7 @@
 	return nil
 }
 
-func (dh *deviceHandler) resetFsms(ctx context.Context) error {
+func (dh *deviceHandler) resetFsms(ctx context.Context, includingMibSyncFsm bool) error {
 	//all possible FSM's are stopped or reset here to ensure their transition to 'disabled'
 	//it is not sufficient to stop/reset the latest running FSM as done in previous versions
 	//  as after down/up procedures all FSM's might be active/ongoing (in theory)
@@ -1502,11 +1531,13 @@
 		logger.Errorw(ctx, "No valid OnuDevice -aborting", log.Fields{"device-id": dh.deviceID})
 		return fmt.Errorf("no valid OnuDevice: %s", dh.deviceID)
 	}
-	//the MibSync FSM might be active all the ONU-active time,
-	// hence it must be stopped unconditionally
-	pMibUlFsm := pDevEntry.pMibUploadFsm.pFsm
-	if pMibUlFsm != nil {
-		_ = pMibUlFsm.Event(ulEvStop) //TODO!! verify if MibSyncFsm stop-processing is sufficient (to allow it again afterwards)
+	if includingMibSyncFsm {
+		//the MibSync FSM might be active all the ONU-active time,
+		// hence it must be stopped unconditionally
+		pMibUlFsm := pDevEntry.pMibUploadFsm.pFsm
+		if pMibUlFsm != nil {
+			_ = pMibUlFsm.Event(ulEvStop) //TODO!! verify if MibSyncFsm stop-processing is sufficient (to allow it again afterwards)
+		}
 	}
 	//MibDownload may run
 	pMibDlFsm := pDevEntry.pMibDownloadFsm.pFsm
@@ -1544,10 +1575,10 @@
 			}
 		}
 	}
-
-	// Stop collector routine
-	dh.stopCollector <- true
-
+	if dh.getCollectorIsRunning() {
+		// Stop collector routine
+		dh.stopCollector <- true
+	}
 	return nil
 }
 
@@ -1798,8 +1829,6 @@
 			// which may be the case from some previous actvity on another UNI Port of the ONU
 			// or even some previous flow add activity on the same port
 			_ = dh.deviceReasonUpdate(ctx, drOmciFlowsPushed, !dh.reconciling)
-			// request MDS-value for test and logging purposes
-			dh.pOnuOmciDevice.requestMdsValue(ctx)
 			if dh.reconciling {
 				go dh.reconcileMetrics(ctx)
 			}
@@ -1810,6 +1839,10 @@
 			_ = dh.deviceReasonUpdate(ctx, drOmciFlowsDeleted, true)
 		}
 	}
+	if err := dh.storePersistentData(ctx); err != nil {
+		logger.Warnw(ctx, "store persistent data error - continue for now as there will be additional write attempts",
+			log.Fields{"device-id": dh.deviceID, "err": err})
+	}
 }
 
 //deviceProcStatusUpdate evaluates possible processing events and initiates according next activities
@@ -2646,9 +2679,11 @@
 	// Normally done when the onu_metrics_manager is initialized the first time, but needed again later when ONU is
 	// reset like onu rebooted.
 	dh.pOnuMetricsMgr.initializeMetricCollectionTime(ctx)
+	dh.setCollectorIsRunning(true)
 	for {
 		select {
 		case <-dh.stopCollector:
+			dh.setCollectorIsRunning(false)
 			logger.Debugw(ctx, "stopping-collector-for-onu", log.Fields{"device-id": dh.device.Id})
 			dh.pOnuMetricsMgr.stopProcessingOmciResponses <- true // Stop the OMCI GET response processing routine
 			return
@@ -2708,3 +2743,95 @@
 	portStatus := NewUniPortStatus(dh.pOnuOmciDevice.PDevOmciCC)
 	return portStatus.getUniPortStatus(ctx, uniInfo.UniIndex)
 }
+
+func (dh *deviceHandler) isFsmInState(ctx context.Context, pFsm *fsm.FSM, wantedState string) bool {
+	var currentState string
+	if pFsm != nil {
+		currentState = pFsm.Current()
+		if currentState == wantedState {
+			return true
+		}
+	} else {
+		logger.Warnw(ctx, "FSM not defined!", log.Fields{"wantedState": wantedState, "device-id": dh.deviceID})
+	}
+	return false
+}
+
+func (dh *deviceHandler) mibUploadFsmInIdleState(ctx context.Context, idleState string) bool {
+	return dh.isFsmInState(ctx, dh.pOnuOmciDevice.pMibUploadFsm.pFsm, idleState)
+}
+
+func (dh *deviceHandler) mibDownloadFsmInIdleState(ctx context.Context, idleState string) bool {
+	return dh.isFsmInState(ctx, dh.pOnuOmciDevice.pMibDownloadFsm.pFsm, idleState)
+}
+
+func (dh *deviceHandler) devUniLockFsmInIdleState(ctx context.Context, idleState string) bool {
+	return dh.isFsmInState(ctx, dh.pLockStateFsm.pAdaptFsm.pFsm, idleState)
+}
+
+func (dh *deviceHandler) devUniUnlockFsmInIdleState(ctx context.Context, idleState string) bool {
+	return dh.isFsmInState(ctx, dh.pUnlockStateFsm.pAdaptFsm.pFsm, idleState)
+}
+
+func (dh *deviceHandler) devAniConfigFsmInIdleState(ctx context.Context, idleState string) bool {
+	if dh.pOnuTP.pAniConfigFsm != nil {
+		for _, v := range dh.pOnuTP.pAniConfigFsm {
+			if !dh.isFsmInState(ctx, v.pAdaptFsm.pFsm, idleState) {
+				return false
+			}
+		}
+		return true
+	}
+	logger.Warnw(ctx, "AniConfig FSM not defined!", log.Fields{"device-id": dh.deviceID})
+	return false
+}
+
+func (dh *deviceHandler) devUniVlanConfigFsmInIdleState(ctx context.Context, idleState string) bool {
+	if dh.UniVlanConfigFsmMap != nil {
+		for _, v := range dh.UniVlanConfigFsmMap {
+			if !dh.isFsmInState(ctx, v.pAdaptFsm.pFsm, idleState) {
+				return false
+			}
+		}
+		return true
+	}
+	logger.Warnw(ctx, "UniVlanConfig FSM not defined!", log.Fields{"device-id": dh.deviceID})
+	return false
+}
+
+func (dh *deviceHandler) allButCallingFsmInIdleState(ctx context.Context, callingFsm usedOmciConfigFsms) bool {
+	for fsmName, fsmStruct := range fsmIdleStateFuncMap {
+		if fsmName != callingFsm && !fsmStruct.idleCheckFunc(dh, ctx, fsmStruct.idleState) {
+			return false
+		}
+	}
+	return true
+}
+
+func (dh *deviceHandler) prepareReconcilingWithActiveAdapter(ctx context.Context) {
+	logger.Debugw(ctx, "prepare to reconcile the ONU with adapter using persistency data", log.Fields{"device-id": dh.device.Id})
+	if err := dh.resetFsms(ctx, false); err != nil {
+		logger.Errorw(ctx, "reset of FSMs failed!", log.Fields{"device-id": dh.deviceID, "error": err})
+		// TODO: fatal error reset ONU, delete deviceHandler!
+		return
+	}
+	if !dh.getCollectorIsRunning() {
+		// Start PM collector routine
+		go dh.startCollector(ctx)
+	}
+	dh.uniEntityMap = make(map[uint32]*onuUniPort)
+	dh.reconciling = true
+}
+
+func (dh *deviceHandler) setCollectorIsRunning(flagValue bool) {
+	dh.mutexCollectorFlag.Lock()
+	dh.collectorIsRunning = flagValue
+	dh.mutexCollectorFlag.Unlock()
+}
+
+func (dh *deviceHandler) getCollectorIsRunning() bool {
+	dh.mutexCollectorFlag.RLock()
+	flagValue := dh.collectorIsRunning
+	dh.mutexCollectorFlag.RUnlock()
+	return flagValue
+}
diff --git a/internal/pkg/onuadaptercore/mib_sync.go b/internal/pkg/onuadaptercore/mib_sync.go
index 5c4ba92..9e1a6a5 100644
--- a/internal/pkg/onuadaptercore/mib_sync.go
+++ b/internal/pkg/onuadaptercore/mib_sync.go
@@ -74,9 +74,12 @@
 func (oo *OnuDeviceEntry) enterResettingMibState(ctx context.Context, e *fsm.Event) {
 	logger.Debugw(ctx, "MibSync FSM", log.Fields{"Start MibTemplate processing in State": e.FSM.Current(), "device-id": oo.deviceID})
 
+	if !oo.isNewOnu() {
+		oo.baseDeviceHandler.prepareReconcilingWithActiveAdapter(ctx)
+		oo.devState = DeviceStatusInit
+	}
 	logger.Debugw(ctx, "MibSync FSM", log.Fields{"send mibReset in State": e.FSM.Current(), "device-id": oo.deviceID})
 	_ = oo.PDevOmciCC.sendMibReset(log.WithSpanFromContext(context.TODO(), ctx), ConstDefaultOmciTimeout, true)
-
 	//TODO: needs to handle timeouts
 }
 
@@ -148,29 +151,29 @@
 			mibTmpBytes := []byte(mibTmpString)
 			logger.Debugf(ctx, "MibSync FSM - MibTemplate tokens swapped out: %s", mibTmpBytes)
 
-			var fistLevelMap map[string]interface{}
-			if err = json.Unmarshal(mibTmpBytes, &fistLevelMap); err != nil {
+			var firstLevelMap map[string]interface{}
+			if err = json.Unmarshal(mibTmpBytes, &firstLevelMap); err != nil {
 				logger.Errorw(ctx, "MibSync FSM - Failed to unmarshal template", log.Fields{"error": err, "device-id": oo.deviceID})
 			} else {
-				for fistLevelKey, firstLevelValue := range fistLevelMap {
-					logger.Debugw(ctx, "MibSync FSM - fistLevelKey", log.Fields{"fistLevelKey": fistLevelKey})
-					if uint16ValidNumber, err := strconv.ParseUint(fistLevelKey, 10, 16); err == nil {
+				for firstLevelKey, firstLevelValue := range firstLevelMap {
+					//logger.Debugw(ctx, "MibSync FSM - firstLevelKey", log.Fields{"firstLevelKey": firstLevelKey})
+					if uint16ValidNumber, err := strconv.ParseUint(firstLevelKey, 10, 16); err == nil {
 						meClassID := me.ClassID(uint16ValidNumber)
-						logger.Debugw(ctx, "MibSync FSM - fistLevelKey is a number in uint16-range", log.Fields{"uint16ValidNumber": uint16ValidNumber})
+						//logger.Debugw(ctx, "MibSync FSM - firstLevelKey is a number in uint16-range", log.Fields{"uint16ValidNumber": uint16ValidNumber})
 						if isSupportedClassID(meClassID) {
-							logger.Debugw(ctx, "MibSync FSM - fistLevelKey is a supported classID", log.Fields{"meClassID": meClassID})
+							//logger.Debugw(ctx, "MibSync FSM - firstLevelKey is a supported classID", log.Fields{"meClassID": meClassID})
 							secondLevelMap := firstLevelValue.(map[string]interface{})
 							for secondLevelKey, secondLevelValue := range secondLevelMap {
-								logger.Debugw(ctx, "MibSync FSM - secondLevelKey", log.Fields{"secondLevelKey": secondLevelKey})
+								//logger.Debugw(ctx, "MibSync FSM - secondLevelKey", log.Fields{"secondLevelKey": secondLevelKey})
 								if uint16ValidNumber, err := strconv.ParseUint(secondLevelKey, 10, 16); err == nil {
 									meEntityID := uint16(uint16ValidNumber)
-									logger.Debugw(ctx, "MibSync FSM - secondLevelKey is a number and a valid EntityId", log.Fields{"meEntityID": meEntityID})
+									//logger.Debugw(ctx, "MibSync FSM - secondLevelKey is a number and a valid EntityId", log.Fields{"meEntityID": meEntityID})
 									thirdLevelMap := secondLevelValue.(map[string]interface{})
 									for thirdLevelKey, thirdLevelValue := range thirdLevelMap {
 										if thirdLevelKey == "Attributes" {
-											logger.Debugw(ctx, "MibSync FSM - thirdLevelKey refers to attributes", log.Fields{"thirdLevelKey": thirdLevelKey})
+											//logger.Debugw(ctx, "MibSync FSM - thirdLevelKey refers to attributes", log.Fields{"thirdLevelKey": thirdLevelKey})
 											attributesMap := thirdLevelValue.(map[string]interface{})
-											logger.Debugw(ctx, "MibSync FSM - attributesMap", log.Fields{"attributesMap": attributesMap})
+											//logger.Debugw(ctx, "MibSync FSM - attributesMap", log.Fields{"attributesMap": attributesMap})
 											oo.pOnuDB.PutMe(ctx, meClassID, meEntityID, attributesMap)
 											meStoredFromTemplate = true
 										}
@@ -221,10 +224,25 @@
 	_ = oo.PDevOmciCC.sendMibUpload(log.WithSpanFromContext(context.TODO(), ctx), ConstDefaultOmciTimeout, true)
 }
 
-func (oo *OnuDeviceEntry) enterInSyncState(ctx context.Context, e *fsm.Event) {
-	oo.mibLastDbSync = uint32(time.Now().Unix())
+func (oo *OnuDeviceEntry) enterUploadDoneState(ctx context.Context, e *fsm.Event) {
 	logger.Debugw(ctx, "MibSync FSM", log.Fields{"send notification to core in State": e.FSM.Current(), "device-id": oo.deviceID})
 	oo.transferSystemEvent(ctx, MibDatabaseSync)
+	go func() {
+		_ = oo.pMibUploadFsm.pFsm.Event(ulEvSuccess)
+	}()
+}
+
+func (oo *OnuDeviceEntry) enterInSyncState(ctx context.Context, e *fsm.Event) {
+	oo.sOnuPersistentData.PersMibLastDbSync = uint32(time.Now().Unix())
+	if oo.mibAuditDelay > 0 {
+		logger.Debugw(ctx, "MibSync FSM", log.Fields{"trigger next Audit in State": e.FSM.Current(), "oo.mibAuditDelay": oo.mibAuditDelay, "device-id": oo.deviceID})
+		go func() {
+			time.Sleep(time.Duration(oo.mibAuditDelay) * time.Second)
+			if err := oo.pMibUploadFsm.pFsm.Event(ulEvAuditMib); err != nil {
+				logger.Debugw(ctx, "MibSyncFsm: Can't go to state auditing", log.Fields{"device-id": oo.deviceID, "err": err})
+			}
+		}()
+	}
 }
 
 func (oo *OnuDeviceEntry) enterExaminingMdsState(ctx context.Context, e *fsm.Event) {
@@ -235,11 +253,35 @@
 func (oo *OnuDeviceEntry) enterResynchronizingState(ctx context.Context, e *fsm.Event) {
 	logger.Debugw(ctx, "MibSync FSM", log.Fields{"Start MibResync processing in State": e.FSM.Current(), "device-id": oo.deviceID})
 	logger.Debug(ctx, "function not implemented yet")
+	// TODOs:
+	// VOL-3805 - Provide exclusive OMCI channel for one FSM
+	// VOL-3785 - New event notifications and corresponding performance counters for openonu-adapter-go
+	// VOL-3792 - Support periodical audit via mib resync
+	// VOL-3793 - ONU-reconcile handling after adapter restart based on mib resync
 }
 
 func (oo *OnuDeviceEntry) enterAuditingState(ctx context.Context, e *fsm.Event) {
-	logger.Debugw(ctx, "MibSync FSM", log.Fields{"Start MibResync processing in State": e.FSM.Current(), "device-id": oo.deviceID})
-	logger.Debug(ctx, "function not implemented yet")
+	logger.Debugw(ctx, "MibSync FSM", log.Fields{"Start MibAudit processing in State": e.FSM.Current(), "device-id": oo.deviceID})
+	if oo.baseDeviceHandler.allButCallingFsmInIdleState(ctx, cUploadFsm) {
+		oo.requestMdsValue(ctx)
+	} else {
+		logger.Debugw(ctx, "MibSync FSM", log.Fields{"Configuration is ongoing - skip auditing!": e.FSM.Current(), "device-id": oo.deviceID})
+		go func() {
+			_ = oo.pMibUploadFsm.pFsm.Event(ulEvSuccess)
+		}()
+	}
+}
+
+func (oo *OnuDeviceEntry) enterReAuditingState(ctx context.Context, e *fsm.Event) {
+	logger.Debugw(ctx, "MibSync FSM", log.Fields{"Start retest MdsValue processing in State": e.FSM.Current(), "device-id": oo.deviceID})
+	if oo.baseDeviceHandler.allButCallingFsmInIdleState(ctx, cUploadFsm) {
+		oo.requestMdsValue(ctx)
+	} else {
+		logger.Debugw(ctx, "MibSync FSM", log.Fields{"Configuration is ongoing - skip re-auditing!": e.FSM.Current(), "device-id": oo.deviceID})
+		go func() {
+			_ = oo.pMibUploadFsm.pFsm.Event(ulEvSuccess)
+		}()
+	}
 }
 
 func (oo *OnuDeviceEntry) enterOutOfSyncState(ctx context.Context, e *fsm.Event) {
@@ -301,7 +343,7 @@
 			if msgOk {
 				logger.Debugw(ctx, "MibResetResponse Data", log.Fields{"data-fields": msgObj})
 				if msgObj.Result == me.Success {
-					oo.mibDataSyncAdpt = 0
+					oo.sOnuPersistentData.PersMibDataSyncAdpt = 0
 					// trigger retrieval of VendorId and SerialNumber
 					_ = oo.pMibUploadFsm.pFsm.Event(ulEvGetVendorAndSerial)
 					return
@@ -458,17 +500,7 @@
 				_ = oo.pMibUploadFsm.pFsm.Event(ulEvGetMibTemplate)
 				return nil
 			case "OnuData":
-				mibDataSyncOnu := meAttributes["MibDataSync"].(uint8)
-				logger.Debugw(ctx, "MibSync FSM - GetResponse Data for Onu-Data - MibDataSync", log.Fields{"device-id": oo.deviceID,
-					"mibDataSyncOnu": mibDataSyncOnu, "oo.mibDataSyncAdpt": oo.mibDataSyncAdpt})
-				if oo.pMibUploadFsm.pFsm.Is(ulStExaminingMds) {
-					// Examine MDS value
-					if oo.mibDataSyncAdpt == mibDataSyncOnu {
-						_ = oo.pMibUploadFsm.pFsm.Event(ulEvSuccess)
-					} else {
-						_ = oo.pMibUploadFsm.pFsm.Event(ulEvMismatch)
-					}
-				}
+				oo.checkMdsValue(ctx, meAttributes["MibDataSync"].(uint8))
 				return nil
 			}
 		} else {
@@ -537,7 +569,7 @@
 }
 
 func (oo *OnuDeviceEntry) isNewOnu() bool {
-	return oo.mibLastDbSync == 0
+	return oo.sOnuPersistentData.PersMibLastDbSync == 0
 }
 
 func isSupportedClassID(meClassID me.ClassID) bool {
@@ -645,18 +677,37 @@
 	oo.PDevOmciCC.pLastTxMeInstance = meInstance
 }
 
-// func (onuDeviceEntry *OnuDeviceEntry) MibTemplateTask() error {
-// 	return errors.New("not_implemented")
-// }
-// func (onuDeviceEntry *OnuDeviceEntry) MibUploadTask() error {
-// 	return errors.New("not_implemented")
-// }
-// func (onuDeviceEntry *OnuDeviceEntry) GetMdsTask() error {
-// 	return errors.New("not_implemented")
-// }
-// func (onuDeviceEntry *OnuDeviceEntry) MibResyncTask() error {
-// 	return errors.New("not_implemented")
-// }
-// func (onuDeviceEntry *OnuDeviceEntry) MibReconcileTask() error {
-// 	return errors.New("not_implemented")
-// }
+func (oo *OnuDeviceEntry) checkMdsValue(ctx context.Context, mibDataSyncOnu uint8) {
+	logger.Debugw(ctx, "MibSync FSM - GetResponse Data for Onu-Data - MibDataSync", log.Fields{"device-id": oo.deviceID,
+		"mibDataSyncOnu": mibDataSyncOnu, "PersMibDataSyncAdpt": oo.sOnuPersistentData.PersMibDataSyncAdpt})
+
+	mdsCheckOk := oo.sOnuPersistentData.PersMibDataSyncAdpt == mibDataSyncOnu
+	if oo.pMibUploadFsm.pFsm.Is(ulStAuditing) {
+		if mdsCheckOk {
+			logger.Debugw(ctx, "MibSync FSM - mib audit - MDS check ok", log.Fields{"device-id": oo.deviceID})
+			_ = oo.pMibUploadFsm.pFsm.Event(ulEvSuccess)
+		} else {
+			logger.Warnw(ctx, "MibSync FSM - mib audit - MDS check failed for the first time!", log.Fields{"device-id": oo.deviceID})
+			_ = oo.pMibUploadFsm.pFsm.Event(ulEvMismatch)
+		}
+	} else if oo.pMibUploadFsm.pFsm.Is(ulStReAuditing) {
+		if mdsCheckOk {
+			logger.Debugw(ctx, "MibSync FSM - mib reaudit - MDS check ok", log.Fields{"device-id": oo.deviceID})
+			_ = oo.pMibUploadFsm.pFsm.Event(ulEvSuccess)
+		} else {
+			logger.Errorw(ctx, "MibSync FSM - mib audit - MDS check failed for the second time!", log.Fields{"device-id": oo.deviceID})
+			//TODO: send new event notification "MDS counter mismatch" to the core
+			_ = oo.pMibUploadFsm.pFsm.Event(ulEvMismatch)
+		}
+	} else if oo.pMibUploadFsm.pFsm.Is(ulStExaminingMds) {
+		if mdsCheckOk {
+			logger.Debugw(ctx, "MibSync FSM - MDS examination ok", log.Fields{"device-id": oo.deviceID})
+			_ = oo.pMibUploadFsm.pFsm.Event(ulEvSuccess)
+		} else {
+			logger.Debugw(ctx, "MibSync FSM - MDS examination failed - new provisioning", log.Fields{"device-id": oo.deviceID})
+			_ = oo.pMibUploadFsm.pFsm.Event(ulEvMismatch)
+		}
+	} else {
+		logger.Warnw(ctx, "wrong state for MDS evaluation!", log.Fields{"state": oo.pMibUploadFsm.pFsm.Current(), "device-id": oo.deviceID})
+	}
+}
diff --git a/internal/pkg/onuadaptercore/omci_ani_config.go b/internal/pkg/onuadaptercore/omci_ani_config.go
index f908ea0..6cb96a3 100644
--- a/internal/pkg/onuadaptercore/omci_ani_config.go
+++ b/internal/pkg/onuadaptercore/omci_ani_config.go
@@ -81,6 +81,7 @@
 	aniStRemoveDone          = "aniStRemoveDone"
 	aniStResetting           = "aniStResetting"
 )
+const cAniFsmIdleState = aniStConfigDone
 
 const (
 	tpIDOffset = 64
diff --git a/internal/pkg/onuadaptercore/omci_vlan_config.go b/internal/pkg/onuadaptercore/omci_vlan_config.go
index 42a35f5..6725098 100644
--- a/internal/pkg/onuadaptercore/omci_vlan_config.go
+++ b/internal/pkg/onuadaptercore/omci_vlan_config.go
@@ -77,7 +77,7 @@
 )
 
 const (
-	// events of config PON ANI port FSM
+	// events of config UNI port VLAN FSM
 	vlanEvStart           = "vlanEvStart"
 	vlanEvWaitTechProf    = "vlanEvWaitTechProf"
 	vlanEvContinueConfig  = "vlanEvContinueConfig"
@@ -97,7 +97,7 @@
 )
 
 const (
-	// states of config PON ANI port FSM
+	// states of config UNI port VLAN FSM
 	vlanStDisabled        = "vlanStDisabled"
 	vlanStStarting        = "vlanStStarting"
 	vlanStWaitingTechProf = "vlanStWaitingTechProf"
@@ -110,6 +110,7 @@
 	vlanStCleanupDone     = "vlanStCleanupDone"
 	vlanStResetting       = "vlanStResetting"
 )
+const cVlanFsmIdleState = vlanStConfigDone
 
 type uniVlanRuleParams struct {
 	TpID         uint8  `json:"tp_id"`
diff --git a/internal/pkg/onuadaptercore/onu_device_entry.go b/internal/pkg/onuadaptercore/onu_device_entry.go
index dcc47b0..37d16fe 100644
--- a/internal/pkg/onuadaptercore/onu_device_entry.go
+++ b/internal/pkg/onuadaptercore/onu_device_entry.go
@@ -75,12 +75,15 @@
 	ulStGettingMacAddress      = "ulStGettingMacAddress"
 	ulStGettingMibTemplate     = "ulStGettingMibTemplate"
 	ulStUploading              = "ulStUploading"
+	ulStUploadDone             = "ulStUploadDone"
 	ulStInSync                 = "ulStInSync"
 	ulStExaminingMds           = "ulStExaminingMds"
 	ulStResynchronizing        = "ulStResynchronizing"
 	ulStAuditing               = "ulStAuditing"
+	ulStReAuditing             = "ulStReAuditing"
 	ulStOutOfSync              = "ulStOutOfSync"
 )
+const cMibUlFsmIdleState = ulStInSync
 
 const (
 	// events of MibDownload FSM
@@ -104,6 +107,7 @@
 	dlStDownloaded   = "dlStDownloaded"
 	dlStResetting    = "dlStResetting"
 )
+const cMibDlFsmIdleState = dlStDisabled
 
 const (
 	// NOTE that this hardcoded to service/voltha as the MIB template is shared across stacks
@@ -211,14 +215,17 @@
 }
 
 type onuPersistentData struct {
-	PersOnuID          uint32          `json:"onu_id"`
-	PersIntfID         uint32          `json:"intf_id"`
-	PersSnr            string          `json:"serial_number"`
-	PersAdminState     string          `json:"admin_state"`
-	PersOperState      string          `json:"oper_state"`
-	PersUniUnlockDone  bool            `json:"uni_unlock_done"`
-	PersUniDisableDone bool            `json:"uni_disable_done"`
-	PersUniConfig      []uniPersConfig `json:"uni_config"`
+	PersOnuID           uint32          `json:"onu_id"`
+	PersIntfID          uint32          `json:"intf_id"`
+	PersSnr             string          `json:"serial_number"`
+	PersAdminState      string          `json:"admin_state"`
+	PersOperState       string          `json:"oper_state"`
+	PersUniUnlockDone   bool            `json:"uni_unlock_done"`
+	PersUniDisableDone  bool            `json:"uni_disable_done"`
+	PersMibAuditDelay   uint16          `json:"mib_audit_delay"`
+	PersMibLastDbSync   uint32          `json:"mib_last_db_sync"`
+	PersMibDataSyncAdpt uint8           `json:"mib_data_sync_adpt"`
+	PersUniConfig       []uniPersConfig `json:"uni_config"`
 }
 
 // OnuDeviceEntry - ONU device info and FSM events.
@@ -249,9 +256,9 @@
 	supportedFsms OmciDeviceFsms
 	devState      OnuDeviceEvent
 	// Audit and MDS
-	mibAuditDelay   uint16
-	mibLastDbSync   uint32
-	mibDataSyncAdpt uint8
+	mibAuditDelay uint16
+	// TODO: periodical mib resync will be implemented with story VOL-3792
+	//mibNextDbResync uint32
 
 	// for mibUpload
 	pMibUploadFsm *AdapterFsm //could be handled dynamically and more general as pAdapterFsm - perhaps later
@@ -285,14 +292,15 @@
 	if dh.pOpenOnuAc.pSupportedFsms != nil {
 		onuDeviceEntry.supportedFsms = *dh.pOpenOnuAc.pSupportedFsms
 	} else {
+		// This branch is currently not used and is for potential future usage of alternative MIB Sync FSMs only!
 		//var mibSyncFsm = NewMibSynchronizer()
-		// use some internaö defaults, if not defined from outside
+		// use some internal defaults, if not defined from outside
 		onuDeviceEntry.supportedFsms = OmciDeviceFsms{
 			"mib-synchronizer": {
 				//mibSyncFsm,        // Implements the MIB synchronization state machine
 				onuDeviceEntry.mibDbVolatileDict, // Implements volatile ME MIB database
 				//true,                             // Advertise events on OpenOMCI event bus
-				60, // Time to wait between MIB audits.  0 to disable audits.
+				0, // Time to wait between MIB audits.  0 to disable audits.
 				// map[string]func() error{
 				// 	"mib-upload":    onuDeviceEntry.MibUploadTask,
 				// 	"mib-template":  onuDeviceEntry.MibTemplateTask,
@@ -307,10 +315,18 @@
 	onuDeviceEntry.mibDbClass = onuDeviceEntry.supportedFsms["mib-synchronizer"].databaseClass
 	logger.Debug(ctx, "access2mibDbClass")
 	go onuDeviceEntry.mibDbClass(ctx)
-	onuDeviceEntry.mibAuditDelay = onuDeviceEntry.supportedFsms["mib-synchronizer"].auditDelay
+	if !dh.reconciling {
+		onuDeviceEntry.mibAuditDelay = onuDeviceEntry.supportedFsms["mib-synchronizer"].auditDelay
+		onuDeviceEntry.sOnuPersistentData.PersMibAuditDelay = onuDeviceEntry.mibAuditDelay
+	} else {
+		logger.Debugw(ctx, "reconciling - take audit delay from persistent data", log.Fields{"device-id": dh.deviceID})
+		// TODO: This is a preparation for VOL-3786 to preserve config history in case of
+		// vendor- or deviceID-specific configurations via voltctl-commands
+		onuDeviceEntry.mibAuditDelay = onuDeviceEntry.sOnuPersistentData.PersMibAuditDelay
+	}
 	logger.Debugw(ctx, "MibAudit is set to", log.Fields{"Delay": onuDeviceEntry.mibAuditDelay})
-	onuDeviceEntry.mibLastDbSync = 0
-	onuDeviceEntry.mibDataSyncAdpt = 0
+	// TODO: periodical mib resync will be implemented with story VOL-3792
+	//onuDeviceEntry.mibNextDbResync = 0
 
 	// Omci related Mib upload sync state machine
 	mibUploadChan := make(chan Message, 2048)
@@ -332,9 +348,10 @@
 			{Name: ulEvUploadMib, Src: []string{ulStGettingMibTemplate}, Dst: ulStUploading},
 			{Name: ulEvExamineMds, Src: []string{ulStStarting}, Dst: ulStExaminingMds},
 
-			{Name: ulEvSuccess, Src: []string{ulStGettingMibTemplate}, Dst: ulStInSync},
-			{Name: ulEvSuccess, Src: []string{ulStUploading}, Dst: ulStInSync},
+			{Name: ulEvSuccess, Src: []string{ulStGettingMibTemplate}, Dst: ulStUploadDone},
+			{Name: ulEvSuccess, Src: []string{ulStUploading}, Dst: ulStUploadDone},
 
+			{Name: ulEvSuccess, Src: []string{ulStUploadDone}, Dst: ulStInSync},
 			{Name: ulEvSuccess, Src: []string{ulStExaminingMds}, Dst: ulStInSync},
 			// TODO: As long as mib-resynchronizing is not implemented, failed MDS-examination triggers
 			// mib-reset and new provisioning at this point
@@ -347,19 +364,22 @@
 			{Name: ulEvAuditMib, Src: []string{ulStOutOfSync}, Dst: ulStAuditing},
 
 			{Name: ulEvSuccess, Src: []string{ulStAuditing}, Dst: ulStInSync},
-			{Name: ulEvMismatch, Src: []string{ulStAuditing}, Dst: ulStResynchronizing},
+			{Name: ulEvMismatch, Src: []string{ulStAuditing}, Dst: ulStReAuditing},
 			{Name: ulEvForceResync, Src: []string{ulStAuditing}, Dst: ulStResynchronizing},
 
+			{Name: ulEvSuccess, Src: []string{ulStReAuditing}, Dst: ulStInSync},
+			{Name: ulEvMismatch, Src: []string{ulStReAuditing}, Dst: ulStResettingMib},
+
 			{Name: ulEvSuccess, Src: []string{ulStResynchronizing}, Dst: ulStInSync},
 			{Name: ulEvDiffsFound, Src: []string{ulStResynchronizing}, Dst: ulStOutOfSync},
 
 			{Name: ulEvTimeout, Src: []string{ulStResettingMib, ulStGettingVendorAndSerial, ulStGettingEquipmentID, ulStGettingFirstSwVersion,
 				ulStGettingSecondSwVersion, ulStGettingMacAddress, ulStGettingMibTemplate, ulStUploading, ulStResynchronizing, ulStExaminingMds,
-				ulStInSync, ulStOutOfSync, ulStAuditing}, Dst: ulStStarting},
+				ulStUploadDone, ulStInSync, ulStOutOfSync, ulStAuditing, ulStReAuditing}, Dst: ulStStarting},
 
 			{Name: ulEvStop, Src: []string{ulStStarting, ulStResettingMib, ulStGettingVendorAndSerial, ulStGettingEquipmentID, ulStGettingFirstSwVersion,
 				ulStGettingSecondSwVersion, ulStGettingMacAddress, ulStGettingMibTemplate, ulStUploading, ulStResynchronizing, ulStExaminingMds,
-				ulStInSync, ulStOutOfSync, ulStAuditing}, Dst: ulStDisabled},
+				ulStUploadDone, ulStInSync, ulStOutOfSync, ulStAuditing, ulStReAuditing}, Dst: ulStDisabled},
 		},
 
 		fsm.Callbacks{
@@ -373,9 +393,11 @@
 			"enter_" + ulStGettingMacAddress:      func(e *fsm.Event) { onuDeviceEntry.enterGettingMacAddressState(ctx, e) },
 			"enter_" + ulStGettingMibTemplate:     func(e *fsm.Event) { onuDeviceEntry.enterGettingMibTemplate(ctx, e) },
 			"enter_" + ulStUploading:              func(e *fsm.Event) { onuDeviceEntry.enterUploadingState(ctx, e) },
+			"enter_" + ulStUploadDone:             func(e *fsm.Event) { onuDeviceEntry.enterUploadDoneState(ctx, e) },
 			"enter_" + ulStExaminingMds:           func(e *fsm.Event) { onuDeviceEntry.enterExaminingMdsState(ctx, e) },
 			"enter_" + ulStResynchronizing:        func(e *fsm.Event) { onuDeviceEntry.enterResynchronizingState(ctx, e) },
 			"enter_" + ulStAuditing:               func(e *fsm.Event) { onuDeviceEntry.enterAuditingState(ctx, e) },
+			"enter_" + ulStReAuditing:             func(e *fsm.Event) { onuDeviceEntry.enterReAuditingState(ctx, e) },
 			"enter_" + ulStOutOfSync:              func(e *fsm.Event) { onuDeviceEntry.enterOutOfSyncState(ctx, e) },
 			"enter_" + ulStInSync:                 func(e *fsm.Event) { onuDeviceEntry.enterInSyncState(ctx, e) },
 		},
@@ -540,7 +562,7 @@
 		logger.Debugw(ctx, "onuKVStore not set - abort", log.Fields{"device-id": oo.deviceID})
 		return fmt.Errorf(fmt.Sprintf("onuKVStore-not-set-abort-%s", oo.deviceID))
 	}
-	oo.sOnuPersistentData = onuPersistentData{0, 0, "", "", "", false, false, make([]uniPersConfig, 0)}
+	oo.sOnuPersistentData = onuPersistentData{0, 0, "", "", "", false, false, oo.mibAuditDelay, 0, 0, make([]uniPersConfig, 0)}
 	Value, err := oo.onuKVStore.Get(ctx, oo.onuKVStorePath)
 	if err == nil {
 		if Value != nil {
@@ -586,8 +608,8 @@
 func (oo *OnuDeviceEntry) deletePersistentData(ctx context.Context, aProcessingStep uint8) {
 
 	logger.Debugw(ctx, "delete and clear internal persistency data", log.Fields{"device-id": oo.deviceID})
-	oo.sOnuPersistentData.PersUniConfig = nil                                                           //releasing all UniConfig entries to garbage collector
-	oo.sOnuPersistentData = onuPersistentData{0, 0, "", "", "", false, false, make([]uniPersConfig, 0)} //default entry
+	oo.sOnuPersistentData.PersUniConfig = nil                                                                                   //releasing all UniConfig entries to garbage collector
+	oo.sOnuPersistentData = onuPersistentData{0, 0, "", "", "", false, false, oo.mibAuditDelay, 0, 0, make([]uniPersConfig, 0)} //default entry
 
 	logger.Debugw(ctx, "delete ONU-data from KVStore", log.Fields{"device-id": oo.deviceID})
 	err := oo.onuKVStore.Delete(ctx, oo.onuKVStorePath)
@@ -764,11 +786,11 @@
 }
 
 func (oo *OnuDeviceEntry) incrementMibDataSync(ctx context.Context) {
-	if oo.mibDataSyncAdpt < 255 {
-		oo.mibDataSyncAdpt++
+	if oo.sOnuPersistentData.PersMibDataSyncAdpt < 255 {
+		oo.sOnuPersistentData.PersMibDataSyncAdpt++
 	} else {
 		// per G.984 and G.988 overflow starts over at 1 given 0 is reserved for reset
-		oo.mibDataSyncAdpt = 1
+		oo.sOnuPersistentData.PersMibDataSyncAdpt = 1
 	}
-	logger.Debugf(ctx, "mibDataSync updated - mds: %d - device-id: %s", oo.mibDataSyncAdpt, oo.deviceID)
+	logger.Debugf(ctx, "mibDataSync updated - mds: %d - device-id: %s", oo.sOnuPersistentData.PersMibDataSyncAdpt, oo.deviceID)
 }
diff --git a/internal/pkg/onuadaptercore/openonu.go b/internal/pkg/onuadaptercore/openonu.go
index 51978dc..1eeac1a 100644
--- a/internal/pkg/onuadaptercore/openonu.go
+++ b/internal/pkg/onuadaptercore/openonu.go
@@ -104,6 +104,7 @@
 			//mibSyncFsm,        // Implements the MIB synchronization state machine
 			mibDbVolatileDictImpl, // Implements volatile ME MIB database
 			//true,                  // Advertise events on OpenOMCI event bus
+			//TODO: audit delay should come from command parameters configured via helm charts (covered by VOL-3786)
 			cMibAuditDelayImpl, // Time to wait between MIB audits.  0 to disable audits.
 			// map[string]func() error{
 			// 	"mib-upload":    onuDeviceEntry.MibUploadTask,
diff --git a/internal/pkg/onuadaptercore/openonuimpl.go b/internal/pkg/onuadaptercore/openonuimpl.go
index c8cde8c..6250a03 100644
--- a/internal/pkg/onuadaptercore/openonuimpl.go
+++ b/internal/pkg/onuadaptercore/openonuimpl.go
@@ -86,7 +86,8 @@
 }
 */
 
-// do not use MibAudit
+// mib audit is deactivated by default - story planned to make it configurable:
+// TODO: VOL-3786 - Support configuration of mib data sync audit via helm charts and voltctl commands
 const cMibAuditDelayImpl = 0
 
 //suppose global methods per adapter ...
diff --git a/internal/pkg/onuadaptercore/uniportadmin.go b/internal/pkg/onuadaptercore/uniportadmin.go
index 69f3792..95bea42 100644
--- a/internal/pkg/onuadaptercore/uniportadmin.go
+++ b/internal/pkg/onuadaptercore/uniportadmin.go
@@ -63,6 +63,7 @@
 	uniStAdminDone   = "uniStAdminDone"
 	uniStResetting   = "uniStResetting"
 )
+const cUniFsmIdleState = uniStDisabled
 
 //newLockStateFsm is the 'constructor' for the state machine to lock/unlock the ONU UNI ports via OMCI
 func newLockStateFsm(ctx context.Context, apDevOmciCC *omciCC, aAdminState bool, aRequestEvent OnuDeviceEvent,