[VOL-4352] Device reconcile fails if flows are removed while the openonu-adapter is down

Change-Id: I2f39371bd4d6c30a147690d845088969e8a2a003
diff --git a/VERSION b/VERSION
index 1431ef2..01dd90f 100755
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.1.2-dev257
+2.1.2-dev258
diff --git a/internal/pkg/common/interfaces.go b/internal/pkg/common/interfaces.go
index 7c0d835..e1ef21f 100755
--- a/internal/pkg/common/interfaces.go
+++ b/internal/pkg/common/interfaces.go
@@ -59,6 +59,7 @@
 	GetPonPortNumber() *uint32
 	GetOnuIndication() *openolt.OnuIndication
 	GetUniVlanConfigFsm(uint8) IuniVlanConfigFsm
+	GetTechProfileInstanceFromParentAdapter(context.Context, uint8, string) (*ia.TechProfileDownloadMessage, error)
 
 	GetDeviceReasonString() string
 	ReasonUpdate(context.Context, uint8, bool) error
@@ -91,6 +92,8 @@
 	StartReconciling(context.Context, bool)
 	IsReconciling() bool
 	IsSkipOnuConfigReconciling() bool
+	SetReconcilingReasonUpdate(bool)
+	IsReconcilingReasonUpdate() bool
 	PrepareReconcilingWithActiveAdapter(context.Context)
 	ReconcileDeviceTechProf(context.Context) bool
 	ReconcileDeviceFlowConfig(context.Context)
diff --git a/internal/pkg/core/device_handler.go b/internal/pkg/core/device_handler.go
index d651df5..2ef1926 100755
--- a/internal/pkg/core/device_handler.go
+++ b/internal/pkg/core/device_handler.go
@@ -47,6 +47,7 @@
 	vc "github.com/opencord/voltha-protos/v5/go/common"
 	ca "github.com/opencord/voltha-protos/v5/go/core_adapter"
 	"github.com/opencord/voltha-protos/v5/go/extension"
+	"github.com/opencord/voltha-protos/v5/go/inter_adapter"
 	ia "github.com/opencord/voltha-protos/v5/go/inter_adapter"
 	of "github.com/opencord/voltha-protos/v5/go/openflow_13"
 	"github.com/opencord/voltha-protos/v5/go/openolt"
@@ -204,6 +205,8 @@
 	upgradeCanceled                bool
 	reconciling                    uint8
 	mutexReconcilingFlag           sync.RWMutex
+	reconcilingReasonUpdate        bool
+	mutexReconcilingReasonUpdate   sync.RWMutex
 	chUniVlanConfigReconcilingDone chan uint16 //channel to indicate that VlanConfig reconciling for a specific UNI has been finished
 	chReconcilingFinished          chan bool   //channel to indicate that reconciling has been finished
 	reconcileExpiryComplete        time.Duration
@@ -249,6 +252,7 @@
 	dh.lockUpgradeFsm = sync.RWMutex{}
 	dh.UniVlanConfigFsmMap = make(map[uint8]*avcfg.UniVlanConfigFsm)
 	dh.reconciling = cNoReconciling
+	dh.reconcilingReasonUpdate = false
 	dh.chReconcilingFinished = make(chan bool)
 	dh.reconcileExpiryComplete = adapter.maxTimeoutReconciling //assumption is to have it as duration in s!
 	rECSeconds := int(dh.reconcileExpiryComplete / time.Second)
@@ -850,7 +854,7 @@
 
 	pDevEntry := dh.GetOnuDeviceEntry(ctx, true)
 	if pDevEntry == nil {
-		logger.Errorw(ctx, "No valid OnuDevice - aborting", log.Fields{"device-id": dh.DeviceID})
+		logger.Errorw(ctx, "reconciling - no valid OnuDevice - aborting", log.Fields{"device-id": dh.DeviceID})
 		dh.stopReconciling(ctx, false, cWaitReconcileFlowNoActivity)
 		return continueWithFlowConfig
 	}
@@ -871,10 +875,11 @@
 	techProfInstLoadFailed := false
 outerLoop:
 	for _, uniData := range pDevEntry.SOnuPersistentData.PersUniConfig {
+		uniID := uniData.PersUniID
 		//TODO: check for uni-port specific reconcilement in case of multi-uni-port-per-onu-support
 		if !dh.anyTpPathExists(uniData.PersTpPathMap) {
 			logger.Debugw(ctx, "reconciling - no TPs stored for uniID",
-				log.Fields{"uni-id": uniData.PersUniID, "device-id": dh.DeviceID})
+				log.Fields{"uni-id": uniID, "device-id": dh.DeviceID})
 			continue
 		}
 		//release MutexPersOnuConfig before TechProfile (ANIConfig) processing as otherwise the reception of
@@ -883,37 +888,28 @@
 		pDevEntry.MutexPersOnuConfig.RUnlock()
 		persMutexLock = false
 		techProfsFound = true // set to true if we found TP once for any UNI port
+		var iaTechTpInst ia.TechProfileDownloadMessage
+		var ok bool
 		for tpID := range uniData.PersTpPathMap {
-			// Request the TpInstance again from the openolt adapter in case of reconcile
-			iaTechTpInst, err := dh.getTechProfileInstanceFromParentAdapter(ctx,
-				dh.device.ProxyAddress.AdapterEndpoint,
-				&ia.TechProfileInstanceRequestMessage{
-					DeviceId:       dh.device.Id,
-					TpInstancePath: uniData.PersTpPathMap[tpID],
-					ParentDeviceId: dh.parentID,
-					ParentPonPort:  dh.device.ParentPortNo,
-					OnuId:          dh.device.ProxyAddress.OnuId,
-					UniId:          uint32(uniData.PersUniID),
-				})
-			if err != nil || iaTechTpInst == nil {
-				// TODO: During the absence of the ONU adapter there seem to have been TP specific configurations!
-				// The no longer available TPs and the associated flows must be deleted from the ONU KV store
-				// and after a MIB reset a new reconciling attempt with OMCI configuration must be started.
-				logger.Errorw(ctx, "error fetching tp instance",
+			pDevEntry.MutexReconciledTpInstances.RLock()
+			if iaTechTpInst, ok = pDevEntry.ReconciledTpInstances[uniID][tpID]; !ok {
+				logger.Errorw(ctx, "reconciling - no reconciled tp instance available",
 					log.Fields{"tp-id": tpID, "tpPath": uniData.PersTpPathMap[tpID], "uni-id": uniData.PersUniID,
-						"device-id": dh.DeviceID, "err": err})
+						"device-id": dh.DeviceID})
 				techProfInstLoadFailed = true // stop loading tp instance as soon as we hit failure
+				pDevEntry.MutexReconciledTpInstances.RUnlock()
 				break outerLoop
 			}
+			pDevEntry.MutexReconciledTpInstances.RUnlock()
 			continueWithFlowConfig = true // valid TP found - try flow configuration later
 			var tpInst tech_profile.TechProfileInstance
 			switch techTpInst := iaTechTpInst.TechTpInstance.(type) {
 			case *ia.TechProfileDownloadMessage_TpInstance: // supports only GPON, XGPON, XGS-PON
 				tpInst = *techTpInst.TpInstance
-				logger.Debugw(ctx, "received-tp-instance-successfully-after-reconcile", log.Fields{
+				logger.Debugw(ctx, "reconciling - received-tp-instance-successfully-after-reconcile", log.Fields{
 					"tp-id": tpID, "tpPath": uniData.PersTpPathMap[tpID], "uni-id": uniData.PersUniID, "device-id": dh.DeviceID})
 			default: // do not support epon or other tech
-				logger.Errorw(ctx, "unsupported-tech-profile", log.Fields{
+				logger.Errorw(ctx, "reconciling - unsupported-tech-profile", log.Fields{
 					"tp-id": tpID, "tpPath": uniData.PersTpPathMap[tpID], "uni-id": uniData.PersUniID, "device-id": dh.DeviceID})
 				techProfInstLoadFailed = true // stop loading tp instance as soon as we hit failure
 				break outerLoop
@@ -960,11 +956,11 @@
 		return
 	}
 	if abTechProfInstLoadFailed {
-		_ = dh.ReasonUpdate(ctx, cmn.DrTechProfileConfigDownloadFailed, false)
+		_ = dh.ReasonUpdate(ctx, cmn.DrTechProfileConfigDownloadFailed, dh.IsReconcilingReasonUpdate())
 		dh.stopReconciling(ctx, false, cWaitReconcileFlowNoActivity)
 		return
 	} else if dh.IsSkipOnuConfigReconciling() {
-		_ = dh.ReasonUpdate(ctx, cmn.DrTechProfileConfigDownloadSuccess, false)
+		_ = dh.ReasonUpdate(ctx, cmn.DrTechProfileConfigDownloadSuccess, dh.IsReconcilingReasonUpdate())
 	}
 	if !abFlowsFound {
 		logger.Debugw(ctx, "reconciling - no flows have been stored before adapter restart - terminate reconcilement",
@@ -978,7 +974,7 @@
 
 	pDevEntry := dh.GetOnuDeviceEntry(ctx, true)
 	if pDevEntry == nil {
-		logger.Errorw(ctx, "No valid OnuDevice - aborting", log.Fields{"device-id": dh.DeviceID})
+		logger.Errorw(ctx, "reconciling - no valid OnuDevice - aborting", log.Fields{"device-id": dh.DeviceID})
 		dh.stopReconciling(ctx, false, cWaitReconcileFlowNoActivity)
 		return
 	}
@@ -1056,7 +1052,7 @@
 				pDevEntry.SendChReconcilingFlowsFinished(true)
 			}
 		} else {
-			logger.Errorw(ctx, "timeout waiting for reconciling flows for all UNI's to be finished!",
+			logger.Errorw(ctx, "reconciling - timeout waiting for reconciling flows for all UNI's to be finished!",
 				log.Fields{"device-id": dh.DeviceID})
 			dh.stopReconciling(ctx, false, cWaitReconcileFlowAbortOnError)
 			if pDevEntry != nil {
@@ -1064,7 +1060,7 @@
 			}
 			return
 		}
-		_ = dh.ReasonUpdate(ctx, cmn.DrOmciFlowsPushed, false)
+		_ = dh.ReasonUpdate(ctx, cmn.DrOmciFlowsPushed, dh.IsReconcilingReasonUpdate())
 	}
 }
 
@@ -1989,8 +1985,7 @@
 	if err := pDevEntry.Start(log.WithSpanFromContext(context.TODO(), ctx)); err != nil {
 		return err
 	}
-
-	_ = dh.ReasonUpdate(ctx, cmn.DrStartingOpenomci, !dh.IsReconciling())
+	_ = dh.ReasonUpdate(ctx, cmn.DrStartingOpenomci, !dh.IsReconciling() || dh.IsReconcilingReasonUpdate())
 
 	/* this might be a good time for Omci Verify message?  */
 	verifyExec := make(chan bool)
@@ -2104,9 +2099,9 @@
 					return fmt.Errorf("can't go to state resetting_mib: %s", dh.DeviceID)
 				}
 			} else {
-				if err := pMibUlFsm.Event(mib.UlEvExamineMds); err != nil {
-					logger.Errorw(ctx, "MibSyncFsm: Can't go to state examine_mds", log.Fields{"device-id": dh.DeviceID, "err": err})
-					return fmt.Errorf("can't go to examine_mds: %s", dh.DeviceID)
+				if err := pMibUlFsm.Event(mib.UlEvVerifyAndStoreTPs); err != nil {
+					logger.Errorw(ctx, "MibSyncFsm: Can't go to state verify and store TPs", log.Fields{"device-id": dh.DeviceID, "err": err})
+					return fmt.Errorf("can't go to state verify and store TPs: %s", dh.DeviceID)
 				}
 				logger.Debugw(ctx, "state of MibSyncFsm", log.Fields{"state": string(pMibUlFsm.Current())})
 			}
@@ -2285,7 +2280,7 @@
 		logger.Warnw(ctx, "store persistent data error - continue as there will be additional write attempts",
 			log.Fields{"device-id": dh.DeviceID, "err": err})
 	}
-	_ = dh.ReasonUpdate(ctx, cmn.DrDiscoveryMibsyncComplete, !dh.IsReconciling())
+	_ = dh.ReasonUpdate(ctx, cmn.DrDiscoveryMibsyncComplete, !dh.IsReconciling() || dh.IsReconcilingReasonUpdate())
 	dh.AddAllUniPorts(ctx)
 
 	/* 200605: lock processing after initial MIBUpload removed now as the ONU should be in the lock state per default here */
@@ -2369,7 +2364,7 @@
 		logger.Debugw(ctx, "reconciling - don't notify core about DeviceStateUpdate to ACTIVE",
 			log.Fields{"device-id": dh.DeviceID})
 	}
-	_ = dh.ReasonUpdate(ctx, cmn.DrInitialMibDownloaded, !dh.IsReconciling())
+	_ = dh.ReasonUpdate(ctx, cmn.DrInitialMibDownloaded, !dh.IsReconciling() || dh.IsReconcilingReasonUpdate())
 
 	if !dh.GetCollectorIsRunning() {
 		// Start PM collector routine
@@ -2537,7 +2532,7 @@
 		//  - which may cause some inconsistency
 		if dh.getDeviceReason() != cmn.DrTechProfileConfigDownloadSuccess {
 			// which may be the case from some previous activity even on this UNI Port (but also other UNI ports)
-			_ = dh.ReasonUpdate(ctx, cmn.DrTechProfileConfigDownloadSuccess, !dh.IsReconciling())
+			_ = dh.ReasonUpdate(ctx, cmn.DrTechProfileConfigDownloadSuccess, !dh.IsReconciling() || dh.IsReconcilingReasonUpdate())
 		}
 		if dh.IsReconciling() {
 			go dh.ReconcileDeviceFlowConfig(ctx)
@@ -2563,7 +2558,7 @@
 		if dh.getDeviceReason() != cmn.DrOmciFlowsPushed {
 			// which may be the case from some previous activity on another UNI Port of the ONU
 			// or even some previous flow add activity on the same port
-			_ = dh.ReasonUpdate(ctx, cmn.DrOmciFlowsPushed, !dh.IsReconciling())
+			_ = dh.ReasonUpdate(ctx, cmn.DrOmciFlowsPushed, !dh.IsReconciling() || dh.IsReconcilingReasonUpdate())
 			if dh.IsReconciling() {
 				go dh.reconcileEnd(ctx)
 			}
@@ -4051,6 +4046,15 @@
 			dh.mutexReconcilingFlag.Lock()
 			dh.reconciling = cNoReconciling
 			dh.mutexReconcilingFlag.Unlock()
+			dh.SetReconcilingReasonUpdate(false)
+
+			if onuDevEntry := dh.GetOnuDeviceEntry(ctx, true); onuDevEntry == nil {
+				logger.Errorw(ctx, "No valid OnuDevice", log.Fields{"device-id": dh.DeviceID})
+			} else {
+				onuDevEntry.MutexReconciledTpInstances.Lock()
+				onuDevEntry.ReconciledTpInstances = make(map[uint8]map[uint8]inter_adapter.TechProfileDownloadMessage)
+				onuDevEntry.MutexReconciledTpInstances.Unlock()
+			}
 		}()
 	}
 	dh.mutexReconcilingFlag.Lock()
@@ -4086,6 +4090,18 @@
 	return dh.reconciling == cSkipOnuConfigReconciling
 }
 
+func (dh *deviceHandler) SetReconcilingReasonUpdate(value bool) {
+	dh.mutexReconcilingReasonUpdate.Lock()
+	dh.reconcilingReasonUpdate = value
+	dh.mutexReconcilingReasonUpdate.Unlock()
+}
+
+func (dh *deviceHandler) IsReconcilingReasonUpdate() bool {
+	dh.mutexReconcilingReasonUpdate.RLock()
+	defer dh.mutexReconcilingReasonUpdate.RUnlock()
+	return dh.reconcilingReasonUpdate
+}
+
 func (dh *deviceHandler) getDeviceReason() uint8 {
 	dh.mutexDeviceReason.RLock()
 	value := dh.deviceReason
@@ -4217,16 +4233,26 @@
 Helper functions to communicate with parent adapter
 */
 
-func (dh *deviceHandler) getTechProfileInstanceFromParentAdapter(ctx context.Context, parentEndpoint string,
-	request *ia.TechProfileInstanceRequestMessage) (*ia.TechProfileDownloadMessage, error) {
-	pgClient, err := dh.pOpenOnuAc.getParentAdapterServiceClient(parentEndpoint)
+func (dh *deviceHandler) GetTechProfileInstanceFromParentAdapter(ctx context.Context, aUniID uint8,
+	aTpPath string) (*ia.TechProfileDownloadMessage, error) {
+
+	var request = ia.TechProfileInstanceRequestMessage{
+		DeviceId:       dh.DeviceID,
+		TpInstancePath: aTpPath,
+		ParentDeviceId: dh.parentID,
+		ParentPonPort:  dh.device.ParentPortNo,
+		OnuId:          dh.device.ProxyAddress.OnuId,
+		UniId:          uint32(aUniID),
+	}
+
+	pgClient, err := dh.pOpenOnuAc.getParentAdapterServiceClient(dh.device.ProxyAddress.AdapterEndpoint)
 	if err != nil || pgClient == nil {
 		return nil, err
 	}
 	subCtx, cancel := context.WithTimeout(log.WithSpanFromContext(context.Background(), ctx), dh.config.MaxTimeoutInterAdapterComm)
 	defer cancel()
-	logger.Debugw(subCtx, "get-tech-profile-instance", log.Fields{"request": request, "parent-endpoint": parentEndpoint})
-	return pgClient.GetTechProfileInstance(subCtx, request)
+	logger.Debugw(subCtx, "get-tech-profile-instance", log.Fields{"request": request, "parent-endpoint": dh.device.ProxyAddress.AdapterEndpoint})
+	return pgClient.GetTechProfileInstance(subCtx, &request)
 }
 
 // This routine is unique per ONU ID and blocks on flowControlBlock channel for incoming flows
diff --git a/internal/pkg/mib/mib_sync.go b/internal/pkg/mib/mib_sync.go
index 6ff7425..8da6244 100755
--- a/internal/pkg/mib/mib_sync.go
+++ b/internal/pkg/mib/mib_sync.go
@@ -37,6 +37,7 @@
 	"github.com/opencord/voltha-lib-go/v7/pkg/log"
 	cmn "github.com/opencord/voltha-openonu-adapter-go/internal/pkg/common"
 	devdb "github.com/opencord/voltha-openonu-adapter-go/internal/pkg/devdb"
+	"github.com/opencord/voltha-protos/v5/go/inter_adapter"
 )
 
 type sLastTxMeParameter struct {
@@ -283,6 +284,67 @@
 	}
 }
 
+func (oo *OnuDeviceEntry) enterVerifyingAndStoringTPsState(ctx context.Context, e *fsm.Event) {
+	logger.Debugw(ctx, "MibSync FSM", log.Fields{"Start verifying and storing TPs in State": e.FSM.Current(), "device-id": oo.deviceID})
+
+	allTpInstPresent := true
+	oo.MutexPersOnuConfig.Lock()
+	oo.MutexReconciledTpInstances.Lock()
+	for indexUni, uniData := range oo.SOnuPersistentData.PersUniConfig {
+		uniID := uniData.PersUniID
+		oo.ReconciledTpInstances[uniID] = make(map[uint8]inter_adapter.TechProfileDownloadMessage)
+		for tpID, tpPath := range uniData.PersTpPathMap {
+			if tpPath != "" {
+				// Request the TP instance from the openolt adapter
+				iaTechTpInst, err := oo.baseDeviceHandler.GetTechProfileInstanceFromParentAdapter(ctx, uniID, tpPath)
+				if err == nil && iaTechTpInst != nil {
+					logger.Debugw(ctx, "reconciling - store Tp instance", log.Fields{"uniID": uniID, "tpID": tpID,
+						"*iaTechTpInst": iaTechTpInst, "device-id": oo.deviceID})
+					oo.ReconciledTpInstances[uniID][tpID] = *iaTechTpInst
+				} else {
+					// During the absence of the ONU adapter there seem to have been TP specific configurations!
+					// The no longer available TP and the associated flows must be deleted from the ONU KV store
+					// and after a MIB reset a new reconciling attempt with OMCI configuration must be started.
+					allTpInstPresent = false
+					logger.Infow(ctx, "reconciling - can't get tp instance - delete tp and associated flows",
+						log.Fields{"tp-id": tpID, "tpPath": tpPath, "uni-id": uniID, "device-id": oo.deviceID, "err": err})
+					delete(oo.SOnuPersistentData.PersUniConfig[indexUni].PersTpPathMap, tpID)
+					flowSlice := oo.SOnuPersistentData.PersUniConfig[indexUni].PersFlowParams
+					for indexFlow, flowData := range flowSlice {
+						if flowData.VlanRuleParams.TpID == tpID {
+							if len(flowSlice) == 1 {
+								flowSlice = []cmn.UniVlanFlowParams{}
+							} else {
+								flowSlice = append(flowSlice[:indexFlow], flowSlice[indexFlow+1:]...)
+							}
+							oo.SOnuPersistentData.PersUniConfig[indexUni].PersFlowParams = flowSlice
+						}
+					}
+				}
+			}
+		}
+	}
+	oo.MutexReconciledTpInstances.Unlock()
+	oo.MutexPersOnuConfig.Unlock()
+
+	if allTpInstPresent {
+		logger.Debugw(ctx, "MibSync FSM", log.Fields{"reconciling - verifying TPs successful": e.FSM.Current(), "device-id": oo.deviceID})
+		go func() {
+			_ = oo.PMibUploadFsm.PFsm.Event(UlEvSuccess)
+		}()
+	} else {
+		logger.Debugw(ctx, "MibSync FSM", log.Fields{"reconciling - verifying TPs not successful": e.FSM.Current(), "device-id": oo.deviceID})
+		oo.baseDeviceHandler.SetReconcilingReasonUpdate(true)
+		go func() {
+			if err := oo.baseDeviceHandler.StorePersistentData(ctx); err != nil {
+				logger.Warnw(ctx, "reconciling - store persistent data error - continue for now as there will be additional write attempts",
+					log.Fields{"device-id": oo.deviceID, "err": err})
+			}
+			_ = oo.PMibUploadFsm.PFsm.Event(UlEvMismatch)
+		}()
+	}
+}
+
 func (oo *OnuDeviceEntry) enterExaminingMdsState(ctx context.Context, e *fsm.Event) {
 	logger.Debugw(ctx, "MibSync FSM", log.Fields{"Start GetMds processing in State": e.FSM.Current(), "device-id": oo.deviceID})
 	oo.requestMdsValue(ctx)
@@ -305,7 +367,7 @@
 	if oo.getMibFromTemplate(ctx) {
 		oo.baseDeviceHandler.StartReconciling(ctx, true)
 		oo.baseDeviceHandler.AddAllUniPorts(ctx)
-		_ = oo.baseDeviceHandler.ReasonUpdate(ctx, cmn.DrInitialMibDownloaded, false)
+		_ = oo.baseDeviceHandler.ReasonUpdate(ctx, cmn.DrInitialMibDownloaded, oo.baseDeviceHandler.IsReconcilingReasonUpdate())
 		oo.baseDeviceHandler.SetReadyForOmciConfig(true)
 
 		if !oo.baseDeviceHandler.GetCollectorIsRunning() {
@@ -328,7 +390,7 @@
 		if oo.SOnuPersistentData.PersUniDisableDone {
 			oo.MutexPersOnuConfig.RUnlock()
 			oo.baseDeviceHandler.DisableUniPortStateUpdate(ctx)
-			_ = oo.baseDeviceHandler.ReasonUpdate(ctx, cmn.DrOmciAdminLock, false)
+			_ = oo.baseDeviceHandler.ReasonUpdate(ctx, cmn.DrOmciAdminLock, oo.baseDeviceHandler.IsReconcilingReasonUpdate())
 		} else {
 			oo.MutexPersOnuConfig.RUnlock()
 			oo.baseDeviceHandler.EnableUniPortStateUpdate(ctx)
diff --git a/internal/pkg/mib/onu_device_entry.go b/internal/pkg/mib/onu_device_entry.go
index 517a94a..8ebf084 100755
--- a/internal/pkg/mib/onu_device_entry.go
+++ b/internal/pkg/mib/onu_device_entry.go
@@ -31,6 +31,7 @@
 	"github.com/opencord/voltha-lib-go/v7/pkg/db"
 	"github.com/opencord/voltha-lib-go/v7/pkg/db/kvstore"
 	vgrpc "github.com/opencord/voltha-lib-go/v7/pkg/grpc"
+	"github.com/opencord/voltha-protos/v5/go/inter_adapter"
 
 	"github.com/opencord/voltha-lib-go/v7/pkg/log"
 
@@ -50,6 +51,7 @@
 	UlEvGetMacAddress      = "UlEvGetMacAddress"
 	UlEvGetMibTemplate     = "UlEvGetMibTemplate"
 	UlEvUploadMib          = "UlEvUploadMib"
+	UlEvVerifyAndStoreTPs  = "UlEvVerifyAndStoreTPs"
 	UlEvExamineMds         = "UlEvExamineMds"
 	UlEvSuccess            = "UlEvSuccess"
 	UlEvMismatch           = "UlEvMismatch"
@@ -74,6 +76,7 @@
 	UlStUploading              = "UlStUploading"
 	UlStUploadDone             = "UlStUploadDone"
 	UlStInSync                 = "UlStInSync"
+	UlStVerifyingAndStoringTPs = "UlStVerifyingAndStoringTPs"
 	UlStExaminingMds           = "UlStExaminingMds"
 	UlStResynchronizing        = "UlStResynchronizing"
 	UlStExaminingMdsSuccess    = "UlStExaminingMdsSuccess"
@@ -147,6 +150,8 @@
 	PersTcontMap           map[uint16]uint16 `json:"tcont_map"` //alloc-id to me-instance-id map
 }
 
+//type UniTpidInstances map[uint8]map[uint8]inter_adapter.TechProfileDownloadMessage
+
 // OnuDeviceEntry - ONU device info and FSM events.
 type OnuDeviceEntry struct {
 	deviceID                   string
@@ -159,6 +164,8 @@
 	mibTemplateKVStore         *db.Backend
 	MutexPersOnuConfig         sync.RWMutex
 	SOnuPersistentData         onuPersistentData
+	ReconciledTpInstances      map[uint8]map[uint8]inter_adapter.TechProfileDownloadMessage
+	MutexReconciledTpInstances sync.RWMutex
 	reconcilingFlows           bool
 	mutexReconcilingFlowsFlag  sync.RWMutex
 	chReconcilingFlowsFinished chan bool //channel to indicate that reconciling of flows has been finished
@@ -210,6 +217,7 @@
 	onuDeviceEntry.devState = cmn.DeviceStatusInit
 	onuDeviceEntry.SOnuPersistentData.PersUniConfig = make([]uniPersConfig, 0)
 	onuDeviceEntry.SOnuPersistentData.PersTcontMap = make(map[uint16]uint16)
+	onuDeviceEntry.ReconciledTpInstances = make(map[uint8]map[uint8]inter_adapter.TechProfileDownloadMessage)
 	onuDeviceEntry.chReconcilingFlowsFinished = make(chan bool)
 	onuDeviceEntry.reconcilingFlows = false
 	onuDeviceEntry.chOnuKvProcessingStep = make(chan uint8)
@@ -281,7 +289,10 @@
 			{Name: UlEvGetMibTemplate, Src: []string{UlStGettingMacAddress}, Dst: UlStGettingMibTemplate},
 
 			{Name: UlEvUploadMib, Src: []string{UlStGettingMibTemplate}, Dst: UlStUploading},
-			{Name: UlEvExamineMds, Src: []string{UlStStarting}, Dst: UlStExaminingMds},
+
+			{Name: UlEvVerifyAndStoreTPs, Src: []string{UlStStarting}, Dst: UlStVerifyingAndStoringTPs},
+			{Name: UlEvSuccess, Src: []string{UlStVerifyingAndStoringTPs}, Dst: UlStExaminingMds},
+			{Name: UlEvMismatch, Src: []string{UlStVerifyingAndStoringTPs}, Dst: UlStResettingMib},
 
 			{Name: UlEvSuccess, Src: []string{UlStGettingMibTemplate}, Dst: UlStUploadDone},
 			{Name: UlEvSuccess, Src: []string{UlStUploading}, Dst: UlStUploadDone},
@@ -313,12 +324,12 @@
 			{Name: UlEvDiffsFound, Src: []string{UlStResynchronizing}, Dst: UlStOutOfSync},
 
 			{Name: UlEvTimeout, Src: []string{UlStResettingMib, UlStGettingVendorAndSerial, UlStGettingEquipmentID, UlStGettingFirstSwVersion,
-				UlStGettingSecondSwVersion, UlStGettingMacAddress, UlStGettingMibTemplate, UlStUploading, UlStResynchronizing, UlStExaminingMds,
-				UlStUploadDone, UlStInSync, UlStOutOfSync, UlStAuditing, UlStReAuditing}, Dst: UlStStarting},
+				UlStGettingSecondSwVersion, UlStGettingMacAddress, UlStGettingMibTemplate, UlStUploading, UlStResynchronizing, UlStVerifyingAndStoringTPs,
+				UlStExaminingMds, UlStUploadDone, UlStInSync, UlStOutOfSync, UlStAuditing, UlStReAuditing}, Dst: UlStStarting},
 
 			{Name: UlEvStop, Src: []string{UlStStarting, UlStResettingMib, UlStGettingVendorAndSerial, UlStGettingEquipmentID, UlStGettingFirstSwVersion,
-				UlStGettingSecondSwVersion, UlStGettingMacAddress, UlStGettingMibTemplate, UlStUploading, UlStResynchronizing, UlStExaminingMds,
-				UlStUploadDone, UlStInSync, UlStOutOfSync, UlStAuditing, UlStReAuditing}, Dst: UlStDisabled},
+				UlStGettingSecondSwVersion, UlStGettingMacAddress, UlStGettingMibTemplate, UlStUploading, UlStResynchronizing, UlStVerifyingAndStoringTPs,
+				UlStExaminingMds, UlStUploadDone, UlStInSync, UlStOutOfSync, UlStAuditing, UlStReAuditing}, Dst: UlStDisabled},
 		},
 
 		fsm.Callbacks{
@@ -334,6 +345,7 @@
 			"enter_" + UlStUploading:              func(e *fsm.Event) { onuDeviceEntry.enterUploadingState(ctx, e) },
 			"enter_" + UlStUploadDone:             func(e *fsm.Event) { onuDeviceEntry.enterUploadDoneState(ctx, e) },
 			"enter_" + UlStExaminingMds:           func(e *fsm.Event) { onuDeviceEntry.enterExaminingMdsState(ctx, e) },
+			"enter_" + UlStVerifyingAndStoringTPs: func(e *fsm.Event) { onuDeviceEntry.enterVerifyingAndStoringTPsState(ctx, e) },
 			"enter_" + UlStResynchronizing:        func(e *fsm.Event) { onuDeviceEntry.enterResynchronizingState(ctx, e) },
 			"enter_" + UlStExaminingMdsSuccess:    func(e *fsm.Event) { onuDeviceEntry.enterExaminingMdsSuccessState(ctx, e) },
 			"enter_" + UlStAuditing:               func(e *fsm.Event) { onuDeviceEntry.enterAuditingState(ctx, e) },