[VOL-4352] Device reconcile fails if flows are removed while the openonu-adapter is down

Change-Id: I2f39371bd4d6c30a147690d845088969e8a2a003
diff --git a/internal/pkg/mib/mib_sync.go b/internal/pkg/mib/mib_sync.go
index 6ff7425..8da6244 100755
--- a/internal/pkg/mib/mib_sync.go
+++ b/internal/pkg/mib/mib_sync.go
@@ -37,6 +37,7 @@
 	"github.com/opencord/voltha-lib-go/v7/pkg/log"
 	cmn "github.com/opencord/voltha-openonu-adapter-go/internal/pkg/common"
 	devdb "github.com/opencord/voltha-openonu-adapter-go/internal/pkg/devdb"
+	"github.com/opencord/voltha-protos/v5/go/inter_adapter"
 )
 
 type sLastTxMeParameter struct {
@@ -283,6 +284,67 @@
 	}
 }
 
+func (oo *OnuDeviceEntry) enterVerifyingAndStoringTPsState(ctx context.Context, e *fsm.Event) {
+	logger.Debugw(ctx, "MibSync FSM", log.Fields{"Start verifying and storing TPs in State": e.FSM.Current(), "device-id": oo.deviceID})
+
+	allTpInstPresent := true
+	oo.MutexPersOnuConfig.Lock()
+	oo.MutexReconciledTpInstances.Lock()
+	for indexUni, uniData := range oo.SOnuPersistentData.PersUniConfig {
+		uniID := uniData.PersUniID
+		oo.ReconciledTpInstances[uniID] = make(map[uint8]inter_adapter.TechProfileDownloadMessage)
+		for tpID, tpPath := range uniData.PersTpPathMap {
+			if tpPath != "" {
+				// Request the TP instance from the openolt adapter
+				iaTechTpInst, err := oo.baseDeviceHandler.GetTechProfileInstanceFromParentAdapter(ctx, uniID, tpPath)
+				if err == nil && iaTechTpInst != nil {
+					logger.Debugw(ctx, "reconciling - store Tp instance", log.Fields{"uniID": uniID, "tpID": tpID,
+						"*iaTechTpInst": iaTechTpInst, "device-id": oo.deviceID})
+					oo.ReconciledTpInstances[uniID][tpID] = *iaTechTpInst
+				} else {
+					// During the absence of the ONU adapter there seem to have been TP specific configurations!
+					// The no longer available TP and the associated flows must be deleted from the ONU KV store
+					// and after a MIB reset a new reconciling attempt with OMCI configuration must be started.
+					allTpInstPresent = false
+					logger.Infow(ctx, "reconciling - can't get tp instance - delete tp and associated flows",
+						log.Fields{"tp-id": tpID, "tpPath": tpPath, "uni-id": uniID, "device-id": oo.deviceID, "err": err})
+					delete(oo.SOnuPersistentData.PersUniConfig[indexUni].PersTpPathMap, tpID)
+					flowSlice := oo.SOnuPersistentData.PersUniConfig[indexUni].PersFlowParams
+					for indexFlow, flowData := range flowSlice {
+						if flowData.VlanRuleParams.TpID == tpID {
+							if len(flowSlice) == 1 {
+								flowSlice = []cmn.UniVlanFlowParams{}
+							} else {
+								flowSlice = append(flowSlice[:indexFlow], flowSlice[indexFlow+1:]...)
+							}
+							oo.SOnuPersistentData.PersUniConfig[indexUni].PersFlowParams = flowSlice
+						}
+					}
+				}
+			}
+		}
+	}
+	oo.MutexReconciledTpInstances.Unlock()
+	oo.MutexPersOnuConfig.Unlock()
+
+	if allTpInstPresent {
+		logger.Debugw(ctx, "MibSync FSM", log.Fields{"reconciling - verifying TPs successful": e.FSM.Current(), "device-id": oo.deviceID})
+		go func() {
+			_ = oo.PMibUploadFsm.PFsm.Event(UlEvSuccess)
+		}()
+	} else {
+		logger.Debugw(ctx, "MibSync FSM", log.Fields{"reconciling - verifying TPs not successful": e.FSM.Current(), "device-id": oo.deviceID})
+		oo.baseDeviceHandler.SetReconcilingReasonUpdate(true)
+		go func() {
+			if err := oo.baseDeviceHandler.StorePersistentData(ctx); err != nil {
+				logger.Warnw(ctx, "reconciling - store persistent data error - continue for now as there will be additional write attempts",
+					log.Fields{"device-id": oo.deviceID, "err": err})
+			}
+			_ = oo.PMibUploadFsm.PFsm.Event(UlEvMismatch)
+		}()
+	}
+}
+
 func (oo *OnuDeviceEntry) enterExaminingMdsState(ctx context.Context, e *fsm.Event) {
 	logger.Debugw(ctx, "MibSync FSM", log.Fields{"Start GetMds processing in State": e.FSM.Current(), "device-id": oo.deviceID})
 	oo.requestMdsValue(ctx)
@@ -305,7 +367,7 @@
 	if oo.getMibFromTemplate(ctx) {
 		oo.baseDeviceHandler.StartReconciling(ctx, true)
 		oo.baseDeviceHandler.AddAllUniPorts(ctx)
-		_ = oo.baseDeviceHandler.ReasonUpdate(ctx, cmn.DrInitialMibDownloaded, false)
+		_ = oo.baseDeviceHandler.ReasonUpdate(ctx, cmn.DrInitialMibDownloaded, oo.baseDeviceHandler.IsReconcilingReasonUpdate())
 		oo.baseDeviceHandler.SetReadyForOmciConfig(true)
 
 		if !oo.baseDeviceHandler.GetCollectorIsRunning() {
@@ -328,7 +390,7 @@
 		if oo.SOnuPersistentData.PersUniDisableDone {
 			oo.MutexPersOnuConfig.RUnlock()
 			oo.baseDeviceHandler.DisableUniPortStateUpdate(ctx)
-			_ = oo.baseDeviceHandler.ReasonUpdate(ctx, cmn.DrOmciAdminLock, false)
+			_ = oo.baseDeviceHandler.ReasonUpdate(ctx, cmn.DrOmciAdminLock, oo.baseDeviceHandler.IsReconcilingReasonUpdate())
 		} else {
 			oo.MutexPersOnuConfig.RUnlock()
 			oo.baseDeviceHandler.EnableUniPortStateUpdate(ctx)
diff --git a/internal/pkg/mib/onu_device_entry.go b/internal/pkg/mib/onu_device_entry.go
index 517a94a..8ebf084 100755
--- a/internal/pkg/mib/onu_device_entry.go
+++ b/internal/pkg/mib/onu_device_entry.go
@@ -31,6 +31,7 @@
 	"github.com/opencord/voltha-lib-go/v7/pkg/db"
 	"github.com/opencord/voltha-lib-go/v7/pkg/db/kvstore"
 	vgrpc "github.com/opencord/voltha-lib-go/v7/pkg/grpc"
+	"github.com/opencord/voltha-protos/v5/go/inter_adapter"
 
 	"github.com/opencord/voltha-lib-go/v7/pkg/log"
 
@@ -50,6 +51,7 @@
 	UlEvGetMacAddress      = "UlEvGetMacAddress"
 	UlEvGetMibTemplate     = "UlEvGetMibTemplate"
 	UlEvUploadMib          = "UlEvUploadMib"
+	UlEvVerifyAndStoreTPs  = "UlEvVerifyAndStoreTPs"
 	UlEvExamineMds         = "UlEvExamineMds"
 	UlEvSuccess            = "UlEvSuccess"
 	UlEvMismatch           = "UlEvMismatch"
@@ -74,6 +76,7 @@
 	UlStUploading              = "UlStUploading"
 	UlStUploadDone             = "UlStUploadDone"
 	UlStInSync                 = "UlStInSync"
+	UlStVerifyingAndStoringTPs = "UlStVerifyingAndStoringTPs"
 	UlStExaminingMds           = "UlStExaminingMds"
 	UlStResynchronizing        = "UlStResynchronizing"
 	UlStExaminingMdsSuccess    = "UlStExaminingMdsSuccess"
@@ -147,6 +150,8 @@
 	PersTcontMap           map[uint16]uint16 `json:"tcont_map"` //alloc-id to me-instance-id map
 }
 
+//type UniTpidInstances map[uint8]map[uint8]inter_adapter.TechProfileDownloadMessage
+
 // OnuDeviceEntry - ONU device info and FSM events.
 type OnuDeviceEntry struct {
 	deviceID                   string
@@ -159,6 +164,8 @@
 	mibTemplateKVStore         *db.Backend
 	MutexPersOnuConfig         sync.RWMutex
 	SOnuPersistentData         onuPersistentData
+	ReconciledTpInstances      map[uint8]map[uint8]inter_adapter.TechProfileDownloadMessage
+	MutexReconciledTpInstances sync.RWMutex
 	reconcilingFlows           bool
 	mutexReconcilingFlowsFlag  sync.RWMutex
 	chReconcilingFlowsFinished chan bool //channel to indicate that reconciling of flows has been finished
@@ -210,6 +217,7 @@
 	onuDeviceEntry.devState = cmn.DeviceStatusInit
 	onuDeviceEntry.SOnuPersistentData.PersUniConfig = make([]uniPersConfig, 0)
 	onuDeviceEntry.SOnuPersistentData.PersTcontMap = make(map[uint16]uint16)
+	onuDeviceEntry.ReconciledTpInstances = make(map[uint8]map[uint8]inter_adapter.TechProfileDownloadMessage)
 	onuDeviceEntry.chReconcilingFlowsFinished = make(chan bool)
 	onuDeviceEntry.reconcilingFlows = false
 	onuDeviceEntry.chOnuKvProcessingStep = make(chan uint8)
@@ -281,7 +289,10 @@
 			{Name: UlEvGetMibTemplate, Src: []string{UlStGettingMacAddress}, Dst: UlStGettingMibTemplate},
 
 			{Name: UlEvUploadMib, Src: []string{UlStGettingMibTemplate}, Dst: UlStUploading},
-			{Name: UlEvExamineMds, Src: []string{UlStStarting}, Dst: UlStExaminingMds},
+
+			{Name: UlEvVerifyAndStoreTPs, Src: []string{UlStStarting}, Dst: UlStVerifyingAndStoringTPs},
+			{Name: UlEvSuccess, Src: []string{UlStVerifyingAndStoringTPs}, Dst: UlStExaminingMds},
+			{Name: UlEvMismatch, Src: []string{UlStVerifyingAndStoringTPs}, Dst: UlStResettingMib},
 
 			{Name: UlEvSuccess, Src: []string{UlStGettingMibTemplate}, Dst: UlStUploadDone},
 			{Name: UlEvSuccess, Src: []string{UlStUploading}, Dst: UlStUploadDone},
@@ -313,12 +324,12 @@
 			{Name: UlEvDiffsFound, Src: []string{UlStResynchronizing}, Dst: UlStOutOfSync},
 
 			{Name: UlEvTimeout, Src: []string{UlStResettingMib, UlStGettingVendorAndSerial, UlStGettingEquipmentID, UlStGettingFirstSwVersion,
-				UlStGettingSecondSwVersion, UlStGettingMacAddress, UlStGettingMibTemplate, UlStUploading, UlStResynchronizing, UlStExaminingMds,
-				UlStUploadDone, UlStInSync, UlStOutOfSync, UlStAuditing, UlStReAuditing}, Dst: UlStStarting},
+				UlStGettingSecondSwVersion, UlStGettingMacAddress, UlStGettingMibTemplate, UlStUploading, UlStResynchronizing, UlStVerifyingAndStoringTPs,
+				UlStExaminingMds, UlStUploadDone, UlStInSync, UlStOutOfSync, UlStAuditing, UlStReAuditing}, Dst: UlStStarting},
 
 			{Name: UlEvStop, Src: []string{UlStStarting, UlStResettingMib, UlStGettingVendorAndSerial, UlStGettingEquipmentID, UlStGettingFirstSwVersion,
-				UlStGettingSecondSwVersion, UlStGettingMacAddress, UlStGettingMibTemplate, UlStUploading, UlStResynchronizing, UlStExaminingMds,
-				UlStUploadDone, UlStInSync, UlStOutOfSync, UlStAuditing, UlStReAuditing}, Dst: UlStDisabled},
+				UlStGettingSecondSwVersion, UlStGettingMacAddress, UlStGettingMibTemplate, UlStUploading, UlStResynchronizing, UlStVerifyingAndStoringTPs,
+				UlStExaminingMds, UlStUploadDone, UlStInSync, UlStOutOfSync, UlStAuditing, UlStReAuditing}, Dst: UlStDisabled},
 		},
 
 		fsm.Callbacks{
@@ -334,6 +345,7 @@
 			"enter_" + UlStUploading:              func(e *fsm.Event) { onuDeviceEntry.enterUploadingState(ctx, e) },
 			"enter_" + UlStUploadDone:             func(e *fsm.Event) { onuDeviceEntry.enterUploadDoneState(ctx, e) },
 			"enter_" + UlStExaminingMds:           func(e *fsm.Event) { onuDeviceEntry.enterExaminingMdsState(ctx, e) },
+			"enter_" + UlStVerifyingAndStoringTPs: func(e *fsm.Event) { onuDeviceEntry.enterVerifyingAndStoringTPsState(ctx, e) },
 			"enter_" + UlStResynchronizing:        func(e *fsm.Event) { onuDeviceEntry.enterResynchronizingState(ctx, e) },
 			"enter_" + UlStExaminingMdsSuccess:    func(e *fsm.Event) { onuDeviceEntry.enterExaminingMdsSuccessState(ctx, e) },
 			"enter_" + UlStAuditing:               func(e *fsm.Event) { onuDeviceEntry.enterAuditingState(ctx, e) },