[VOL-3800] : Random ping failures seen on alpha and iskratel ONUs during sanity test
- Process gem port deletes after any active flow deletes are completed.
  On certain models of ONUs it is seen that if flow deletes and gem port
  deletes are interleaved with each other and it leaves some stale configuration
  on the ONUs. As a result any further datapath related flow configuration
  will not work predictabily - there are some invalid packet taggings.

  Since during tech profile add, we setup the gemport, tcont before
  adding the flows, it is reasonable to remove flows before deleting
  the tech profile.

Change-Id: I62dbbf046f2ea5f9df5416f99060093b930bd448
diff --git a/internal/pkg/onuadaptercore/omci_ani_config.go b/internal/pkg/onuadaptercore/omci_ani_config.go
index fdd7830..f908ea0 100644
--- a/internal/pkg/onuadaptercore/omci_ani_config.go
+++ b/internal/pkg/onuadaptercore/omci_ani_config.go
@@ -47,6 +47,8 @@
 	aniEvRxPrioqsResp      = "aniEvRxPrioqsResp"
 	aniEvRxDot1pmapSResp   = "aniEvRxDot1pmapSResp"
 	aniEvRemGemiw          = "aniEvRemGemiw"
+	aniEvWaitFlowRem       = "aniEvWaitFlowRem"
+	aniEvFlowRemDone       = "aniEvFlowRemDone"
 	aniEvRxRemGemiwResp    = "aniEvRxRemGemiwResp"
 	aniEvRxRemGemntpResp   = "aniEvRxRemGemntpResp"
 	aniEvRemTcontPath      = "aniEvRemTcontPath"
@@ -71,6 +73,7 @@
 	aniStSettingDot1PMapper  = "aniStSettingDot1PMapper"
 	aniStConfigDone          = "aniStConfigDone"
 	aniStRemovingGemIW       = "aniStRemovingGemIW"
+	aniStWaitingFlowRem      = "aniStWaitingFlowRem"
 	aniStRemovingGemNCTP     = "aniStRemovingGemNCTP"
 	aniStResetTcont          = "aniStResetTcont"
 	aniStRemDot1PMapper      = "aniStRemDot1PMapper"
@@ -167,6 +170,8 @@
 
 			//for removing Gem related resources
 			{Name: aniEvRemGemiw, Src: []string{aniStConfigDone}, Dst: aniStRemovingGemIW},
+			{Name: aniEvWaitFlowRem, Src: []string{aniStRemovingGemIW}, Dst: aniStWaitingFlowRem},
+			{Name: aniEvFlowRemDone, Src: []string{aniStWaitingFlowRem}, Dst: aniStRemovingGemIW},
 			{Name: aniEvRxRemGemiwResp, Src: []string{aniStRemovingGemIW}, Dst: aniStRemovingGemNCTP},
 			{Name: aniEvRxRemGemntpResp, Src: []string{aniStRemovingGemNCTP}, Dst: aniStConfigDone},
 
@@ -631,6 +636,25 @@
 }
 
 func (oFsm *uniPonAniConfigFsm) enterRemovingGemIW(ctx context.Context, e *fsm.Event) {
+
+	if oFsm.pDeviceHandler.UniVlanConfigFsmMap[oFsm.pOnuUniPort.uniID].IsFlowRemovePending() {
+		logger.Debugw(ctx, "flow remove pending - wait before processing gem port delete",
+			log.Fields{"device-id": oFsm.deviceID, "uni-id": oFsm.pOnuUniPort.uniID, "techProfile-id": oFsm.techProfileID})
+		// if flow remove is pending then wait for flow remove to finish first before proceeding with gem port delete
+		pConfigAniStateAFsm := oFsm.pAdaptFsm
+		if pConfigAniStateAFsm != nil {
+			// obviously calling some FSM event here directly does not work - so trying to decouple it ...
+			go func(aPAFsm *AdapterFsm) {
+				if aPAFsm != nil && aPAFsm.pFsm != nil {
+					_ = oFsm.pAdaptFsm.pFsm.Event(aniEvWaitFlowRem)
+				}
+			}(pConfigAniStateAFsm)
+		} else {
+			logger.Errorw(ctx, "pConfigAniStateAFsm is nil", log.Fields{"device-id": oFsm.deviceID, "uni-id": oFsm.pOnuUniPort.uniID, "techProfile-id": oFsm.techProfileID})
+		}
+		return
+	}
+
 	// get the related GemPort entity Id from pUniTechProf, OMCI Gem* entityID is set to be equal to GemPortId!
 	oFsm.pUniTechProf.mutexTPState.Lock()
 	loGemPortID := (*(oFsm.pUniTechProf.mapRemoveGemEntry[oFsm.uniTpKey])).gemPortID