[VOL-3800] : Random ping failures seen on alpha and iskratel ONUs during sanity test
- Process gem port deletes after any active flow deletes are completed.
On certain models of ONUs it is seen that if flow deletes and gem port
deletes are interleaved with each other and it leaves some stale configuration
on the ONUs. As a result any further datapath related flow configuration
will not work predictabily - there are some invalid packet taggings.
Since during tech profile add, we setup the gemport, tcont before
adding the flows, it is reasonable to remove flows before deleting
the tech profile.
Change-Id: I62dbbf046f2ea5f9df5416f99060093b930bd448
diff --git a/VERSION b/VERSION
index 524cb55..f4f2e28 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-1.1.1
+1.1.2-dev159
diff --git a/internal/pkg/onuadaptercore/device_handler.go b/internal/pkg/onuadaptercore/device_handler.go
index bc8a185..a9683dd 100644
--- a/internal/pkg/onuadaptercore/device_handler.go
+++ b/internal/pkg/onuadaptercore/device_handler.go
@@ -2414,6 +2414,35 @@
delete(dh.UniVlanConfigFsmMap, apUniPort.uniID)
}
+//ProcessPendingTpDelete processes any pending TP delete (if available)
+func (dh *deviceHandler) ProcessPendingTpDelete(ctx context.Context, apUniPort *onuUniPort, aTpID uint8) {
+ logger.Debugw(ctx, "enter processing pending tp delete", log.Fields{"device-id": dh.deviceID, "tpID": aTpID})
+ if apUniPort == nil {
+ logger.Errorw(ctx, "uni port is nil", log.Fields{"device-id": dh.deviceID})
+ return
+ }
+ k := uniTP{uniID: apUniPort.uniID, tpID: aTpID}
+ if pAniConfigFsm, ok := dh.pOnuTP.pAniConfigFsm[k]; pAniConfigFsm != nil && ok {
+ pAniConfigStatemachine := pAniConfigFsm.pAdaptFsm.pFsm
+ if pAniConfigStatemachine != nil {
+ //If the gem port delete was waiting on flow remove, indicate event that flow remove is done
+ if pAniConfigStatemachine.Is(aniStWaitingFlowRem) {
+ logger.Debugw(ctx, "ani fsm in aniStWaitingFlowRem state - handling aniEvFlowRemDone event",
+ log.Fields{"device-id": dh.deviceID, "tpID": aTpID})
+ if err := pAniConfigStatemachine.Event(aniEvFlowRemDone); err != nil {
+ logger.Warnw(ctx, "AniConfigFsm: can't continue processing", log.Fields{"err": err,
+ "device-id": dh.deviceID, "UniPort": apUniPort.portNo, "tpID": aTpID})
+ return
+ }
+ } else {
+ logger.Debugw(ctx, "ani fsm not in aniStWaitingFlowRem state", log.Fields{"device-id": dh.deviceID, "tpID": aTpID})
+ return
+ }
+ }
+ return
+ }
+}
+
//storePersUniFlowConfig updates local storage of OnuUniFlowConfig and writes it into kv-store afterwards to have it
//available for potential reconcilement
diff --git a/internal/pkg/onuadaptercore/omci_ani_config.go b/internal/pkg/onuadaptercore/omci_ani_config.go
index fdd7830..f908ea0 100644
--- a/internal/pkg/onuadaptercore/omci_ani_config.go
+++ b/internal/pkg/onuadaptercore/omci_ani_config.go
@@ -47,6 +47,8 @@
aniEvRxPrioqsResp = "aniEvRxPrioqsResp"
aniEvRxDot1pmapSResp = "aniEvRxDot1pmapSResp"
aniEvRemGemiw = "aniEvRemGemiw"
+ aniEvWaitFlowRem = "aniEvWaitFlowRem"
+ aniEvFlowRemDone = "aniEvFlowRemDone"
aniEvRxRemGemiwResp = "aniEvRxRemGemiwResp"
aniEvRxRemGemntpResp = "aniEvRxRemGemntpResp"
aniEvRemTcontPath = "aniEvRemTcontPath"
@@ -71,6 +73,7 @@
aniStSettingDot1PMapper = "aniStSettingDot1PMapper"
aniStConfigDone = "aniStConfigDone"
aniStRemovingGemIW = "aniStRemovingGemIW"
+ aniStWaitingFlowRem = "aniStWaitingFlowRem"
aniStRemovingGemNCTP = "aniStRemovingGemNCTP"
aniStResetTcont = "aniStResetTcont"
aniStRemDot1PMapper = "aniStRemDot1PMapper"
@@ -167,6 +170,8 @@
//for removing Gem related resources
{Name: aniEvRemGemiw, Src: []string{aniStConfigDone}, Dst: aniStRemovingGemIW},
+ {Name: aniEvWaitFlowRem, Src: []string{aniStRemovingGemIW}, Dst: aniStWaitingFlowRem},
+ {Name: aniEvFlowRemDone, Src: []string{aniStWaitingFlowRem}, Dst: aniStRemovingGemIW},
{Name: aniEvRxRemGemiwResp, Src: []string{aniStRemovingGemIW}, Dst: aniStRemovingGemNCTP},
{Name: aniEvRxRemGemntpResp, Src: []string{aniStRemovingGemNCTP}, Dst: aniStConfigDone},
@@ -631,6 +636,25 @@
}
func (oFsm *uniPonAniConfigFsm) enterRemovingGemIW(ctx context.Context, e *fsm.Event) {
+
+ if oFsm.pDeviceHandler.UniVlanConfigFsmMap[oFsm.pOnuUniPort.uniID].IsFlowRemovePending() {
+ logger.Debugw(ctx, "flow remove pending - wait before processing gem port delete",
+ log.Fields{"device-id": oFsm.deviceID, "uni-id": oFsm.pOnuUniPort.uniID, "techProfile-id": oFsm.techProfileID})
+ // if flow remove is pending then wait for flow remove to finish first before proceeding with gem port delete
+ pConfigAniStateAFsm := oFsm.pAdaptFsm
+ if pConfigAniStateAFsm != nil {
+ // obviously calling some FSM event here directly does not work - so trying to decouple it ...
+ go func(aPAFsm *AdapterFsm) {
+ if aPAFsm != nil && aPAFsm.pFsm != nil {
+ _ = oFsm.pAdaptFsm.pFsm.Event(aniEvWaitFlowRem)
+ }
+ }(pConfigAniStateAFsm)
+ } else {
+ logger.Errorw(ctx, "pConfigAniStateAFsm is nil", log.Fields{"device-id": oFsm.deviceID, "uni-id": oFsm.pOnuUniPort.uniID, "techProfile-id": oFsm.techProfileID})
+ }
+ return
+ }
+
// get the related GemPort entity Id from pUniTechProf, OMCI Gem* entityID is set to be equal to GemPortId!
oFsm.pUniTechProf.mutexTPState.Lock()
loGemPortID := (*(oFsm.pUniTechProf.mapRemoveGemEntry[oFsm.uniTpKey])).gemPortID
diff --git a/internal/pkg/onuadaptercore/omci_vlan_config.go b/internal/pkg/onuadaptercore/omci_vlan_config.go
index 21cdbda..42a35f5 100644
--- a/internal/pkg/onuadaptercore/omci_vlan_config.go
+++ b/internal/pkg/onuadaptercore/omci_vlan_config.go
@@ -1061,12 +1061,20 @@
"device-id": oFsm.deviceID})
// Can't call FSM Event directly, decoupling it
go func(a_pBaseFsm *fsm.FSM) {
- _ = a_pBaseFsm.Event(vlanEvRemFlowDone)
+ _ = a_pBaseFsm.Event(vlanEvRemFlowDone, loRuleParams.TpID)
}(pConfigVlanStateBaseFsm)
}
}
func (oFsm *UniVlanConfigFsm) enterVlanCleanupDone(ctx context.Context, e *fsm.Event) {
+ var tpID uint8
+ // Extract the tpID
+ if len(e.Args) > 0 {
+ tpID = e.Args[0].(uint8)
+ logger.Debugw(ctx, "UniVlanConfigFsm - flow removed for tp id", log.Fields{"device-id": oFsm.deviceID, "tpID": e.Args[0].(uint8)})
+ } else {
+ logger.Warnw(ctx, "UniVlanConfigFsm - tp id not available", log.Fields{"device-id": oFsm.deviceID})
+ }
logger.Debugw(ctx, "UniVlanConfigFsm - removing the removal data", log.Fields{
"in state": e.FSM.Current(), "device-id": oFsm.deviceID})
@@ -1096,6 +1104,16 @@
}
}(pConfigVlanStateAFsm)
}
+
+ oFsm.mutexFlowParams.RLock()
+ noOfFlowRem := len(oFsm.uniRemoveFlowsSlice)
+ oFsm.mutexFlowParams.RUnlock()
+ // If all pending flow removes are completed and TP ID is valid, processing any pending TP delete
+ if noOfFlowRem == 0 && tpID > 0 {
+ logger.Debugw(ctx, "processing pending tp delete", log.Fields{"device-id": oFsm.deviceID, "tpID": tpID})
+ // If we are here then all flows are removed.
+ oFsm.pDeviceHandler.ProcessPendingTpDelete(ctx, oFsm.pOnuUniPort, tpID)
+ }
}
func (oFsm *UniVlanConfigFsm) enterResetting(ctx context.Context, e *fsm.Event) {
@@ -1835,7 +1853,7 @@
// if Config has been done for all EVTOCD entries let the FSM proceed
logger.Debugw(ctx, "EVTOCD filter remove loop finished", log.Fields{"device-id": oFsm.deviceID})
- _ = oFsm.pAdaptFsm.pFsm.Event(vlanEvRemFlowDone)
+ _ = oFsm.pAdaptFsm.pFsm.Event(vlanEvRemFlowDone, aRuleParams.TpID)
}
func (oFsm *UniVlanConfigFsm) waitforOmciResponse(ctx context.Context) error {
@@ -2055,3 +2073,10 @@
}
return nil
}
+
+// IsFlowRemovePending returns true if there are pending flows to remove, else false.
+func (oFsm *UniVlanConfigFsm) IsFlowRemovePending() bool {
+ oFsm.mutexFlowParams.RLock()
+ defer oFsm.mutexFlowParams.RUnlock()
+ return len(oFsm.uniRemoveFlowsSlice) > 0
+}