[VOL-3441] Code extensions to support multiple incremental flows in parallel per UNI

Signed-off-by: Holger Hildebrandt <holger.hildebrandt@adtran.com>
Change-Id: I951fe11b4e967b01e071808d4490d61e0d04fb4d
diff --git a/internal/pkg/onuadaptercore/device_handler.go b/internal/pkg/onuadaptercore/device_handler.go
index 379daa0..f7dad9a 100644
--- a/internal/pkg/onuadaptercore/device_handler.go
+++ b/internal/pkg/onuadaptercore/device_handler.go
@@ -526,8 +526,6 @@
 }
 
 //reEnableDevice unlocks the ONU and its UNI/VEIP ports (admin unlock via OMCI)
-// TODO!!! Clarify usage of this method, compare above DisableDevice, usage may clarify resulting states
-//         maybe it is obsolete by now
 func (dh *deviceHandler) reEnableDevice(device *voltha.Device) {
 	logger.Debugw("reenable-device", log.Fields{"device-id": device.Id, "SerialNumber": device.SerialNumber})
 
@@ -538,12 +536,12 @@
 		logger.Errorw("error-updating-device-state", log.Fields{"device-id": dh.deviceID, "error": err})
 	}
 
-	// TODO!!! DeviceReason to be set here could be more accurate, for now just ...(like python code)
-	if err := dh.coreProxy.DeviceReasonUpdate(context.TODO(), dh.deviceID, "initial-mib-downloaded"); err != nil {
+	// DeviceReason to update acc.to modified py code as per beginning of Sept 2020
+	if err := dh.coreProxy.DeviceReasonUpdate(context.TODO(), dh.deviceID, "onu-reenabled"); err != nil {
 		//TODO with VOL-3045/VOL-3046: return the error and stop further processing
 		logger.Errorw("error-updating-reason-state", log.Fields{"device-id": dh.deviceID, "error": err})
 	}
-	dh.deviceReason = "initial-mib-downloaded"
+	dh.deviceReason = "onu-reenabled"
 
 	// enable ONU/UNI ports
 	// *** should generate UniAdminStateDone event - unrelated to DeviceProcStatusUpdate!!
@@ -1822,12 +1820,8 @@
 		}
 		logger.Debugw("FlowAdd vlan-set", log.Fields{"device-id": dh.deviceID})
 	}
-	//TODO!!: further FlowAdd requests may be valid even in case the FSM is already running,
-	//  e.g. for multi-step flow configuration, error treatment must be redefined in this context as requested in [VOL-3441]
 	if _, exist := dh.UniVlanConfigFsmMap[apUniPort.uniID]; exist {
-		logger.Errorw("FlowAdd aborted - FSM already running", log.Fields{
-			"device-id": dh.deviceID, "UniPort": apUniPort.portNo})
-		return errors.New("flowAdd FSM already running")
+		return dh.UniVlanConfigFsmMap[apUniPort.uniID].SetUniFlowParams(loMatchVlan, loSetVlan, loSetPcp)
 	}
 	return dh.createVlanFilterFsm(apUniPort,
 		loTpID, loMatchVlan, loSetVlan, loSetPcp, OmciVlanFilterDone)
diff --git a/internal/pkg/onuadaptercore/omci_cc.go b/internal/pkg/onuadaptercore/omci_cc.go
index f49a370..06458ee 100644
--- a/internal/pkg/onuadaptercore/omci_cc.go
+++ b/internal/pkg/onuadaptercore/omci_cc.go
@@ -1443,6 +1443,51 @@
 	return nil
 }
 
+func (oo *omciCC) sendSetVtfdVar(ctx context.Context, timeout int, highPrio bool,
+	rxChan chan Message, params ...me.ParamData) *me.ManagedEntity {
+	tid := oo.getNextTid(highPrio)
+	logger.Debugw("send VTFD-Set-msg:", log.Fields{"device-id": oo.deviceID,
+		"SequNo": strconv.FormatInt(int64(tid), 16),
+		"InstId": strconv.FormatInt(int64(params[0].EntityID), 16)})
+
+	meInstance, omciErr := me.NewVlanTaggingFilterData(params[0])
+	if omciErr.GetError() == nil {
+		omciLayer, msgLayer, err := omci.EncodeFrame(meInstance, omci.SetRequestType,
+			omci.TransactionID(tid))
+		if err != nil {
+			logger.Errorw("Cannot encode VTFD for set", log.Fields{
+				"Err": err, "device-id": oo.deviceID})
+			//TODO!!: refactoring improvement requested, here as an example for [VOL-3457]:
+			//  return (dual format) error code that can be used at caller for immediate error treatment
+			//  (relevant to all used sendXX() methods and their error conditions)
+			return nil
+		}
+
+		pkt, err := serializeOmciLayer(omciLayer, msgLayer)
+		if err != nil {
+			logger.Errorw("Cannot serialize VTFD set", log.Fields{
+				"Err": err, "device-id": oo.deviceID})
+			return nil
+		}
+
+		omciRxCallbackPair := callbackPair{
+			cbKey:   tid,
+			cbEntry: callbackPairEntry{rxChan, oo.receiveOmciResponse},
+		}
+		err = oo.send(ctx, pkt, timeout, 0, highPrio, omciRxCallbackPair)
+		if err != nil {
+			logger.Errorw("Cannot send VTFD set", log.Fields{
+				"Err": err, "device-id": oo.deviceID})
+			return nil
+		}
+		logger.Debug("send VTFD-Set-msg done")
+		return meInstance
+	}
+	logger.Errorw("Cannot generate VTFD Instance", log.Fields{
+		"Err": omciErr.GetError(), "device-id": oo.deviceID})
+	return nil
+}
+
 func (oo *omciCC) sendSetEvtocdVar(ctx context.Context, timeout int, highPrio bool,
 	rxChan chan Message, params ...me.ParamData) *me.ManagedEntity {
 	tid := oo.getNextTid(highPrio)
diff --git a/internal/pkg/onuadaptercore/omci_vlan_config.go b/internal/pkg/onuadaptercore/omci_vlan_config.go
index 3421f30..a414186 100644
--- a/internal/pkg/onuadaptercore/omci_vlan_config.go
+++ b/internal/pkg/onuadaptercore/omci_vlan_config.go
@@ -22,6 +22,7 @@
 	"encoding/binary"
 	"errors"
 	"strconv"
+	"sync"
 	"time"
 
 	"github.com/looplab/fsm"
@@ -35,6 +36,7 @@
 	// internal predefined values
 	cDefaultDownstreamMode = 0
 	cDefaultTpid           = 0x8100
+	cMaxAllowedFlows       = 12 //which might be under discussion, for the moment connected to limit of VLAN's within VTFD
 )
 
 const (
@@ -79,11 +81,12 @@
 	vlanEvStartConfig    = "vlanEvStartConfig"
 	vlanEvRxConfigVtfd   = "vlanEvRxConfigVtfd"
 	vlanEvRxConfigEvtocd = "vlanEvRxConfigEvtocd"
+	vlanEvIncrFlowConfig = "vlanEvIncrFlowConfig"
 	//vlanEvCleanupConfig  = "vlanEvCleanupConfig"
-	//vlanEvRxCleanVtfd = "vlanEvRxCleanVtfd"
-	//vlanEvRxCleanEvtocd = "vlanEvRxCleanEvtocd"
-	//vlanEvTimeoutSimple = "vlanEvTimeoutSimple"
-	//vlanEvTimeoutMids = "vlanEvTimeoutMids"
+	//vlanEvRxCleanVtfd    = "vlanEvRxCleanVtfd"
+	//vlanEvRxCleanEvtocd  = "vlanEvRxCleanEvtocd"
+	//vlanEvTimeoutSimple  = "vlanEvTimeoutSimple"
+	//vlanEvTimeoutMids    = "vlanEvTimeoutMids"
 	vlanEvReset   = "vlanEvReset"
 	vlanEvRestart = "vlanEvRestart"
 )
@@ -95,12 +98,22 @@
 	vlanStConfigVtfd      = "vlanStConfigVtfd"
 	vlanStConfigEvtocd    = "vlanStConfigEvtocd"
 	vlanStConfigDone      = "vlanStConfigDone"
+	vlanStConfigIncrFlow  = "vlanStConfigIncrFlow"
 	vlanStCleanEvtocd     = "vlanStCleanEvtocd"
 	vlanStCleanVtfd       = "vlanStCleanVtfd"
 	vlanStCleanupDone     = "vlanStCleanupDone"
 	vlanStResetting       = "vlanStResetting"
 )
 
+type uniVlanFlowParameter struct {
+	//use uint32 types for allowing immediate bitshifting
+	matchVid     uint32
+	matchPcp     uint32
+	tagsToRemove uint32
+	setVid       uint32
+	setPcp       uint32
+}
+
 //UniVlanConfigFsm defines the structure for the state machine to config the PON ANI ports of ONU UNI ports via OMCI
 type UniVlanConfigFsm struct {
 	pDeviceHandler              *deviceHandler
@@ -113,14 +126,14 @@
 	omciMIdsResponseReceived    chan bool //seperate channel needed for checking multiInstance OMCI message responses
 	pAdaptFsm                   *AdapterFsm
 	acceptIncrementalEvtoOption bool
-	//use uint32 types for allowing immediate bitshifting
-	matchVid     uint32
-	matchPcp     uint32
-	tagsToRemove uint32
-	setVid       uint32
-	setPcp       uint32
-	vtfdID       uint16
-	evtocdID     uint16
+	mutexFlowParams             sync.Mutex
+	uniFlowParamsSlice          []uniVlanFlowParameter
+	numUniFlows                 uint8 // expected number of flows should be less than 12
+	configuredUniFlow           uint8
+	numVlanFilterEntries        uint8
+	vlanFilterList              [12]uint16
+	vtfdID                      uint16
+	evtocdID                    uint16
 }
 
 //NewUniVlanConfigFsm is the 'constructor' for the state machine to config the PON ANI ports of ONU UNI ports via OMCI
@@ -137,24 +150,8 @@
 		techProfileID:               aTechProfileID,
 		requestEvent:                aRequestEvent,
 		acceptIncrementalEvtoOption: aAcceptIncrementalEvto,
-		matchVid:                    uint32(aMatchVlan),
-		setVid:                      uint32(aSetVlan),
-		setPcp:                      uint32(aSetPcp),
-	}
-	// some automatic adjustments on the filter/treat parameters as not specifically configured/ensured by flow configuration parameters
-	instFsm.tagsToRemove = 1            //one tag to remove as default setting
-	instFsm.matchPcp = cPrioDoNotFilter // do not Filter on prio as default
-	if instFsm.matchVid == uint32(of.OfpVlanId_OFPVID_PRESENT) {
-		// no prio/vid filtering requested
-		instFsm.tagsToRemove = 0          //no tag pop action
-		instFsm.matchPcp = cPrioIgnoreTag // no vlan tag filtering
-		if instFsm.setPcp == cCopyPrioFromInner {
-			//in case of no filtering and configured PrioCopy ensure default prio setting to 0
-			// which is required for stacking of untagged, but obviously also ensures prio setting for prio/singletagged
-			// might collide with NoMatchVid/CopyPrio(/setVid) setting
-			// this was some precondition setting taken over from py adapter ..
-			instFsm.setPcp = 0
-		}
+		numUniFlows:                 0,
+		configuredUniFlow:           0,
 	}
 
 	instFsm.pAdaptFsm = NewAdapterFsm(aName, aDeviceID, aCommChannel)
@@ -163,7 +160,6 @@
 			"device-id": aDeviceID})
 		return nil
 	}
-
 	instFsm.pAdaptFsm.pFsm = fsm.NewFSM(
 		vlanStDisabled,
 		fsm.Events{
@@ -172,8 +168,9 @@
 			{Name: vlanEvContinueConfig, Src: []string{vlanStWaitingTechProf}, Dst: vlanStConfigVtfd},
 			{Name: vlanEvStartConfig, Src: []string{vlanStStarting}, Dst: vlanStConfigVtfd},
 			{Name: vlanEvRxConfigVtfd, Src: []string{vlanStConfigVtfd}, Dst: vlanStConfigEvtocd},
-			{Name: vlanEvRxConfigEvtocd, Src: []string{vlanStConfigEvtocd}, Dst: vlanStConfigDone},
-			//TODO:!!! Also define state transitions for cleanup states and timeouts
+			{Name: vlanEvRxConfigEvtocd, Src: []string{vlanStConfigEvtocd, vlanStConfigIncrFlow},
+				Dst: vlanStConfigDone},
+			{Name: vlanEvIncrFlowConfig, Src: []string{vlanStConfigDone}, Dst: vlanStConfigIncrFlow},
 			/*
 				{Name: vlanEvTimeoutSimple, Src: []string{
 					vlanStCreatingDot1PMapper, vlanStCreatingMBPCD, vlanStSettingTconts, vlanStSettingDot1PMapper}, Dst: vlanStStarting},
@@ -182,24 +179,24 @@
 			*/
 			// exceptional treatment for all states except vlanStResetting
 			{Name: vlanEvReset, Src: []string{vlanStStarting, vlanStWaitingTechProf,
-				vlanStConfigVtfd, vlanStConfigEvtocd, vlanStConfigDone,
+				vlanStConfigVtfd, vlanStConfigEvtocd, vlanStConfigDone, vlanStConfigIncrFlow,
 				vlanStCleanEvtocd, vlanStCleanVtfd, vlanStCleanupDone},
 				Dst: vlanStResetting},
 			// the only way to get to resource-cleared disabled state again is via "resseting"
 			{Name: vlanEvRestart, Src: []string{vlanStResetting}, Dst: vlanStDisabled},
 		},
-
 		fsm.Callbacks{
-			"enter_state":                   func(e *fsm.Event) { instFsm.pAdaptFsm.logFsmStateChange(e) },
-			("enter_" + vlanStStarting):     func(e *fsm.Event) { instFsm.enterConfigStarting(e) },
-			("enter_" + vlanStConfigVtfd):   func(e *fsm.Event) { instFsm.enterConfigVtfd(e) },
-			("enter_" + vlanStConfigEvtocd): func(e *fsm.Event) { instFsm.enterConfigEvtocd(e) },
-			("enter_" + vlanStConfigDone):   func(e *fsm.Event) { instFsm.enterVlanConfigDone(e) },
-			("enter_" + vlanStCleanVtfd):    func(e *fsm.Event) { instFsm.enterCleanVtfd(e) },
-			("enter_" + vlanStCleanEvtocd):  func(e *fsm.Event) { instFsm.enterCleanEvtocd(e) },
-			("enter_" + vlanStCleanupDone):  func(e *fsm.Event) { instFsm.enterVlanCleanupDone(e) },
-			("enter_" + vlanStResetting):    func(e *fsm.Event) { instFsm.enterResetting(e) },
-			("enter_" + vlanStDisabled):     func(e *fsm.Event) { instFsm.enterDisabled(e) },
+			"enter_state":                     func(e *fsm.Event) { instFsm.pAdaptFsm.logFsmStateChange(e) },
+			("enter_" + vlanStStarting):       func(e *fsm.Event) { instFsm.enterConfigStarting(e) },
+			("enter_" + vlanStConfigVtfd):     func(e *fsm.Event) { instFsm.enterConfigVtfd(e) },
+			("enter_" + vlanStConfigEvtocd):   func(e *fsm.Event) { instFsm.enterConfigEvtocd(e) },
+			("enter_" + vlanStConfigDone):     func(e *fsm.Event) { instFsm.enterVlanConfigDone(e) },
+			("enter_" + vlanStConfigIncrFlow): func(e *fsm.Event) { instFsm.enterConfigIncrFlow(e) },
+			("enter_" + vlanStCleanVtfd):      func(e *fsm.Event) { instFsm.enterCleanVtfd(e) },
+			("enter_" + vlanStCleanEvtocd):    func(e *fsm.Event) { instFsm.enterCleanEvtocd(e) },
+			("enter_" + vlanStCleanupDone):    func(e *fsm.Event) { instFsm.enterVlanCleanupDone(e) },
+			("enter_" + vlanStResetting):      func(e *fsm.Event) { instFsm.enterResetting(e) },
+			("enter_" + vlanStDisabled):       func(e *fsm.Event) { instFsm.enterDisabled(e) },
 		},
 	)
 	if instFsm.pAdaptFsm.pFsm == nil {
@@ -208,14 +205,88 @@
 		return nil
 	}
 
+	_ = instFsm.SetUniFlowParams(aMatchVlan, aSetVlan, aSetPcp)
+
 	logger.Infow("UniVlanConfigFsm created", log.Fields{"device-id": aDeviceID,
-		"accIncrEvto": instFsm.acceptIncrementalEvtoOption,
-		"matchVid":    strconv.FormatInt(int64(instFsm.matchVid), 16),
-		"setVid":      strconv.FormatInt(int64(instFsm.setVid), 16),
-		"setPcp":      instFsm.setPcp})
+		"accIncrEvto": instFsm.acceptIncrementalEvtoOption})
 	return instFsm
 }
 
+//SetUniFlowParams verifies on existence of flow parameters to be configured
+// and appends a new flow if there is space
+func (oFsm *UniVlanConfigFsm) SetUniFlowParams(aMatchVlan uint16, aSetVlan uint16, aSetPcp uint8) error {
+	loFlowParams := uniVlanFlowParameter{
+		matchVid: uint32(aMatchVlan),
+		setVid:   uint32(aSetVlan),
+		setPcp:   uint32(aSetPcp),
+	}
+	// some automatic adjustments on the filter/treat parameters as not specifically configured/ensured by flow configuration parameters
+	loFlowParams.tagsToRemove = 1            //one tag to remove as default setting
+	loFlowParams.matchPcp = cPrioDoNotFilter // do not Filter on prio as default
+
+	if loFlowParams.setVid == uint32(of.OfpVlanId_OFPVID_PRESENT) {
+		//then matchVlan is don't care and should be overwritten to 'transparent' here to avoid unneeded multiple flow entries
+		loFlowParams.matchVid = uint32(of.OfpVlanId_OFPVID_PRESENT)
+		//TODO!!: maybe be needed to be re-checked at flow deletion (but assume all flows are always deleted togehther)
+	} else {
+		if !oFsm.acceptIncrementalEvtoOption {
+			//then matchVlan is don't care and should be overwritten to 'transparent' here to avoid unneeded multiple flow entries
+			loFlowParams.matchVid = uint32(of.OfpVlanId_OFPVID_PRESENT)
+		}
+	}
+
+	if loFlowParams.matchVid == uint32(of.OfpVlanId_OFPVID_PRESENT) {
+		// no prio/vid filtering requested
+		loFlowParams.tagsToRemove = 0          //no tag pop action
+		loFlowParams.matchPcp = cPrioIgnoreTag // no vlan tag filtering
+		if loFlowParams.setPcp == cCopyPrioFromInner {
+			//in case of no filtering and configured PrioCopy ensure default prio setting to 0
+			// which is required for stacking of untagged, but obviously also ensures prio setting for prio/singletagged
+			// might collide with NoMatchVid/CopyPrio(/setVid) setting
+			// this was some precondition setting taken over from py adapter ..
+			loFlowParams.setPcp = 0
+		}
+	}
+	flowEntryMatch := false
+	//mutex protection is required for possible concurrent access to FSM members
+	oFsm.mutexFlowParams.Lock()
+	defer oFsm.mutexFlowParams.Unlock()
+	for _, storedUniFlowParams := range oFsm.uniFlowParamsSlice {
+		if storedUniFlowParams == loFlowParams {
+			flowEntryMatch = true
+			break
+		}
+	}
+	if flowEntryMatch {
+		logger.Debugw("UniVlanConfigFsm flow setting - flow already exists (ignore)", log.Fields{
+			"device-id": oFsm.pAdaptFsm.deviceID})
+	} else {
+		if oFsm.numUniFlows < cMaxAllowedFlows {
+			oFsm.uniFlowParamsSlice = append(oFsm.uniFlowParamsSlice, loFlowParams)
+			oFsm.numUniFlows++
+			logger.Debugw("UniVlanConfigFsm flow added", log.Fields{
+				"matchVid": strconv.FormatInt(int64(loFlowParams.matchVid), 16),
+				"setVid":   strconv.FormatInt(int64(loFlowParams.setVid), 16),
+				"setPcp":   loFlowParams.setPcp, "numberofFlows": oFsm.numUniFlows,
+				"device-id": oFsm.pAdaptFsm.deviceID})
+			pConfigVlanStateBaseFsm := oFsm.pAdaptFsm.pFsm
+			if pConfigVlanStateBaseFsm.Is(vlanStConfigDone) {
+				//have to re-trigger the FSM to proceed with outstanding incremental flow configuration
+				// calling some FSM event must be decoupled
+				go func(a_pBaseFsm *fsm.FSM) {
+					_ = a_pBaseFsm.Event(vlanEvIncrFlowConfig)
+				}(pConfigVlanStateBaseFsm)
+			} // in all other states a new entry will be automatically considered later in that state or
+			// ignored as not anymore relevant
+		} else {
+			logger.Errorw("UniVlanConfigFsm flow limit exceeded", log.Fields{
+				"device-id": oFsm.pAdaptFsm.deviceID})
+			return errors.New(" UniVlanConfigFsm flow limit exceeded")
+		}
+	}
+	return nil
+}
+
 func (oFsm *UniVlanConfigFsm) enterConfigStarting(e *fsm.Event) {
 	logger.Debugw("UniVlanConfigFsm start", log.Fields{"in state": e.FSM.Current(),
 		"device-id": oFsm.pAdaptFsm.deviceID})
@@ -248,8 +319,11 @@
 }
 
 func (oFsm *UniVlanConfigFsm) enterConfigVtfd(e *fsm.Event) {
-	if oFsm.setVid == uint32(of.OfpVlanId_OFPVID_PRESENT) {
+	//mutex protection is required for possible concurrent access to FSM members
+	oFsm.mutexFlowParams.Lock()
+	if oFsm.uniFlowParamsSlice[0].setVid == uint32(of.OfpVlanId_OFPVID_PRESENT) {
 		// meaning transparent setup - no specific VTFD setting required
+		oFsm.mutexFlowParams.Unlock()
 		logger.Debugw("UniVlanConfigFsm: no VTFD config required", log.Fields{
 			"in state": e.FSM.Current(), "device-id": oFsm.pAdaptFsm.deviceID})
 		// let the FSM proceed ... (from within this state all internal pointers may be expected to be correct)
@@ -262,16 +336,21 @@
 		logger.Debugw("UniVlanConfigFsm create VTFD", log.Fields{
 			"EntitytId": strconv.FormatInt(int64(oFsm.vtfdID), 16),
 			"in state":  e.FSM.Current(), "device-id": oFsm.pAdaptFsm.deviceID})
-		vlanFilterList := make([]uint16, 12)
-		vlanFilterList[0] = uint16(oFsm.setVid) // setVid is assumed to be masked already by the caller to 12 bit
+		oFsm.vlanFilterList[0] = uint16(oFsm.uniFlowParamsSlice[0].setVid) // setVid is assumed to be masked already by the caller to 12 bit
+		oFsm.mutexFlowParams.Unlock()
+		vtfdFilterList := make([]uint16, 12) //needed for parameter serialization
+		vtfdFilterList[0] = oFsm.vlanFilterList[0]
+		oFsm.numVlanFilterEntries = 1
 		meParams := me.ParamData{
 			EntityID: oFsm.vtfdID,
 			Attributes: me.AttributeValueMap{
-				"VlanFilterList":   vlanFilterList,
-				"ForwardOperation": uint8(0x10), //VID investigation
-				"NumberOfEntries":  uint8(1),
+				"VlanFilterList":   vtfdFilterList, //omci lib wants a slice for serialization
+				"ForwardOperation": uint8(0x10),    //VID investigation
+				"NumberOfEntries":  oFsm.numVlanFilterEntries,
 			},
 		}
+		logger.Debugw("UniVlanConfigFsm sendcreate VTFD", log.Fields{
+			"in state": e.FSM.Current(), "device-id": oFsm.pAdaptFsm.deviceID})
 		meInstance := oFsm.pOmciCC.sendCreateVtfdVar(context.TODO(), ConstDefaultOmciTimeout, true,
 			oFsm.pAdaptFsm.commChan, meParams)
 		//accept also nil as (error) return value for writing to LastTx
@@ -286,17 +365,114 @@
 func (oFsm *UniVlanConfigFsm) enterConfigEvtocd(e *fsm.Event) {
 	logger.Debugw("UniVlanConfigFsm - start config EVTOCD loop", log.Fields{
 		"in state": e.FSM.Current(), "device-id": oFsm.pAdaptFsm.deviceID})
-	go oFsm.performConfigEvtocdEntries()
+	go oFsm.performConfigEvtocdEntries(0)
 }
 
 func (oFsm *UniVlanConfigFsm) enterVlanConfigDone(e *fsm.Event) {
-	logger.Debugw("UniVlanConfigFsm - VLAN config done: send dh event notification", log.Fields{
+	logger.Debugw("UniVlanConfigFsm - checking on more flows", log.Fields{
 		"in state": e.FSM.Current(), "device-id": oFsm.pAdaptFsm.deviceID})
+	oFsm.configuredUniFlow++ // one (more) flow configured
+	if oFsm.numUniFlows > oFsm.configuredUniFlow {
+		//some further flows are to be configured
+		// calling some FSM event must be decoupled
+		pConfigVlanStateBaseFsm := oFsm.pAdaptFsm.pFsm
+		go func(a_pBaseFsm *fsm.FSM) {
+			_ = a_pBaseFsm.Event(vlanEvIncrFlowConfig)
+		}(pConfigVlanStateBaseFsm)
+		return
+	}
+
+	logger.Debugw("UniVlanConfigFsm - VLAN config done: send dh event notification", log.Fields{
+		"device-id": oFsm.pAdaptFsm.deviceID})
+	// it might appear that some flows are requested also after 'flowPushed' event has been generated ...
+	// state transition notification is checked in deviceHandler
 	if oFsm.pDeviceHandler != nil {
 		oFsm.pDeviceHandler.deviceProcStatusUpdate(oFsm.requestEvent)
 	}
 }
 
+func (oFsm *UniVlanConfigFsm) enterConfigIncrFlow(e *fsm.Event) {
+	logger.Debugw("UniVlanConfigFsm - start config further incremental flow", log.Fields{
+		"in state": e.FSM.Current(), "recent flow-number": (oFsm.configuredUniFlow),
+		"device-id": oFsm.pAdaptFsm.deviceID})
+	oFsm.mutexFlowParams.Lock()
+
+	if oFsm.uniFlowParamsSlice[oFsm.configuredUniFlow].setVid == uint32(of.OfpVlanId_OFPVID_PRESENT) {
+		// meaning transparent setup - no specific VTFD setting required
+		oFsm.mutexFlowParams.Unlock()
+		logger.Debugw("UniVlanConfigFsm: no VTFD config required", log.Fields{
+			"in state": e.FSM.Current(), "device-id": oFsm.pAdaptFsm.deviceID})
+	} else {
+		if oFsm.numVlanFilterEntries == 0 {
+			//no VTFD yet created
+			logger.Debugw("UniVlanConfigFsm create VTFD", log.Fields{
+				"EntitytId": strconv.FormatInt(int64(oFsm.vtfdID), 16),
+				"in state":  e.FSM.Current(), "device-id": oFsm.pAdaptFsm.deviceID})
+			oFsm.vlanFilterList[0] = uint16(oFsm.uniFlowParamsSlice[oFsm.configuredUniFlow].setVid) // setVid is assumed to be masked already by the caller to 12 bit
+			oFsm.mutexFlowParams.Unlock()
+			vtfdFilterList := make([]uint16, 12) //needed for parameter serialization
+			vtfdFilterList[0] = oFsm.vlanFilterList[0]
+			oFsm.numVlanFilterEntries = 1
+			meParams := me.ParamData{
+				EntityID: oFsm.vtfdID,
+				Attributes: me.AttributeValueMap{
+					"VlanFilterList":   vtfdFilterList,
+					"ForwardOperation": uint8(0x10), //VID investigation
+					"NumberOfEntries":  oFsm.numVlanFilterEntries,
+				},
+			}
+			meInstance := oFsm.pOmciCC.sendCreateVtfdVar(context.TODO(), ConstDefaultOmciTimeout, true,
+				oFsm.pAdaptFsm.commChan, meParams)
+			//accept also nil as (error) return value for writing to LastTx
+			//  - this avoids misinterpretation of new received OMCI messages
+			//TODO!!: refactoring improvement requested, here as an example for [VOL-3457]:
+			//  send shall return (dual format) error code that can be used here for immediate error treatment
+			//  (relevant to all used sendXX() methods in this (and other) FSM's)
+			oFsm.pOmciCC.pLastTxMeInstance = meInstance
+		} else {
+			//VTFD already exists - just modify by 'set'
+			//TODO!!: but only if the VID is not already present, skipped by now to test basic working
+			logger.Debugw("UniVlanConfigFsm set VTFD", log.Fields{
+				"EntitytId": strconv.FormatInt(int64(oFsm.vtfdID), 16),
+				"in state":  e.FSM.Current(), "device-id": oFsm.pAdaptFsm.deviceID})
+			// setVid is assumed to be masked already by the caller to 12 bit
+			oFsm.vlanFilterList[oFsm.numVlanFilterEntries] =
+				uint16(oFsm.uniFlowParamsSlice[oFsm.configuredUniFlow].setVid)
+			oFsm.mutexFlowParams.Unlock()
+			vtfdFilterList := make([]uint16, 12) //needed for parameter serialization
+			for i := uint8(0); i <= oFsm.numVlanFilterEntries; i++ {
+				vtfdFilterList[i] = oFsm.vlanFilterList[i]
+			}
+
+			oFsm.numVlanFilterEntries++
+			meParams := me.ParamData{
+				EntityID: oFsm.vtfdID,
+				Attributes: me.AttributeValueMap{
+					"VlanFilterList":  vtfdFilterList,
+					"NumberOfEntries": oFsm.numVlanFilterEntries,
+				},
+			}
+			meInstance := oFsm.pOmciCC.sendSetVtfdVar(context.TODO(), ConstDefaultOmciTimeout, true,
+				oFsm.pAdaptFsm.commChan, meParams)
+			//accept also nil as (error) return value for writing to LastTx
+			//  - this avoids misinterpretation of new received OMCI messages
+			//TODO!!: refactoring improvement requested, here as an example for [VOL-3457]:
+			//  send shall return (dual format) error code that can be used here for immediate error treatment
+			//  (relevant to all used sendXX() methods in this (and other) FSM's)
+			oFsm.pOmciCC.pLastTxMeInstance = meInstance
+		}
+		//verify response
+		err := oFsm.waitforOmciResponse()
+		if err != nil {
+			logger.Errorw("VTFD create/set failed, aborting VlanConfig FSM!",
+				log.Fields{"device-id": oFsm.pAdaptFsm.deviceID})
+			_ = oFsm.pAdaptFsm.pFsm.Event(vlanEvReset)
+			return
+		}
+	}
+	go oFsm.performConfigEvtocdEntries(oFsm.configuredUniFlow)
+}
+
 func (oFsm *UniVlanConfigFsm) enterCleanVtfd(e *fsm.Event) {
 	logger.Debugw("UniVlanConfigFsm Tx Delete::VTFD", log.Fields{
 		/*"EntitytId": strconv.FormatInt(int64(oFsm.mapperSP0ID), 16),*/
@@ -421,8 +597,13 @@
 				//   - might be checked, but so far I go for sure and have to inspect the concrete state events ...
 				switch oFsm.pOmciCC.pLastTxMeInstance.GetName() {
 				case "VlanTaggingFilterData":
-					{ // let the FSM proceed ...
-						_ = oFsm.pAdaptFsm.pFsm.Event(vlanEvRxConfigVtfd)
+					{
+						if oFsm.configuredUniFlow == 0 {
+							// Only if CreateResponse is received from first flow entry - let the FSM proceed ...
+							_ = oFsm.pAdaptFsm.pFsm.Event(vlanEvRxConfigVtfd)
+						} else { // let the MultiEntity config proceed by stopping the wait function
+							oFsm.omciMIdsResponseReceived <- true
+						}
 					}
 				}
 			}
@@ -448,8 +629,9 @@
 			if msgObj.EntityClass == oFsm.pOmciCC.pLastTxMeInstance.GetClassID() &&
 				msgObj.EntityInstance == oFsm.pOmciCC.pLastTxMeInstance.GetEntityID() {
 				switch oFsm.pOmciCC.pLastTxMeInstance.GetName() {
-				case "ExtendedVlanTaggingOperationConfigurationData":
-					{ // let the EVTO config proceed by stopping the wait function
+				case "VlanTaggingFilterData",
+					"ExtendedVlanTaggingOperationConfigurationData":
+					{ // let the MultiEntity config proceed by stopping the wait function
 						oFsm.omciMIdsResponseReceived <- true
 					}
 				}
@@ -463,8 +645,9 @@
 	}
 }
 
-func (oFsm *UniVlanConfigFsm) performConfigEvtocdEntries() {
-	{ // for local var
+func (oFsm *UniVlanConfigFsm) performConfigEvtocdEntries(aFlowEntryNo uint8) {
+	if aFlowEntryNo == 0 {
+		// EthType set only at first flow element
 		// EVTOCD ME is expected to exist at this point already from MIB-Download (with AssociationType/Pointer)
 		// we need to extend the configuration by EthType definition and, to be sure, downstream 'inverse' mode
 		logger.Debugw("UniVlanConfigFsm Tx Set::EVTOCD", log.Fields{
@@ -493,10 +676,12 @@
 			_ = oFsm.pAdaptFsm.pFsm.Event(vlanEvReset)
 			return
 		}
-	} //for local var
+	} //first flow element
 
-	if oFsm.setVid == uint32(of.OfpVlanId_OFPVID_PRESENT) {
+	oFsm.mutexFlowParams.Lock()
+	if oFsm.uniFlowParamsSlice[aFlowEntryNo].setVid == uint32(of.OfpVlanId_OFPVID_PRESENT) {
 		//transparent transmission required
+		oFsm.mutexFlowParams.Unlock()
 		logger.Debugw("UniVlanConfigFsm Tx Set::EVTOCD single tagged transparent rule", log.Fields{
 			"device-id": oFsm.pAdaptFsm.deviceID})
 		sliceEvtocdRule := make([]uint8, 16)
@@ -557,21 +742,22 @@
 					cDoNotFilterTPID<<cFilterTpidOffset) // Do not filter on outer TPID field
 
 			binary.BigEndian.PutUint32(sliceEvtocdRule[cFilterInnerOffset:],
-				oFsm.matchPcp<<cFilterPrioOffset| // either DNFonPrio or ignore tag (default) on innerVLAN
-					oFsm.matchVid<<cFilterVidOffset| // either DNFonVid or real filter VID
+				oFsm.uniFlowParamsSlice[aFlowEntryNo].matchPcp<<cFilterPrioOffset| // either DNFonPrio or ignore tag (default) on innerVLAN
+					oFsm.uniFlowParamsSlice[aFlowEntryNo].matchVid<<cFilterVidOffset| // either DNFonVid or real filter VID
 					cDoNotFilterTPID<<cFilterTpidOffset| // Do not filter on inner TPID field
 					cDoNotFilterEtherType<<cFilterEtherTypeOffset) // Do not filter of EtherType
 
 			binary.BigEndian.PutUint32(sliceEvtocdRule[cTreatOuterOffset:],
-				oFsm.tagsToRemove<<cTreatTTROffset| // either 1 or 0
+				oFsm.uniFlowParamsSlice[aFlowEntryNo].tagsToRemove<<cTreatTTROffset| // either 1 or 0
 					cDoNotAddPrio<<cTreatPrioOffset| // do not add outer tag
 					cDontCareVid<<cTreatVidOffset| // Outer VID don't care
 					cDontCareTpid<<cTreatTpidOffset) // Outer TPID field don't care
 
 			binary.BigEndian.PutUint32(sliceEvtocdRule[cTreatInnerOffset:],
-				oFsm.setPcp<<cTreatPrioOffset| // as configured in flow
-					oFsm.setVid<<cTreatVidOffset| //as configured in flow
+				oFsm.uniFlowParamsSlice[aFlowEntryNo].setPcp<<cTreatPrioOffset| // as configured in flow
+					oFsm.uniFlowParamsSlice[aFlowEntryNo].setVid<<cTreatVidOffset| //as configured in flow
 					cSetOutputTpidCopyDei<<cTreatTpidOffset) // Set TPID = 0x8100
+			oFsm.mutexFlowParams.Unlock()
 
 			meParams := me.ParamData{
 				EntityID: oFsm.evtocdID,
@@ -621,9 +807,10 @@
 				binary.BigEndian.PutUint32(sliceEvtocdRule[cTreatInnerOffset:],
 					0<<cTreatPrioOffset| // vlan prio set to 0
 						//   (as done in Py code, maybe better option would be setPcp here, which still could be 0?)
-						oFsm.setVid<<cTreatVidOffset| // Outer VID don't care
+						oFsm.uniFlowParamsSlice[aFlowEntryNo].setVid<<cTreatVidOffset| // Outer VID don't care
 						cSetOutputTpidCopyDei<<cTreatTpidOffset) // Set TPID = 0x8100
 
+				oFsm.mutexFlowParams.Unlock()
 				meParams := me.ParamData{
 					EntityID: oFsm.evtocdID,
 					Attributes: me.AttributeValueMap{
@@ -644,7 +831,7 @@
 					_ = oFsm.pAdaptFsm.pFsm.Event(vlanEvReset)
 					return
 				}
-			} //just for local var's
+			} // just for local var's
 			{ // just for local var's
 				// this defines 'stacking' scenario: priotagged->singletagged
 				logger.Debugw("UniVlanConfigFsm Tx Set::EVTOCD priotagged->singletagged rule", log.Fields{
@@ -668,11 +855,13 @@
 						cDontCareVid<<cTreatVidOffset| // Outer VID don't care
 						cDontCareTpid<<cTreatTpidOffset) // Outer TPID field don't care
 
+				oFsm.mutexFlowParams.Lock()
 				binary.BigEndian.PutUint32(sliceEvtocdRule[cTreatInnerOffset:],
 					cCopyPrioFromInner<<cTreatPrioOffset| // vlan copy from PrioTag
 						//   (as done in Py code, maybe better option would be setPcp here, which still could be PrioCopy?)
-						oFsm.setVid<<cTreatVidOffset| // Outer VID as configured
+						oFsm.uniFlowParamsSlice[aFlowEntryNo].setVid<<cTreatVidOffset| // Outer VID as configured
 						cSetOutputTpidCopyDei<<cTreatTpidOffset) // Set TPID = 0x8100
+				oFsm.mutexFlowParams.Unlock()
 
 				meParams := me.ParamData{
 					EntityID: oFsm.evtocdID,