internal/pkg/core/device_handler.go

/*
 * Copyright 2020-present Open Networking Foundation
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

//Package core provides the utility for onu devices, flows and statistics
package core

import (
	"context"
	"errors"
	"fmt"
	"strconv"
	"sync"
	"time"

	"github.com/opencord/voltha-openonu-adapter-go/internal/pkg/config"

	"github.com/gogo/protobuf/proto"
	"github.com/looplab/fsm"
	me "github.com/opencord/omci-lib-go/v2/generated"
	"github.com/opencord/voltha-lib-go/v7/pkg/db"
	"github.com/opencord/voltha-lib-go/v7/pkg/events/eventif"
	flow "github.com/opencord/voltha-lib-go/v7/pkg/flows"
	vgrpc "github.com/opencord/voltha-lib-go/v7/pkg/grpc"
	"github.com/opencord/voltha-lib-go/v7/pkg/log"
	platform "github.com/opencord/voltha-lib-go/v7/pkg/platform"
	almgr "github.com/opencord/voltha-openonu-adapter-go/internal/pkg/almgr"
	avcfg "github.com/opencord/voltha-openonu-adapter-go/internal/pkg/avcfg"
	cmn "github.com/opencord/voltha-openonu-adapter-go/internal/pkg/common"
	mib "github.com/opencord/voltha-openonu-adapter-go/internal/pkg/mib"
	otst "github.com/opencord/voltha-openonu-adapter-go/internal/pkg/omcitst"
	pmmgr "github.com/opencord/voltha-openonu-adapter-go/internal/pkg/pmmgr"
	"github.com/opencord/voltha-openonu-adapter-go/internal/pkg/swupg"
	uniprt "github.com/opencord/voltha-openonu-adapter-go/internal/pkg/uniprt"
	"github.com/opencord/voltha-protos/v5/go/common"
	vc "github.com/opencord/voltha-protos/v5/go/common"
	ca "github.com/opencord/voltha-protos/v5/go/core_adapter"
	"github.com/opencord/voltha-protos/v5/go/extension"
	"github.com/opencord/voltha-protos/v5/go/inter_adapter"
	ia "github.com/opencord/voltha-protos/v5/go/inter_adapter"
	of "github.com/opencord/voltha-protos/v5/go/openflow_13"
	"github.com/opencord/voltha-protos/v5/go/openolt"
	oop "github.com/opencord/voltha-protos/v5/go/openolt"
	"github.com/opencord/voltha-protos/v5/go/tech_profile"
	"github.com/opencord/voltha-protos/v5/go/voltha"
)

const (
	//constants for reconcile flow check channel
	cWaitReconcileFlowAbortOnSuccess = 0xFFFD
	cWaitReconcileFlowAbortOnError   = 0xFFFE
	cWaitReconcileFlowNoActivity     = 0xFFFF
)

const (
	// constants for timeouts
	cTimeOutRemoveUpgrade = 1 //for usage in seconds
)

const (
	// dummy constant - irregular value for ConnState - used to avoiding setting this state in the updateDeviceState()
	// should better be defined in voltha protobuf or best solution would be to define an interface to just set the OperState
	// as long as such is not available by the libraries - use this workaround
	connectStatusINVALID = 255 // as long as not used as key in voltha.ConnectStatus_Types_name
)

const (
	// events of Device FSM
	devEvDeviceInit       = "devEvDeviceInit"
	devEvGrpcConnected    = "devEvGrpcConnected"
	devEvGrpcDisconnected = "devEvGrpcDisconnected"
	devEvDeviceUpInd      = "devEvDeviceUpInd"
	devEvDeviceDownInd    = "devEvDeviceDownInd"
)
const (
	// states of Device FSM
	devStNull      = "devStNull"
	devStDown      = "devStDown"
	devStInit      = "devStInit"
	devStConnected = "devStConnected"
	devStUp        = "devStUp"
)

//Event category and subcategory definitions - same as defiend for OLT in eventmgr.go  - should be done more centrally
const (
	pon = voltha.EventSubCategory_PON
	//olt           = voltha.EventSubCategory_OLT
	//ont           = voltha.EventSubCategory_ONT
	//onu           = voltha.EventSubCategory_ONU
	//nni           = voltha.EventSubCategory_NNI
	//service       = voltha.EventCategory_SERVICE
	//security      = voltha.EventCategory_SECURITY
	equipment = voltha.EventCategory_EQUIPMENT
	//processing    = voltha.EventCategory_PROCESSING
	//environment   = voltha.EventCategory_ENVIRONMENT
	//communication = voltha.EventCategory_COMMUNICATION
)

const (
	cEventObjectType = "ONU"
)
const (
	cOnuActivatedEvent = "ONU_ACTIVATED"
)

type omciIdleCheckStruct struct {
	omciIdleCheckFunc func(*deviceHandler, context.Context, cmn.UsedOmciConfigFsms, string) bool
	omciIdleState     string
}

var fsmOmciIdleStateFuncMap = map[cmn.UsedOmciConfigFsms]omciIdleCheckStruct{
	cmn.CUploadFsm:        {(*deviceHandler).isFsmInOmciIdleStateDefault, mib.CMibUlFsmIdleState},
	cmn.CDownloadFsm:      {(*deviceHandler).isFsmInOmciIdleStateDefault, mib.CMibDlFsmIdleState},
	cmn.CUniLockFsm:       {(*deviceHandler).isFsmInOmciIdleStateDefault, uniprt.CUniFsmIdleState},
	cmn.CUniUnLockFsm:     {(*deviceHandler).isFsmInOmciIdleStateDefault, uniprt.CUniFsmIdleState},
	cmn.CAniConfigFsm:     {(*deviceHandler).isAniConfigFsmInOmciIdleState, avcfg.CAniFsmIdleState},
	cmn.CUniVlanConfigFsm: {(*deviceHandler).isUniVlanConfigFsmInOmciIdleState, avcfg.CVlanFsmIdleState},
	cmn.CL2PmFsm:          {(*deviceHandler).isFsmInOmciIdleStateDefault, pmmgr.CL2PmFsmIdleState},
	cmn.COnuUpgradeFsm:    {(*deviceHandler).isFsmInOmciIdleStateDefault, swupg.COnuUpgradeFsmIdleState},
}

const (
	cNoReconciling = iota
	cOnuConfigReconciling
	cSkipOnuConfigReconciling
)

// FlowCb is the flow control block containing flow add/delete information along with a response channel
type FlowCb struct {
	ctx          context.Context // Flow handler context
	addFlow      bool            // if true flow to be added, else removed
	flowItem     *of.OfpFlowStats
	uniPort      *cmn.OnuUniPort
	flowMetaData *of.FlowMetadata
	respChan     *chan error // channel to report the Flow handling error
}

//deviceHandler will interact with the ONU ? device.
type deviceHandler struct {
	DeviceID         string
	DeviceType       string
	adminState       string
	device           *voltha.Device
	logicalDeviceID  string
	ProxyAddressID   string
	ProxyAddressType string
	parentID         string
	ponPortNumber    uint32

	coreClient *vgrpc.Client
	EventProxy eventif.EventProxy

	pmConfigs *voltha.PmConfigs
	config    *config.AdapterFlags

	pOpenOnuAc      *OpenONUAC
	pDeviceStateFsm *fsm.FSM
	//pPonPort        *voltha.Port
	deviceEntrySet    chan bool //channel for DeviceEntry set event
	pOnuOmciDevice    *mib.OnuDeviceEntry
	pOnuTP            *avcfg.OnuUniTechProf
	pOnuMetricsMgr    *pmmgr.OnuMetricsManager
	pAlarmMgr         *almgr.OnuAlarmManager
	pSelfTestHdlr     *otst.SelfTestControlBlock
	exitChannel       chan int
	lockDevice        sync.RWMutex
	pOnuIndication    *oop.OnuIndication
	deviceReason      uint8
	mutexDeviceReason sync.RWMutex
	pLockStateFsm     *uniprt.LockStateFsm
	pUnlockStateFsm   *uniprt.LockStateFsm

	//flowMgr       *OpenOltFlowMgr
	//eventMgr      *OpenOltEventMgr
	//resourceMgr   *rsrcMgr.OpenOltResourceMgr

	//discOnus sync.Map
	//onus     sync.Map
	//portStats          *OpenOltStatisticsMgr
	collectorIsRunning             bool
	mutexCollectorFlag             sync.RWMutex
	stopCollector                  chan bool
	alarmManagerIsRunning          bool
	mutextAlarmManagerFlag         sync.RWMutex
	stopAlarmManager               chan bool
	stopHeartbeatCheck             chan bool
	uniEntityMap                   cmn.OnuUniPortMap
	mutexKvStoreContext            sync.Mutex
	lockVlanConfig                 sync.RWMutex
	lockVlanAdd                    sync.RWMutex
	UniVlanConfigFsmMap            map[uint8]*avcfg.UniVlanConfigFsm
	lockUpgradeFsm                 sync.RWMutex
	pOnuUpradeFsm                  *swupg.OnuUpgradeFsm
	upgradeCanceled                bool
	reconciling                    uint8
	mutexReconcilingFlag           sync.RWMutex
	reconcilingFirstPass           bool
	mutexReconcilingFirstPassFlag  sync.RWMutex
	reconcilingReasonUpdate        bool
	mutexReconcilingReasonUpdate   sync.RWMutex
	chUniVlanConfigReconcilingDone chan uint16 //channel to indicate that VlanConfig reconciling for a specific UNI has been finished
	chReconcilingFinished          chan bool   //channel to indicate that reconciling has been finished
	reconcileExpiryComplete        time.Duration
	reconcileExpiryVlanConfig      time.Duration
	mutexReadyForOmciConfig        sync.RWMutex
	readyForOmciConfig             bool
	deletionInProgress             bool
	mutexDeletionInProgressFlag    sync.RWMutex
	pLastUpgradeImageState         *voltha.ImageState
	upgradeFsmChan                 chan struct{}

	flowCbChan                     []chan FlowCb
	mutexFlowMonitoringRoutineFlag sync.RWMutex
	mutexForDisableDeviceRequested sync.RWMutex
	stopFlowMonitoringRoutine      []chan bool // length of slice equal to number of uni ports
	isFlowMonitoringRoutineActive  []bool      // length of slice equal to number of uni ports
	disableDeviceRequested         bool        // this flag identify ONU received disable request or not
}

//newDeviceHandler creates a new device handler
func newDeviceHandler(ctx context.Context, cc *vgrpc.Client, ep eventif.EventProxy, device *voltha.Device, adapter *OpenONUAC) *deviceHandler {
	var dh deviceHandler
	dh.coreClient = cc
	dh.EventProxy = ep
	dh.config = adapter.config
	cloned := (proto.Clone(device)).(*voltha.Device)
	dh.DeviceID = cloned.Id
	dh.DeviceType = cloned.Type
	dh.adminState = "up"
	dh.device = cloned
	dh.pOpenOnuAc = adapter
	dh.exitChannel = make(chan int, 1)
	dh.lockDevice = sync.RWMutex{}
	dh.deviceEntrySet = make(chan bool, 1)
	dh.collectorIsRunning = false
	dh.stopCollector = make(chan bool, 2)
	dh.alarmManagerIsRunning = false
	dh.stopAlarmManager = make(chan bool, 2)
	dh.stopHeartbeatCheck = make(chan bool, 2)
	//dh.metrics = pmmetrics.NewPmMetrics(cloned.Id, pmmetrics.Frequency(150), pmmetrics.FrequencyOverride(false), pmmetrics.Grouped(false), pmmetrics.Metrics(pmNames))
	//TODO initialize the support classes.
	dh.uniEntityMap = make(map[uint32]*cmn.OnuUniPort)
	dh.lockVlanConfig = sync.RWMutex{}
	dh.lockVlanAdd = sync.RWMutex{}
	dh.lockUpgradeFsm = sync.RWMutex{}
	dh.mutexForDisableDeviceRequested = sync.RWMutex{}
	dh.UniVlanConfigFsmMap = make(map[uint8]*avcfg.UniVlanConfigFsm)
	dh.reconciling = cNoReconciling
	dh.reconcilingReasonUpdate = false
	dh.reconcilingFirstPass = true
	dh.disableDeviceRequested = false
	dh.chReconcilingFinished = make(chan bool)
	dh.reconcileExpiryComplete = adapter.maxTimeoutReconciling //assumption is to have it as duration in s!
	rECSeconds := int(dh.reconcileExpiryComplete / time.Second)
	if rECSeconds < 2 {
		dh.reconcileExpiryComplete = time.Duration(2) * time.Second //ensure a minimum expiry time of 2s for complete reconciling
		rECSeconds = 2
	}
	rEVCSeconds := rECSeconds / 2
	dh.reconcileExpiryVlanConfig = time.Duration(rEVCSeconds) * time.Second //set this duration to some according lower value
	dh.readyForOmciConfig = false
	dh.deletionInProgress = false
	dh.pLastUpgradeImageState = &voltha.ImageState{
		DownloadState: voltha.ImageState_DOWNLOAD_UNKNOWN,
		Reason:        voltha.ImageState_UNKNOWN_ERROR,
		ImageState:    voltha.ImageState_IMAGE_UNKNOWN,
	}
	dh.upgradeFsmChan = make(chan struct{})

	if dh.device.PmConfigs != nil { // can happen after onu adapter restart
		dh.pmConfigs = cloned.PmConfigs
	} /* else {
		// will be populated when onu_metrics_mananger is initialized.
	}*/

	// Device related state machine
	dh.pDeviceStateFsm = fsm.NewFSM(
		devStNull,
		fsm.Events{
			{Name: devEvDeviceInit, Src: []string{devStNull, devStDown}, Dst: devStInit},
			{Name: devEvGrpcConnected, Src: []string{devStInit}, Dst: devStConnected},
			{Name: devEvGrpcDisconnected, Src: []string{devStConnected, devStDown}, Dst: devStInit},
			{Name: devEvDeviceUpInd, Src: []string{devStConnected, devStDown}, Dst: devStUp},
			{Name: devEvDeviceDownInd, Src: []string{devStUp}, Dst: devStDown},
		},
		fsm.Callbacks{
			"before_event":                      func(e *fsm.Event) { dh.logStateChange(ctx, e) },
			("before_" + devEvDeviceInit):       func(e *fsm.Event) { dh.doStateInit(ctx, e) },
			("after_" + devEvDeviceInit):        func(e *fsm.Event) { dh.postInit(ctx, e) },
			("before_" + devEvGrpcConnected):    func(e *fsm.Event) { dh.doStateConnected(ctx, e) },
			("before_" + devEvGrpcDisconnected): func(e *fsm.Event) { dh.doStateInit(ctx, e) },
			("after_" + devEvGrpcDisconnected):  func(e *fsm.Event) { dh.postInit(ctx, e) },
			("before_" + devEvDeviceUpInd):      func(e *fsm.Event) { dh.doStateUp(ctx, e) },
			("before_" + devEvDeviceDownInd):    func(e *fsm.Event) { dh.doStateDown(ctx, e) },
		},
	)

	return &dh
}

// start save the device to the data model
func (dh *deviceHandler) start(ctx context.Context) {
	logger.Debugw(ctx, "starting-device-handler", log.Fields{"device": dh.device, "device-id": dh.DeviceID})
	// Add the initial device to the local model
	logger.Debugw(ctx, "device-handler-started", log.Fields{"device": dh.device})
}

/*
// stop stops the device dh.  Not much to do for now
func (dh *deviceHandler) stop(ctx context.Context) {
	logger.Debug("stopping-device-handler")
	dh.exitChannel <- 1
}
*/

// ##########################################################################################
// deviceHandler methods that implement the adapters interface requests ##### begin #########

//adoptOrReconcileDevice adopts the ONU device
func (dh *deviceHandler) adoptOrReconcileDevice(ctx context.Context, device *voltha.Device) {
	logger.Debugw(ctx, "adopt_or_reconcile_device", log.Fields{"device-id": device.Id, "Address": device.GetHostAndPort()})

	logger.Debugw(ctx, "Device FSM: ", log.Fields{"device-id": device.Id, "state": string(dh.pDeviceStateFsm.Current())})
	if dh.pDeviceStateFsm.Is(devStNull) {
		if err := dh.pDeviceStateFsm.Event(devEvDeviceInit); err != nil {
			logger.Errorw(ctx, "Device FSM: Can't go to state DeviceInit", log.Fields{"device-id": device.Id, "err": err})
		}
		logger.Debugw(ctx, "Device FSM: ", log.Fields{"device-id": device.Id, "state": string(dh.pDeviceStateFsm.Current())})
		// device.PmConfigs is not nil in cases when adapter restarts. We should not re-set the core again.
		if device.PmConfigs == nil {
			// Now, set the initial PM configuration for that device
			if err := dh.updatePMConfigInCore(ctx, dh.pmConfigs); err != nil {
				logger.Errorw(ctx, "error updating pm config to core", log.Fields{"device-id": dh.DeviceID, "err": err})
			}
		}
	} else {
		logger.Debugw(ctx, "AdoptOrReconcileDevice: Agent/device init already done", log.Fields{"device-id": device.Id})
	}

}

func (dh *deviceHandler) handleOMCIIndication(ctx context.Context, msg *ia.OmciMessage) error {
	/* msg print moved symmetrically to omci_cc, if wanted here as additional debug, than perhaps only based on additional debug setting!
	//assuming omci message content is hex coded!
	// with restricted output of 16(?) bytes would be ...omciMsg.Message[:16]
	logger.Debugw(ctx, "inter-adapter-recv-omci", log.Fields{
		"device-id": dh.DeviceID, "RxOmciMessage": hex.EncodeToString(omciMsg.Message)})
	*/
	pDevEntry := dh.GetOnuDeviceEntry(ctx, true)
	if pDevEntry != nil {
		if pDevEntry.PDevOmciCC != nil {
			return pDevEntry.PDevOmciCC.ReceiveMessage(log.WithSpanFromContext(context.TODO(), ctx), msg.Message)
		}
		logger.Debugw(ctx, "omciCC not ready to receive omci messages - incoming omci message ignored", log.Fields{"device-id": dh.DeviceID,
			"rxMsg": msg.Message})
	}
	logger.Errorw(ctx, "No valid OnuDevice -aborting", log.Fields{"device-id": dh.DeviceID})
	return fmt.Errorf("no valid OnuDevice: %s", dh.DeviceID)
}

func (dh *deviceHandler) handleTechProfileDownloadRequest(ctx context.Context, techProfMsg *ia.TechProfileDownloadMessage) error {
	logger.Infow(ctx, "tech-profile-download-request", log.Fields{"device-id": dh.DeviceID})

	pDevEntry := dh.GetOnuDeviceEntry(ctx, true)
	if pDevEntry == nil {
		logger.Errorw(ctx, "No valid OnuDevice - aborting", log.Fields{"device-id": dh.DeviceID})
		return fmt.Errorf("no valid OnuDevice: %s", dh.DeviceID)
	}
	if dh.pOnuTP == nil {
		//should normally not happen ...
		logger.Errorw(ctx, "onuTechProf instance not set up for DLMsg request - ignoring request",
			log.Fields{"device-id": dh.DeviceID})
		return fmt.Errorf("techProfile DLMsg request while onuTechProf instance not setup: %s", dh.DeviceID)
	}
	if !dh.IsReadyForOmciConfig() {
		logger.Errorw(ctx, "TechProf-set rejected: improper device state", log.Fields{"device-id": dh.DeviceID,
			"device-state": dh.GetDeviceReasonString()})
		return fmt.Errorf("improper device state %s on device %s", dh.GetDeviceReasonString(), dh.DeviceID)
	}
	//previous state test here was just this one, now extended for more states to reject the SetRequest:
	// at least 'mib-downloaded' should be reached for processing of this specific ONU configuration
	//  if (dh.deviceReason == "stopping-openomci") || (dh.deviceReason == "omci-admin-lock")

	// we have to lock access to TechProfile processing based on different messageType calls or
	// even to fast subsequent calls of the same messageType as well as OnuKVStore processing due
	// to possible concurrent access by flow processing
	dh.pOnuTP.LockTpProcMutex()
	defer dh.pOnuTP.UnlockTpProcMutex()

	if techProfMsg.UniId >= platform.MaxUnisPerOnu {
		return fmt.Errorf(fmt.Sprintf("received UniId value exceeds range: %d, device-id: %s",
			techProfMsg.UniId, dh.DeviceID))
	}
	uniID := uint8(techProfMsg.UniId)
	tpID, err := cmn.GetTpIDFromTpPath(techProfMsg.TpInstancePath)
	if err != nil {
		logger.Errorw(ctx, "error-parsing-tpid-from-tppath",
			log.Fields{"device-id": dh.DeviceID, "err": err, "tp-path": techProfMsg.TpInstancePath})
		return err
	}
	logger.Debugw(ctx, "unmarshal-techprof-msg-body", log.Fields{"device-id": dh.DeviceID,
		"uniID": uniID, "tp-path": techProfMsg.TpInstancePath, "tpID": tpID})

	if bTpModify := pDevEntry.UpdateOnuUniTpPath(ctx, uniID, uint8(tpID), techProfMsg.TpInstancePath); bTpModify {

		switch tpInst := techProfMsg.TechTpInstance.(type) {
		case *ia.TechProfileDownloadMessage_TpInstance:
			logger.Debugw(ctx, "onu-uni-tp-path-modified", log.Fields{"device-id": dh.DeviceID,
				"uniID": uniID, "tp-path": techProfMsg.TpInstancePath, "tpID": tpID})
			//	if there has been some change for some uni TechProfilePath
			//in order to allow concurrent calls to other dh instances we do not wait for execution here
			//but doing so we can not indicate problems to the caller (who does what with that then?)
			//by now we just assume straightforward successful execution
			//TODO!!! Generally: In this scheme it would be good to have some means to indicate
			//  possible problems to the caller later autonomously

			// deadline context to ensure completion of background routines waited for
			//20200721: 10s proved to be less in 8*8 ONU test on local vbox machine with debug, might be further adapted
			deadline := time.Now().Add(dh.pOpenOnuAc.maxTimeoutInterAdapterComm) //allowed run time to finish before execution
			dctx, cancel := context.WithDeadline(context.Background(), deadline)

			dh.pOnuTP.ResetTpProcessingErrorIndication(uniID, tpID)

			var wg sync.WaitGroup
			wg.Add(1) // for the 1 go routine to finish
			// attention: deadline completion check and wg.Done is to be done in both routines
			go dh.pOnuTP.ConfigureUniTp(log.WithSpanFromContext(dctx, ctx), uniID, techProfMsg.TpInstancePath, *tpInst.TpInstance, &wg)
			dh.waitForCompletion(ctx, cancel, &wg, "TechProfDwld") //wait for background process to finish
			if tpErr := dh.pOnuTP.GetTpProcessingErrorIndication(uniID, tpID); tpErr != nil {
				logger.Errorw(ctx, "error-processing-tp", log.Fields{"device-id": dh.DeviceID, "err": tpErr, "tp-path": techProfMsg.TpInstancePath})
				return tpErr
			}
			deadline = time.Now().Add(dh.pOpenOnuAc.maxTimeoutInterAdapterComm) //allowed run time to finish before execution
			dctx2, cancel2 := context.WithDeadline(context.Background(), deadline)
			pDevEntry.ResetKvProcessingErrorIndication()
			wg.Add(1) // for the 1 go routine to finish
			go pDevEntry.UpdateOnuKvStore(log.WithSpanFromContext(dctx2, ctx), &wg)
			dh.waitForCompletion(ctx, cancel2, &wg, "TechProfDwld") //wait for background process to finish
			if kvErr := pDevEntry.GetKvProcessingErrorIndication(); kvErr != nil {
				logger.Errorw(ctx, "error-updating-KV", log.Fields{"device-id": dh.DeviceID, "err": kvErr, "tp-path": techProfMsg.TpInstancePath})
				return kvErr
			}
			return nil
		default:
			logger.Errorw(ctx, "unsupported-tp-instance-type", log.Fields{"device-id": dh.DeviceID, "tp-path": techProfMsg.TpInstancePath})
			return fmt.Errorf("unsupported-tp-instance-type--tp-id-%v", techProfMsg.TpInstancePath)
		}
	}
	// no change, nothing really to do - return success
	logger.Debugw(ctx, "onu-uni-tp-path-not-modified", log.Fields{"device-id": dh.DeviceID,
		"uniID": uniID, "tp-path": techProfMsg.TpInstancePath, "tpID": tpID})
	return nil
}

func (dh *deviceHandler) handleDeleteGemPortRequest(ctx context.Context, delGemPortMsg *ia.DeleteGemPortMessage) error {
	logger.Infow(ctx, "delete-gem-port-request start", log.Fields{"device-id": dh.DeviceID})

	if dh.pOnuTP == nil {
		//should normally not happen ...
		logger.Warnw(ctx, "onuTechProf instance not set up for DelGem request - ignoring request",
			log.Fields{"device-id": dh.DeviceID})
		return fmt.Errorf("techProfile DelGem request while onuTechProf instance not setup: %s", dh.DeviceID)
	}
	//compare TECH_PROFILE_DOWNLOAD_REQUEST
	dh.pOnuTP.LockTpProcMutex()
	defer dh.pOnuTP.UnlockTpProcMutex()

	if delGemPortMsg.UniId >= platform.MaxUnisPerOnu {
		logger.Errorw(ctx, "delete-gem-port UniId exceeds range", log.Fields{
			"device-id": dh.DeviceID, "uni-id": delGemPortMsg.UniId})
		return fmt.Errorf(fmt.Sprintf("received UniId value exceeds range: %d, device-id: %s",
			delGemPortMsg.UniId, dh.DeviceID))
	}
	uniID := uint8(delGemPortMsg.UniId)
	tpID, err := cmn.GetTpIDFromTpPath(delGemPortMsg.TpInstancePath)
	if err != nil {
		logger.Errorw(ctx, "error-extracting-tp-id-from-tp-path", log.Fields{
			"device-id": dh.DeviceID, "err": err, "tp-path": delGemPortMsg.TpInstancePath})
		return err
	}
	logger.Infow(ctx, "delete-gem-port-request", log.Fields{
		"device-id": dh.DeviceID, "uni-id": uniID, "tpID": tpID, "gem": delGemPortMsg.GemPortId})
	//a removal of some GemPort would never remove the complete TechProfile entry (done on T-Cont)

	return dh.deleteTechProfileResource(ctx, uniID, tpID, delGemPortMsg.TpInstancePath,
		avcfg.CResourceGemPort, delGemPortMsg.GemPortId)

}

func (dh *deviceHandler) handleDeleteTcontRequest(ctx context.Context, delTcontMsg *ia.DeleteTcontMessage) error {
	logger.Infow(ctx, "delete-tcont-request start", log.Fields{"device-id": dh.DeviceID, "uni-id": delTcontMsg.UniId, "tcont": delTcontMsg.AllocId})

	pDevEntry := dh.GetOnuDeviceEntry(ctx, true)
	if pDevEntry == nil {
		logger.Errorw(ctx, "No valid OnuDevice - aborting", log.Fields{"device-id": dh.DeviceID})
		return fmt.Errorf("no valid OnuDevice: %s", dh.DeviceID)
	}
	if dh.pOnuTP == nil {
		//should normally not happen ...
		logger.Warnw(ctx, "onuTechProf instance not set up for DelTcont request - ignoring request",
			log.Fields{"device-id": dh.DeviceID})
		return fmt.Errorf("techProfile DelTcont request while onuTechProf instance not setup: %s", dh.DeviceID)
	}

	//compare TECH_PROFILE_DOWNLOAD_REQUEST
	dh.pOnuTP.LockTpProcMutex()
	defer dh.pOnuTP.UnlockTpProcMutex()

	if delTcontMsg.UniId >= platform.MaxUnisPerOnu {
		logger.Errorw(ctx, "delete-tcont UniId exceeds range", log.Fields{
			"device-id": dh.DeviceID, "uni-id": delTcontMsg.UniId})
		return fmt.Errorf(fmt.Sprintf("received UniId value exceeds range: %d, device-id: %s",
			delTcontMsg.UniId, dh.DeviceID))
	}
	uniID := uint8(delTcontMsg.UniId)
	tpPath := delTcontMsg.TpInstancePath
	tpID, err := cmn.GetTpIDFromTpPath(tpPath)
	if err != nil {
		logger.Errorw(ctx, "error-extracting-tp-id-from-tp-path", log.Fields{
			"device-id": dh.DeviceID, "err": err, "tp-path": tpPath})
		return err
	}
	pDevEntry.FreeTcont(ctx, uint16(delTcontMsg.AllocId))

	var wg sync.WaitGroup
	deadline := time.Now().Add(dh.pOpenOnuAc.maxTimeoutInterAdapterComm) //allowed run time to finish before execution
	dctx, cancel := context.WithDeadline(context.Background(), deadline)
	wg.Add(1)
	logger.Debugw(ctx, "remove-tcont-in-kv", log.Fields{"device-id": dh.DeviceID, "uni-id": uniID, "tpID": tpID, "tcont": delTcontMsg.AllocId})
	go pDevEntry.UpdateOnuKvStore(log.WithSpanFromContext(dctx, ctx), &wg)
	dh.waitForCompletion(ctx, cancel, &wg, "DeleteTcont") //wait for background process to finish
	if err := pDevEntry.GetKvProcessingErrorIndication(); err != nil {
		logger.Errorw(ctx, err.Error(), log.Fields{"device-id": dh.DeviceID})
		return err
	}

	return dh.deleteTechProfileResource(ctx, uniID, tpID, delTcontMsg.TpInstancePath,
		avcfg.CResourceTcont, delTcontMsg.AllocId)

}

func (dh *deviceHandler) deleteTechProfileResource(ctx context.Context,
	uniID uint8, tpID uint8, pathString string, resource avcfg.ResourceEntry, entryID uint32) error {
	pDevEntry := dh.GetOnuDeviceEntry(ctx, true)
	if pDevEntry == nil {
		logger.Errorw(ctx, "No valid OnuDevice - aborting", log.Fields{"device-id": dh.DeviceID})
		return fmt.Errorf("no valid OnuDevice: %s", dh.DeviceID)
	}
	var resourceName string
	if avcfg.CResourceGemPort == resource {
		resourceName = "Gem"
	} else {
		resourceName = "Tcont"
	}

	// deadline context to ensure completion of background routines waited for
	deadline := time.Now().Add(dh.pOpenOnuAc.maxTimeoutInterAdapterComm) //allowed run time to finish before execution
	dctx, cancel := context.WithDeadline(context.Background(), deadline)

	dh.pOnuTP.ResetTpProcessingErrorIndication(uniID, tpID)

	var wg sync.WaitGroup
	wg.Add(1) // for the 1 go routine to finish
	go dh.pOnuTP.DeleteTpResource(log.WithSpanFromContext(dctx, ctx), uniID, tpID, pathString,
		resource, entryID, &wg)
	dh.waitForCompletion(ctx, cancel, &wg, resourceName+"Delete") //wait for background process to finish
	if err := dh.pOnuTP.GetTpProcessingErrorIndication(uniID, tpID); err != nil {
		logger.Errorw(ctx, err.Error(), log.Fields{"device-id": dh.DeviceID})
		return err
	}

	if dh.pOnuTP.IsTechProfileConfigCleared(ctx, uniID, tpID) {
		logger.Debugw(ctx, "techProfile-config-cleared", log.Fields{"device-id": dh.DeviceID, "uni-id": uniID, "tpID": tpID})
		if bTpModify := pDevEntry.UpdateOnuUniTpPath(ctx, uniID, tpID, ""); bTpModify {
			pDevEntry.ResetKvProcessingErrorIndication()
			var wg2 sync.WaitGroup
			dctx2, cancel2 := context.WithDeadline(context.Background(), deadline)
			wg2.Add(1)
			// Removal of the gem id mapping represents the removal of the tech profile
			logger.Debugw(ctx, "remove-techProfile-indication-in-kv", log.Fields{"device-id": dh.DeviceID, "uni-id": uniID, "tpID": tpID})
			go pDevEntry.UpdateOnuKvStore(log.WithSpanFromContext(dctx2, ctx), &wg2)
			dh.waitForCompletion(ctx, cancel2, &wg2, "TechProfileDeleteOn"+resourceName) //wait for background process to finish
			if err := pDevEntry.GetKvProcessingErrorIndication(); err != nil {
				logger.Errorw(ctx, err.Error(), log.Fields{"device-id": dh.DeviceID})
				return err
			}
		}
	}
	logger.Debugw(ctx, "delete-tech-profile-resource-completed", log.Fields{"device-id": dh.DeviceID,
		"uni-id": uniID, "tpID": tpID, "resource-type": resourceName, "resource-id": entryID})
	return nil
}

//FlowUpdateIncremental removes and/or adds the flow changes on a given device
func (dh *deviceHandler) FlowUpdateIncremental(ctx context.Context,
	apOfFlowChanges *of.FlowChanges,
	apOfGroupChanges *of.FlowGroupChanges, apFlowMetaData *of.FlowMetadata) error {
	logger.Debugw(ctx, "FlowUpdateIncremental started", log.Fields{"device-id": dh.DeviceID, "flow": apOfFlowChanges, "metadata": apFlowMetaData})
	var errorsList []error
	var retError error
	//Remove flows (always remove flows first - remove old and add new with same cookie may be part of the same request)
	if apOfFlowChanges.ToRemove != nil {
		for _, flowItem := range apOfFlowChanges.ToRemove.Items {
			if flowItem.GetCookie() == 0 {
				logger.Warnw(ctx, "flow-remove no cookie: ignore and continuing on checking further flows", log.Fields{
					"device-id": dh.DeviceID})
				retError = fmt.Errorf("flow-remove no cookie, device-id %s", dh.DeviceID)
				errorsList = append(errorsList, retError)
				continue
			}
			flowInPort := flow.GetInPort(flowItem)
			if flowInPort == uint32(of.OfpPortNo_OFPP_INVALID) {
				logger.Warnw(ctx, "flow-remove inPort invalid: ignore and continuing on checking further flows", log.Fields{"device-id": dh.DeviceID})
				retError = fmt.Errorf("flow-remove inPort invalid, device-id %s", dh.DeviceID)
				errorsList = append(errorsList, retError)
				continue
				//return fmt.Errorf("flow inPort invalid: %s", dh.DeviceID)
			} else if flowInPort == dh.ponPortNumber {
				//this is some downstream flow, not regarded as error, just ignored
				logger.Debugw(ctx, "flow-remove for downstream: ignore and continuing on checking further flows", log.Fields{
					"device-id": dh.DeviceID, "inPort": flowInPort})
				continue
			} else {
				// this is the relevant upstream flow
				var loUniPort *cmn.OnuUniPort
				if uniPort, exist := dh.uniEntityMap[flowInPort]; exist {
					loUniPort = uniPort
				} else {
					logger.Warnw(ctx, "flow-remove inPort not found in UniPorts: ignore and continuing on checking further flows",
						log.Fields{"device-id": dh.DeviceID, "inPort": flowInPort})
					retError = fmt.Errorf("flow-remove inPort not found in UniPorts, inPort %d, device-id %s",
						flowInPort, dh.DeviceID)
					errorsList = append(errorsList, retError)
					continue
				}
				flowOutPort := flow.GetOutPort(flowItem)
				logger.Debugw(ctx, "flow-remove port indications", log.Fields{
					"device-id": dh.DeviceID, "inPort": flowInPort, "outPort": flowOutPort,
					"uniPortName": loUniPort.Name})

				if dh.GetFlowMonitoringIsRunning(loUniPort.UniID) {
					// Step1 : Fill flowControlBlock
					// Step2 : Push the flowControlBlock to ONU channel
					// Step3 : Wait on response channel for response
					// Step4 : Return error value
					startTime := time.Now()
					respChan := make(chan error)
					flowCb := FlowCb{
						ctx:          ctx,
						addFlow:      false,
						flowItem:     flowItem,
						flowMetaData: nil,
						uniPort:      loUniPort,
						respChan:     &respChan,
					}
					dh.flowCbChan[loUniPort.UniID] <- flowCb
					logger.Infow(ctx, "process-flow-remove-start", log.Fields{"device-id": dh.DeviceID})
					// Wait on the channel for flow handlers return value
					retError = <-respChan
					logger.Infow(ctx, "process-flow-remove-end", log.Fields{"device-id": dh.DeviceID, "err": retError, "totalTimeSeconds": time.Since(startTime).Seconds()})
					if retError != nil {
						logger.Warnw(ctx, "flow-delete processing error: continuing on checking further flows",
							log.Fields{"device-id": dh.DeviceID, "error": retError})
						errorsList = append(errorsList, retError)
						continue
					}
				} else {
					retError = fmt.Errorf("flow-handler-routine-not-active-for-onu--device-id-%v", dh.DeviceID)
					errorsList = append(errorsList, retError)
				}
			}
		}
	}
	if apOfFlowChanges.ToAdd != nil {
		for _, flowItem := range apOfFlowChanges.ToAdd.Items {
			if flowItem.GetCookie() == 0 {
				logger.Debugw(ctx, "incremental flow-add no cookie: ignore and continuing on checking further flows", log.Fields{
					"device-id": dh.DeviceID})
				retError = fmt.Errorf("flow-add no cookie, device-id %s", dh.DeviceID)
				errorsList = append(errorsList, retError)
				continue
			}
			flowInPort := flow.GetInPort(flowItem)
			if flowInPort == uint32(of.OfpPortNo_OFPP_INVALID) {
				logger.Warnw(ctx, "flow-add inPort invalid: ignore and continuing on checking further flows", log.Fields{"device-id": dh.DeviceID})
				retError = fmt.Errorf("flow-add inPort invalid, device-id %s", dh.DeviceID)
				errorsList = append(errorsList, retError)
				continue
				//return fmt.Errorf("flow inPort invalid: %s", dh.DeviceID)
			} else if flowInPort == dh.ponPortNumber {
				//this is some downstream flow
				logger.Debugw(ctx, "flow-add for downstream: ignore and continuing on checking further flows", log.Fields{
					"device-id": dh.DeviceID, "inPort": flowInPort})
				continue
			} else {
				// this is the relevant upstream flow
				var loUniPort *cmn.OnuUniPort
				if uniPort, exist := dh.uniEntityMap[flowInPort]; exist {
					loUniPort = uniPort
				} else {
					logger.Warnw(ctx, "flow-add inPort not found in UniPorts: ignore and continuing on checking further flows",
						log.Fields{"device-id": dh.DeviceID, "inPort": flowInPort})
					retError = fmt.Errorf("flow-add inPort not found in UniPorts, inPort %d, device-id %s",
						flowInPort, dh.DeviceID)
					errorsList = append(errorsList, retError)
					continue
				}
				// let's still assume that we receive the flow-add only in some 'active' device state (as so far observed)
				// if not, we just throw some error here to have an indication about that, if we really need to support that
				//   then we would need to create some means to activate the internal stored flows
				//   after the device gets active automatically (and still with its dependency to the TechProfile)
				// for state checking compare also code here: processInterAdapterTechProfileDownloadReqMessage
				// also abort for the other still possible flows here
				if !dh.IsReadyForOmciConfig() {
					logger.Errorw(ctx, "flow-add rejected: improper device state", log.Fields{"device-id": dh.DeviceID,
						"last device-reason": dh.GetDeviceReasonString()})
					retError = fmt.Errorf("improper device state on device %s", dh.DeviceID)
					errorsList = append(errorsList, retError)
					continue
				}

				flowOutPort := flow.GetOutPort(flowItem)
				logger.Debugw(ctx, "flow-add port indications", log.Fields{
					"device-id": dh.DeviceID, "inPort": flowInPort, "outPort": flowOutPort,
					"uniPortName": loUniPort.Name})
				if dh.GetFlowMonitoringIsRunning(loUniPort.UniID) {
					// Step1 : Fill flowControlBlock
					// Step2 : Push the flowControlBlock to ONU channel
					// Step3 : Wait on response channel for response
					// Step4 : Return error value
					startTime := time.Now()
					respChan := make(chan error)
					flowCb := FlowCb{
						ctx:          ctx,
						addFlow:      true,
						flowItem:     flowItem,
						flowMetaData: apFlowMetaData,
						uniPort:      loUniPort,
						respChan:     &respChan,
					}
					dh.flowCbChan[loUniPort.UniID] <- flowCb
					logger.Infow(ctx, "process-flow-add-start", log.Fields{"device-id": dh.DeviceID})
					// Wait on the channel for flow handlers return value
					retError = <-respChan
					logger.Infow(ctx, "process-flow-add-end", log.Fields{"device-id": dh.DeviceID, "err": retError, "totalTimeSeconds": time.Since(startTime).Seconds()})
					if retError != nil {
						logger.Warnw(ctx, "flow-add processing error: continuing on checking further flows",
							log.Fields{"device-id": dh.DeviceID, "error": retError})
						errorsList = append(errorsList, retError)
						continue
					}
				} else {
					retError = fmt.Errorf("flow-handler-routine-not-active-for-onu--device-id-%v", dh.DeviceID)
					errorsList = append(errorsList, retError)
				}
			}
		}
	}
	if len(errorsList) > 0 {
		logger.Errorw(ctx, "error-processing-flow", log.Fields{"device-id": dh.DeviceID, "errList": errorsList})
		return fmt.Errorf("errors-installing-one-or-more-flows-groups, errors:%v", errorsList)
	}
	return nil
}

//disableDevice locks the ONU and its UNI/VEIP ports (admin lock via OMCI)
//following are the expected device states after this activity:
//Device Admin-State : down (on rwCore), Port-State: UNKNOWN, Conn-State: REACHABLE, Reason: omci-admin-lock
// (Conn-State: REACHABLE might conflict with some previous ONU Down indication - maybe to be resolved later)
func (dh *deviceHandler) disableDevice(ctx context.Context, device *voltha.Device) {
	logger.Debugw(ctx, "disable-device", log.Fields{"device-id": device.Id, "SerialNumber": device.SerialNumber})
	dh.mutexForDisableDeviceRequested.Lock()
	dh.disableDeviceRequested = true
	dh.mutexForDisableDeviceRequested.Unlock()
	//admin-lock reason can also be used uniquely for setting the DeviceState accordingly
	//note that disableDevice sequences in some 'ONU active' state may yield also
	// "tech...delete-success" or "omci-flow-deleted" according to further received requests in the end
	// - inblock state checking to prevent possibly unneeded processing (on command repitition)
	if dh.getDeviceReason() != cmn.DrOmciAdminLock {
		//disable-device shall be just a UNi/ONU-G related admin state setting
		//all other configurations/FSM's shall not be impacted and shall execute as required by the system

		if dh.IsReadyForOmciConfig() {
			// disable UNI ports/ONU
			// *** should generate UniDisableStateDone event - used to disable the port(s) on success
			if dh.pLockStateFsm == nil {
				dh.createUniLockFsm(ctx, true, cmn.UniDisableStateDone)
			} else { //LockStateFSM already init
				dh.pLockStateFsm.SetSuccessEvent(cmn.UniDisableStateDone)
				dh.runUniLockFsm(ctx, true)
			}
		} else {
			logger.Debugw(ctx, "DeviceStateUpdate upon disable", log.Fields{
				"OperStatus": voltha.OperStatus_UNKNOWN, "device-id": dh.DeviceID})
			// disable device should have no impact on ConnStatus
			if err := dh.updateDeviceStateInCore(ctx, &ca.DeviceStateFilter{
				DeviceId:   dh.DeviceID,
				ConnStatus: connectStatusINVALID, //use some dummy value to prevent modification of the ConnStatus
				OperStatus: voltha.OperStatus_UNKNOWN,
			}); err != nil {
				//TODO with VOL-3045/VOL-3046: return the error and stop further processing
				logger.Errorw(ctx, "error-updating-device-state", log.Fields{"device-id": dh.DeviceID, "error": err})
			}
			// DeviceReason to update acc.to modified py code as per beginning of Sept 2020

			//TODO with VOL-3045/VOL-3046: catch and return error, valid for all occurrences in the codebase
			_ = dh.ReasonUpdate(ctx, cmn.DrOmciAdminLock, true)
		}
	}
}

//reEnableDevice unlocks the ONU and its UNI/VEIP ports (admin unlock via OMCI)
func (dh *deviceHandler) reEnableDevice(ctx context.Context, device *voltha.Device) {
	logger.Debugw(ctx, "reenable-device", log.Fields{"device-id": device.Id, "SerialNumber": device.SerialNumber})

	//setting readyForOmciConfig here is just a workaround for BBSIM testing in the sequence
	//  OnuSoftReboot-disable-enable, because BBSIM does not generate a new OnuIndication-Up event after SoftReboot
	//  which is the assumption for real ONU's, where the ready-state is then set according to the following MibUpload/Download
	//  for real ONU's that should have nearly no influence
	//  Note that for real ONU's there is anyway a problematic situation with following sequence:
	//		OnuIndication-Dw (or not active at all) (- disable) - enable: here already the LockFsm may run into timeout (no OmciResponse)
	//      but that anyway is hopefully resolved by some OnuIndication-Up event (maybe to be tested)
	//      one could also argue, that a device-enable should also enable attempts for specific omci configuration
	dh.SetReadyForOmciConfig(true) //needed to allow subsequent flow/techProf config (on BBSIM)

	// enable ONU/UNI ports
	// *** should generate cmn.UniEnableStateDone event - used to disable the port(s) on success
	dh.mutexForDisableDeviceRequested.Lock()
	dh.disableDeviceRequested = false
	dh.mutexForDisableDeviceRequested.Unlock()
	if dh.pUnlockStateFsm == nil {
		dh.createUniLockFsm(ctx, false, cmn.UniEnableStateDone)
	} else { //UnlockStateFSM already init
		dh.pUnlockStateFsm.SetSuccessEvent(cmn.UniEnableStateDone)
		dh.runUniLockFsm(ctx, false)
	}
}

func (dh *deviceHandler) reconcileDeviceOnuInd(ctx context.Context) {
	logger.Debugw(ctx, "reconciling - simulate onu indication", log.Fields{"device-id": dh.DeviceID})

	pDevEntry := dh.GetOnuDeviceEntry(ctx, true)
	if pDevEntry == nil {
		logger.Errorw(ctx, "No valid OnuDevice - aborting", log.Fields{"device-id": dh.DeviceID})
		return
	}
	if err := pDevEntry.RestoreDataFromOnuKvStore(log.WithSpanFromContext(context.TODO(), ctx)); err != nil {
		if err == fmt.Errorf("no-ONU-data-found") {
			logger.Debugw(ctx, "no persistent data found - abort reconciling", log.Fields{"device-id": dh.DeviceID})
		} else {
			logger.Errorw(ctx, "reconciling - restoring OnuTp-data failed - abort", log.Fields{"err": err, "device-id": dh.DeviceID})
		}
		dh.stopReconciling(ctx, false, cWaitReconcileFlowNoActivity)
		return
	}
	var onuIndication oop.OnuIndication
	pDevEntry.MutexPersOnuConfig.RLock()
	onuIndication.IntfId = pDevEntry.SOnuPersistentData.PersIntfID
	onuIndication.OnuId = pDevEntry.SOnuPersistentData.PersOnuID
	onuIndication.OperState = pDevEntry.SOnuPersistentData.PersOperState
	onuIndication.AdminState = pDevEntry.SOnuPersistentData.PersAdminState
	pDevEntry.MutexPersOnuConfig.RUnlock()
	_ = dh.createInterface(ctx, &onuIndication)
}

func (dh *deviceHandler) ReconcileDeviceTechProf(ctx context.Context) bool {
	logger.Debugw(ctx, "reconciling - trigger tech profile config", log.Fields{"device-id": dh.DeviceID})

	continueWithFlowConfig := false

	pDevEntry := dh.GetOnuDeviceEntry(ctx, true)
	if pDevEntry == nil {
		logger.Errorw(ctx, "reconciling - no valid OnuDevice - aborting", log.Fields{"device-id": dh.DeviceID})
		dh.stopReconciling(ctx, false, cWaitReconcileFlowNoActivity)
		return continueWithFlowConfig
	}
	dh.pOnuTP.LockTpProcMutex()
	defer dh.pOnuTP.UnlockTpProcMutex()

	pDevEntry.MutexPersOnuConfig.RLock()
	persMutexLock := true
	if len(pDevEntry.SOnuPersistentData.PersUniConfig) == 0 {
		pDevEntry.MutexPersOnuConfig.RUnlock()
		logger.Debugw(ctx, "reconciling - no uni-configs have been stored before adapter restart - terminate reconcilement",
			log.Fields{"device-id": dh.DeviceID})
		dh.stopReconciling(ctx, true, cWaitReconcileFlowNoActivity)
		return continueWithFlowConfig
	}
	flowsFound := false
	techProfsFound := false
	techProfInstLoadFailed := false
outerLoop:
	for _, uniData := range pDevEntry.SOnuPersistentData.PersUniConfig {
		uniID := uniData.PersUniID
		//TODO: check for uni-port specific reconcilement in case of multi-uni-port-per-onu-support
		if !dh.anyTpPathExists(uniData.PersTpPathMap) {
			logger.Debugw(ctx, "reconciling - no TPs stored for uniID",
				log.Fields{"uni-id": uniID, "device-id": dh.DeviceID})
			continue
		}
		//release MutexPersOnuConfig before TechProfile (ANIConfig) processing as otherwise the reception of
		//  OMCI frames may get completely stuck due to lock request within IncrementMibDataSync() at OMCI
		//  frame reception may also lock the complete OMCI reception processing based on mutexRxSchedMap
		pDevEntry.MutexPersOnuConfig.RUnlock()
		persMutexLock = false
		techProfsFound = true // set to true if we found TP once for any UNI port
		var iaTechTpInst ia.TechProfileDownloadMessage
		var ok bool
		for tpID := range uniData.PersTpPathMap {
			pDevEntry.MutexReconciledTpInstances.RLock()
			if iaTechTpInst, ok = pDevEntry.ReconciledTpInstances[uniID][tpID]; !ok {
				logger.Errorw(ctx, "reconciling - no reconciled tp instance available",
					log.Fields{"tp-id": tpID, "tpPath": uniData.PersTpPathMap[tpID], "uni-id": uniData.PersUniID,
						"device-id": dh.DeviceID})
				techProfInstLoadFailed = true // stop loading tp instance as soon as we hit failure
				pDevEntry.MutexReconciledTpInstances.RUnlock()
				break outerLoop
			}
			pDevEntry.MutexReconciledTpInstances.RUnlock()
			continueWithFlowConfig = true // valid TP found - try flow configuration later
			var tpInst tech_profile.TechProfileInstance
			switch techTpInst := iaTechTpInst.TechTpInstance.(type) {
			case *ia.TechProfileDownloadMessage_TpInstance: // supports only GPON, XGPON, XGS-PON
				tpInst = *techTpInst.TpInstance
				logger.Debugw(ctx, "reconciling - received-tp-instance-successfully-after-reconcile", log.Fields{
					"tp-id": tpID, "tpPath": uniData.PersTpPathMap[tpID], "uni-id": uniData.PersUniID, "device-id": dh.DeviceID})
			default: // do not support epon or other tech
				logger.Errorw(ctx, "reconciling - unsupported-tech-profile", log.Fields{
					"tp-id": tpID, "tpPath": uniData.PersTpPathMap[tpID], "uni-id": uniData.PersUniID, "device-id": dh.DeviceID})
				techProfInstLoadFailed = true // stop loading tp instance as soon as we hit failure
				break outerLoop
			}

			// deadline context to ensure completion of background routines waited for
			//20200721: 10s proved to be less in 8*8 ONU test on local vbox machine with debug, might be further adapted
			deadline := time.Now().Add(dh.pOpenOnuAc.maxTimeoutInterAdapterComm) //allowed run time to finish before execution
			dctx, cancel := context.WithDeadline(ctx, deadline)

			dh.pOnuTP.ResetTpProcessingErrorIndication(uniData.PersUniID, tpID)
			var wg sync.WaitGroup
			wg.Add(1) // for the 1 go routine to finish
			go dh.pOnuTP.ConfigureUniTp(log.WithSpanFromContext(dctx, ctx), uniData.PersUniID, uniData.PersTpPathMap[tpID], tpInst, &wg)
			dh.waitForCompletion(ctx, cancel, &wg, "TechProfReconcile") //wait for background process to finish
			if err := dh.pOnuTP.GetTpProcessingErrorIndication(uniData.PersUniID, tpID); err != nil {
				logger.Errorw(ctx, err.Error(), log.Fields{"device-id": dh.DeviceID})
				techProfInstLoadFailed = true // stop loading tp instance as soon as we hit failure
				break outerLoop
			}
		} // for all TpPath entries for this UNI
		if len(uniData.PersFlowParams) != 0 {
			flowsFound = true
		}
		pDevEntry.MutexPersOnuConfig.RLock() //set protection again for loop test on SOnuPersistentData
		persMutexLock = true
	} // for all UNI entries from SOnuPersistentData
	if persMutexLock { // if loop was left with MutexPersOnuConfig still set
		pDevEntry.MutexPersOnuConfig.RUnlock()
	}

	//had to move techProf/flow result evaluation into separate function due to SCA complexity limit
	dh.updateReconcileStates(ctx, techProfsFound, techProfInstLoadFailed, flowsFound)

	return continueWithFlowConfig
}

func (dh *deviceHandler) updateReconcileStates(ctx context.Context,
	abTechProfsFound bool, abTechProfInstLoadFailed bool, abFlowsFound bool) {
	if !abTechProfsFound {
		logger.Debugw(ctx, "reconciling - no TPs have been stored before adapter restart - terminate reconcilement",
			log.Fields{"device-id": dh.DeviceID})
		dh.stopReconciling(ctx, true, cWaitReconcileFlowNoActivity)
		return
	}
	if abTechProfInstLoadFailed {
		_ = dh.ReasonUpdate(ctx, cmn.DrTechProfileConfigDownloadFailed, dh.IsReconcilingReasonUpdate())
		dh.stopReconciling(ctx, false, cWaitReconcileFlowNoActivity)
		return
	} else if dh.IsSkipOnuConfigReconciling() {
		_ = dh.ReasonUpdate(ctx, cmn.DrTechProfileConfigDownloadSuccess, dh.IsReconcilingReasonUpdate())
	}
	if !abFlowsFound {
		logger.Debugw(ctx, "reconciling - no flows have been stored before adapter restart - terminate reconcilement",
			log.Fields{"device-id": dh.DeviceID})
		dh.stopReconciling(ctx, true, cWaitReconcileFlowNoActivity)
	}
}

func (dh *deviceHandler) ReconcileDeviceFlowConfig(ctx context.Context) {
	logger.Debugw(ctx, "reconciling - trigger flow config", log.Fields{"device-id": dh.DeviceID})

	pDevEntry := dh.GetOnuDeviceEntry(ctx, true)
	if pDevEntry == nil {
		logger.Errorw(ctx, "reconciling - no valid OnuDevice - aborting", log.Fields{"device-id": dh.DeviceID})
		dh.stopReconciling(ctx, false, cWaitReconcileFlowNoActivity)
		return
	}

	pDevEntry.MutexPersOnuConfig.RLock()
	if len(pDevEntry.SOnuPersistentData.PersUniConfig) == 0 {
		pDevEntry.MutexPersOnuConfig.RUnlock()
		logger.Debugw(ctx, "reconciling - no uni-configs have been stored before adapter restart - terminate reconcilement",
			log.Fields{"device-id": dh.DeviceID})
		dh.stopReconciling(ctx, true, cWaitReconcileFlowNoActivity)
		return
	}
	flowsFound := false
	var uniVlanConfigEntries []uint8
	var loWaitGroupWTO cmn.WaitGroupWithTimeOut

	for _, uniData := range pDevEntry.SOnuPersistentData.PersUniConfig {
		//TODO: check for uni-port specific reconcilement in case of multi-uni-port-per-onu-support
		if len(uniData.PersFlowParams) == 0 {
			logger.Debugw(ctx, "reconciling - no flows stored for uniID",
				log.Fields{"uni-id": uniData.PersUniID, "device-id": dh.DeviceID})
			continue
		}
		if !dh.anyTpPathExists(uniData.PersTpPathMap) {
			logger.Warnw(ctx, "reconciling flows - but no TPs stored for uniID, abort",
				log.Fields{"uni-id": uniData.PersUniID, "device-id": dh.DeviceID})
			// It doesn't make sense to configure any flows if no TPs are available
			continue
		}
		//release MutexPersOnuConfig before VlanConfig processing as otherwise the reception of
		//  OMCI frames may get completely stuck due to lock request within IncrementMibDataSync() at OMCI
		//  frame reception may also lock the complete OMCI reception processing based on mutexRxSchedMap
		pDevEntry.MutexPersOnuConfig.RUnlock()

		var uniPort *cmn.OnuUniPort
		var exist bool
		uniNo := platform.MkUniPortNum(ctx, dh.pOnuIndication.GetIntfId(), dh.pOnuIndication.GetOnuId(), uint32(uniData.PersUniID))
		if uniPort, exist = dh.uniEntityMap[uniNo]; !exist {
			logger.Errorw(ctx, "reconciling - OnuUniPort data not found  - terminate reconcilement",
				log.Fields{"uniNo": uniNo, "device-id": dh.DeviceID})
			dh.stopReconciling(ctx, false, cWaitReconcileFlowNoActivity)
			return
		}
		//needed to split up function due to sca complexity
		dh.updateReconcileFlowConfig(ctx, uniPort, uniData.PersFlowParams, uniVlanConfigEntries, &loWaitGroupWTO, &flowsFound)

		logger.Debugw(ctx, "reconciling - flows processed", log.Fields{
			"device-id": dh.DeviceID, "uni-id": uniData.PersUniID,
			"NumUniFlows":       dh.UniVlanConfigFsmMap[uniData.PersUniID].NumUniFlows,
			"ConfiguredUniFlow": dh.UniVlanConfigFsmMap[uniData.PersUniID].ConfiguredUniFlow})
		// this can't be used as global finished reconciling flag because
		// assumes is getting called before the state machines for the last flow is completed,
		// while this is not guaranteed.
		pDevEntry.MutexPersOnuConfig.RLock() //set protection again for loop test on SOnuPersistentData
	} // for all UNI entries from SOnuPersistentData
	pDevEntry.MutexPersOnuConfig.RUnlock()

	if !flowsFound {
		logger.Debugw(ctx, "reconciling - no flows have been stored before adapter restart - terminate reconcilement",
			log.Fields{"device-id": dh.DeviceID})
		dh.stopReconciling(ctx, true, cWaitReconcileFlowNoActivity)
		return
	}
	logger.Debugw(ctx, "reconciling flows - waiting on ready indication of requested UNIs", log.Fields{
		"device-id": dh.DeviceID, "expiry": dh.reconcileExpiryVlanConfig})
	if executed := loWaitGroupWTO.WaitTimeout(dh.reconcileExpiryVlanConfig); executed {
		logger.Debugw(ctx, "reconciling flows for all UNI's has been finished in time",
			log.Fields{"device-id": dh.DeviceID})
		dh.stopReconciling(ctx, true, cWaitReconcileFlowAbortOnSuccess)
		if pDevEntry != nil {
			pDevEntry.SendChReconcilingFlowsFinished(ctx, true)
		}
	} else {
		logger.Errorw(ctx, "reconciling - timeout waiting for reconciling flows for all UNI's to be finished!",
			log.Fields{"device-id": dh.DeviceID})
		dh.stopReconciling(ctx, false, cWaitReconcileFlowAbortOnError)
		if pDevEntry != nil {
			pDevEntry.SendChReconcilingFlowsFinished(ctx, false)
		}
		return
	}
	_ = dh.ReasonUpdate(ctx, cmn.DrOmciFlowsPushed, dh.IsReconcilingReasonUpdate())
}

func (dh *deviceHandler) updateReconcileFlowConfig(ctx context.Context, apUniPort *cmn.OnuUniPort,
	aPersFlowParam []cmn.UniVlanFlowParams, aUniVlanConfigEntries []uint8,
	apWaitGroup *cmn.WaitGroupWithTimeOut, apFlowsFound *bool) {
	flowsProcessed := 0
	lastFlowToReconcile := false
	loUniID := apUniPort.UniID
	for _, flowData := range aPersFlowParam {
		if !(*apFlowsFound) {
			*apFlowsFound = true
			syncChannel := make(chan struct{})
			// start go routine with select() on reconciling vlan config channel before
			// starting vlan config reconciling process to prevent loss of any signal
			// this routine just collects all the received 'flow-reconciled' signals - possibly from different UNI's
			go dh.waitOnUniVlanConfigReconcilingReady(ctx, syncChannel, apWaitGroup)
			//block until the wait routine is really blocked on channel input
			//  in order to prevent to early ready signal from VlanConfig processing
			<-syncChannel
		}
		if flowsProcessed == len(aPersFlowParam)-1 {
			var uniAdded bool
			lastFlowToReconcile = true
			if aUniVlanConfigEntries, uniAdded = dh.appendIfMissing(aUniVlanConfigEntries, loUniID); uniAdded {
				apWaitGroup.Add(1) //increment the waiting group
			}
		}
		logger.Debugw(ctx, "reconciling - add flow with cookie slice", log.Fields{
			"device-id": dh.DeviceID, "uni-id": loUniID,
			"flowsProcessed": flowsProcessed, "cookies": flowData.CookieSlice})
		dh.lockVlanConfig.Lock()
		//the CookieSlice can be passed 'by value' here, - which internally passes its reference
		if _, exist := dh.UniVlanConfigFsmMap[loUniID]; exist {
			if err := dh.UniVlanConfigFsmMap[loUniID].SetUniFlowParams(ctx, flowData.VlanRuleParams.TpID,
				flowData.CookieSlice, uint16(flowData.VlanRuleParams.MatchVid), uint8(flowData.VlanRuleParams.MatchPcp), uint16(flowData.VlanRuleParams.SetVid), uint8(flowData.VlanRuleParams.SetPcp), flowData.VlanRuleParams.InnerCvlan, lastFlowToReconcile, flowData.Meter, nil); err != nil {
				logger.Errorw(ctx, err.Error(), log.Fields{"device-id": dh.DeviceID})
			}
		} else {
			if err := dh.createVlanFilterFsm(ctx, apUniPort, flowData.VlanRuleParams.TpID, flowData.CookieSlice,
				uint16(flowData.VlanRuleParams.MatchVid), uint8(flowData.VlanRuleParams.MatchPcp), uint16(flowData.VlanRuleParams.SetVid),
				uint8(flowData.VlanRuleParams.SetPcp), flowData.VlanRuleParams.InnerCvlan, cmn.OmciVlanFilterAddDone, lastFlowToReconcile, flowData.Meter, nil); err != nil {
				logger.Errorw(ctx, err.Error(), log.Fields{"device-id": dh.DeviceID})
			}
		}
		dh.lockVlanConfig.Unlock()
		flowsProcessed++
	} //for all flows of this UNI
}

//waitOnUniVlanConfigReconcilingReady collects all VlanConfigReady signals from VlanConfig FSM processing in reconciling
//  and decrements the according handler wait group waiting for these indications
func (dh *deviceHandler) waitOnUniVlanConfigReconcilingReady(ctx context.Context, aSyncChannel chan<- struct{},
	waitGroup *cmn.WaitGroupWithTimeOut) {
	var reconciledUniVlanConfigEntries []uint8
	var appended bool
	expiry := dh.GetReconcileExpiryVlanConfigAbort()
	logger.Debugw(ctx, "start waiting on reconcile vlanConfig ready indications", log.Fields{
		"device-id": dh.DeviceID, "expiry": expiry})
	// indicate blocking on channel now to the caller
	aSyncChannel <- struct{}{}
	for {
		select {
		case uniIndication := <-dh.chUniVlanConfigReconcilingDone:
			switch uniIndication {
			// no activity requested (should normally not be received) - just continue waiting
			case cWaitReconcileFlowNoActivity:
			// waiting on channel inputs from VlanConfig for all UNI's to be aborted on error condition
			case cWaitReconcileFlowAbortOnError:
				logger.Debugw(ctx, "waitReconcileFlow aborted on error",
					log.Fields{"device-id": dh.DeviceID, "rxEntries": reconciledUniVlanConfigEntries})
				return
			// waiting on channel inputs from VlanConfig for all UNI's to be aborted on success condition
			case cWaitReconcileFlowAbortOnSuccess:
				logger.Debugw(ctx, "waitReconcileFlow aborted on success",
					log.Fields{"device-id": dh.DeviceID, "rxEntries": reconciledUniVlanConfigEntries})
				return
			// this should be a valid UNI vlan config done indication
			default:
				if uniIndication < platform.MaxUnisPerOnu {
					logger.Debugw(ctx, "reconciling flows has been finished in time for this UNI",
						log.Fields{"device-id": dh.DeviceID, "uni-id": uniIndication})
					if reconciledUniVlanConfigEntries, appended =
						dh.appendIfMissing(reconciledUniVlanConfigEntries, uint8(uniIndication)); appended {
						waitGroup.Done()
					}
				} else {
					logger.Errorw(ctx, "received unexpected UNI flowConfig done indication - is ignored",
						log.Fields{"device-id": dh.DeviceID, "uni-id": uniIndication})
				}
			} //switch uniIndication

		case <-time.After(expiry): //a bit longer than reconcileExpiryVlanConfig
			logger.Errorw(ctx, "timeout waiting for reconciling all UNI flows to be finished!",
				log.Fields{"device-id": dh.DeviceID})
			return
		}
	}
}

func (dh *deviceHandler) GetReconcileExpiryVlanConfigAbort() time.Duration {
	return dh.reconcileExpiryVlanConfig + (500 * time.Millisecond)
}

func (dh *deviceHandler) appendIfMissing(slice []uint8, val uint8) ([]uint8, bool) {
	for _, ele := range slice {
		if ele == val {
			return slice, false
		}
	}
	return append(slice, val), true
}

// sendChReconcileFinished - sends true or false on reconcileFinish channel
func (dh *deviceHandler) sendChReconcileFinished(success bool) {
	if dh != nil { //if the object still exists (might have been already deleted in background)
		//use asynchronous channel sending to avoid stucking on non-waiting receiver
		select {
		case dh.chReconcilingFinished <- success:
		default:
		}
	}
}

// SendChUniVlanConfigFinished - sends the Uni number on channel if the flow reconcilement for this UNI is finished
func (dh *deviceHandler) SendChUniVlanConfigFinished(value uint16) {
	if dh != nil { //if the object still exists (might have been already deleted in background)
		//use asynchronous channel sending to avoid stucking on non-waiting receiver
		select {
		case dh.chUniVlanConfigReconcilingDone <- value:
		default:
		}
	}
}

func (dh *deviceHandler) deleteDevicePersistencyData(ctx context.Context) error {
	logger.Debugw(ctx, "delete device persistency data", log.Fields{"device-id": dh.DeviceID})

	pDevEntry := dh.GetOnuDeviceEntry(ctx, false)
	if pDevEntry == nil {
		//IfDevEntry does not exist here, no problem - no persistent data should have been stored
		logger.Debugw(ctx, "OnuDevice does not exist - nothing to delete", log.Fields{"device-id": dh.DeviceID})
		return nil
	}

	// deadline context to ensure completion of background routines waited for
	//20200721: 10s proved to be less in 8*8 ONU test on local vbox machine with debug, might be further adapted
	deadline := time.Now().Add(dh.pOpenOnuAc.maxTimeoutInterAdapterComm) //allowed run time to finish before execution
	dctx, cancel := context.WithDeadline(ctx, deadline)

	pDevEntry.ResetKvProcessingErrorIndication()

	var wg sync.WaitGroup
	wg.Add(1) // for the 1 go routine to finish
	go pDevEntry.DeleteDataFromOnuKvStore(log.WithSpanFromContext(dctx, ctx), &wg)
	dh.waitForCompletion(ctx, cancel, &wg, "DeleteDevice") //wait for background process to finish

	// TODO: further actions - stop metrics and FSMs, remove device ...
	return pDevEntry.GetKvProcessingErrorIndication()
}

//func (dh *deviceHandler) rebootDevice(ctx context.Context, device *voltha.Device) error {
// before this change here return like this was used:
// 		return fmt.Errorf("device-unreachable: %s, %s", dh.DeviceID, device.SerialNumber)
//was and is called in background - error return does not make sense
func (dh *deviceHandler) rebootDevice(ctx context.Context, aCheckDeviceState bool, device *voltha.Device) {
	logger.Infow(ctx, "reboot-device", log.Fields{"device-id": dh.DeviceID, "SerialNumber": dh.device.SerialNumber})
	if aCheckDeviceState && device.ConnectStatus != voltha.ConnectStatus_REACHABLE {
		logger.Errorw(ctx, "device-unreachable", log.Fields{"device-id": device.Id, "SerialNumber": device.SerialNumber})
		return
	}
	if err := dh.pOnuOmciDevice.Reboot(log.WithSpanFromContext(context.TODO(), ctx)); err != nil {
		//TODO with VOL-3045/VOL-3046: return the error and stop further processing
		logger.Errorw(ctx, "error-rebooting-device", log.Fields{"device-id": dh.DeviceID, "error": err})
		return
	}

	//transfer the possibly modified logical uni port state
	dh.DisableUniPortStateUpdate(ctx)

	logger.Debugw(ctx, "call DeviceStateUpdate upon reboot", log.Fields{
		"OperStatus": voltha.OperStatus_DISCOVERED, "device-id": dh.DeviceID})
	// do not set the ConnStatus here as it may conflict with the parallel setting from ONU down indication (updateInterface())
	if err := dh.updateDeviceStateInCore(ctx, &ca.DeviceStateFilter{
		DeviceId:   dh.DeviceID,
		ConnStatus: connectStatusINVALID, //use some dummy value to prevent modification of the ConnStatus
		OperStatus: voltha.OperStatus_DISCOVERED,
	}); err != nil {
		//TODO with VOL-3045/VOL-3046: return the error and stop further processing
		logger.Errorw(ctx, "error-updating-device-state", log.Fields{"device-id": dh.DeviceID, "error": err})
		return
	}
	if err := dh.ReasonUpdate(ctx, cmn.DrRebooting, true); err != nil {
		return
	}
	dh.SetReadyForOmciConfig(false)
	//no specific activity to synchronize any internal FSM to the 'rebooted' state is explicitly done here
	//  the expectation ids for a real device, that it will be synced with the expected following 'down' indication
	//  as BBSIM does not support this testing requires explicite disable/enable device calls in which sequence also
	//  all other FSM's should be synchronized again
}

//doOnuSwUpgrade initiates the SW download transfer to the ONU and on success activates the (inactive) image
//  used only for old - R2.7 style - upgrade API
func (dh *deviceHandler) doOnuSwUpgrade(ctx context.Context, apImageDsc *voltha.ImageDownload,
	apDownloadManager *swupg.AdapterDownloadManager) error {
	logger.Debugw(ctx, "onuSwUpgrade requested", log.Fields{
		"device-id": dh.DeviceID, "image-name": (*apImageDsc).Name})

	var err error
	pDevEntry := dh.GetOnuDeviceEntry(ctx, true)
	if pDevEntry == nil {
		logger.Errorw(ctx, "start Onu SW upgrade rejected: no valid OnuDevice", log.Fields{"device-id": dh.DeviceID})
		return fmt.Errorf("start Onu SW upgrade rejected: no valid OnuDevice for device-id: %s", dh.DeviceID)
	}

	if dh.IsReadyForOmciConfig() {
		var inactiveImageID uint16
		if inactiveImageID, err = pDevEntry.GetInactiveImageMeID(ctx); err == nil {
			dh.lockUpgradeFsm.Lock()
			//lockUpgradeFsm must be release before cancellation as this may implicitly request RemoveOnuUpgradeFsm()
			//  but must be still locked at calling createOnuUpgradeFsm
			if dh.pOnuUpradeFsm == nil {
				err = dh.createOnuUpgradeFsm(ctx, pDevEntry, cmn.OmciOnuSwUpgradeDone)
				dh.lockUpgradeFsm.Unlock()
				if err == nil {
					if err = dh.pOnuUpradeFsm.SetDownloadParams(ctx, inactiveImageID, apImageDsc, apDownloadManager); err != nil {
						logger.Errorw(ctx, "onu upgrade fsm could not set parameters", log.Fields{
							"device-id": dh.DeviceID, "error": err})
					}
				} else {
					logger.Errorw(ctx, "onu upgrade fsm could not be created", log.Fields{
						"device-id": dh.DeviceID, "error": err})
				}
			} else { //OnuSw upgrade already running - restart (with possible abort of running)
				dh.lockUpgradeFsm.Unlock()
				logger.Debugw(ctx, "Onu SW upgrade already running - abort", log.Fields{"device-id": dh.DeviceID})
				if !dh.upgradeCanceled { //avoid double cancelation in case it is already doing the cancelation
					dh.upgradeCanceled = true
					dh.pOnuUpradeFsm.CancelProcessing(ctx, true, voltha.ImageState_CANCELLED_ON_REQUEST) //complete abort
				}
				//no effort spent anymore for the old API to automatically cancel and restart the download
				//  like done for the new API
			}
		} else {
			logger.Errorw(ctx, "start Onu SW upgrade rejected: no inactive image", log.Fields{
				"device-id": dh.DeviceID, "error": err})
		}
	} else {
		logger.Errorw(ctx, "start Onu SW upgrade rejected: no active OMCI connection", log.Fields{"device-id": dh.DeviceID})
		err = fmt.Errorf("start Onu SW upgrade rejected: no active OMCI connection for device-id: %s", dh.DeviceID)
	}
	return err
}

//onuSwUpgradeAfterDownload initiates the SW download transfer to the ONU with activate and commit options
// after the OnuImage has been downloaded to the adapter, called in background
func (dh *deviceHandler) onuSwUpgradeAfterDownload(ctx context.Context, apImageRequest *voltha.DeviceImageDownloadRequest,
	apDownloadManager *swupg.FileDownloadManager, aImageIdentifier string) {

	var err error
	pDevEntry := dh.GetOnuDeviceEntry(ctx, true)
	if pDevEntry == nil {
		logger.Errorw(ctx, "start Onu SW upgrade rejected: no valid OnuDevice", log.Fields{"device-id": dh.DeviceID})
		return
	}

	var inactiveImageID uint16
	if inactiveImageID, err = pDevEntry.GetInactiveImageMeID(ctx); err == nil {
		logger.Debugw(ctx, "onuSwUpgrade requested", log.Fields{
			"device-id": dh.DeviceID, "image-version": apImageRequest.Image.Version, "to onu-image": inactiveImageID})

		dh.lockUpgradeFsm.Lock()
		//lockUpgradeFsm must be release before cancellation as this may implicitly request RemoveOnuUpgradeFsm()
		//  but must be still locked at calling createOnuUpgradeFsm
		//  (and working with a local pointer copy does not work here if asynchronous request are done to fast
		//	[e.g.leaving the local pointer on nil even though a creation is already on the way])
		if dh.pOnuUpradeFsm != nil {
			//OnuSw upgrade already running on this device (e.g. with activate/commit not yet set)
			// abort the current processing, running upgrades are always aborted by newer request
			logger.Debugw(ctx, "Onu SW upgrade already running - abort previous activity", log.Fields{"device-id": dh.DeviceID})
			//flush the remove upgradeFsmChan channel
			select {
			case <-dh.upgradeFsmChan:
				logger.Debugw(ctx, "flushed-upgrade-fsm-channel", log.Fields{"device-id": dh.DeviceID})
			default:
			}
			dh.lockUpgradeFsm.Unlock()
			if !dh.upgradeCanceled { //avoid double cancelation in case it is already doing the cancelation
				dh.upgradeCanceled = true
				dh.pOnuUpradeFsm.CancelProcessing(ctx, true, voltha.ImageState_CANCELLED_ON_REQUEST) //complete abort
			}
			select {
			case <-time.After(cTimeOutRemoveUpgrade * time.Second):
				logger.Errorw(ctx, "could not remove Upgrade FSM in time, aborting", log.Fields{"device-id": dh.DeviceID})
				//should not appear, can't proceed with new upgrade, perhaps operator can retry manually later
				return
			case <-dh.upgradeFsmChan:
				logger.Debugw(ctx, "recent Upgrade FSM removed, proceed with new request", log.Fields{"device-id": dh.DeviceID})
			}
			dh.lockUpgradeFsm.Lock() //lock again for following creation
		}

		//here it can be assumed that no running upgrade processing exists (anymore)
		//OmciOnuSwUpgradeDone could be used to create some event notification with information on upgrade completion,
		//  but none yet defined
		err = dh.createOnuUpgradeFsm(ctx, pDevEntry, cmn.OmciOnuSwUpgradeDone)
		dh.lockUpgradeFsm.Unlock()
		if err == nil {
			if err = dh.pOnuUpradeFsm.SetDownloadParamsAfterDownload(ctx, inactiveImageID,
				apImageRequest, apDownloadManager, aImageIdentifier); err != nil {
				logger.Errorw(ctx, "onu upgrade fsm could not set parameters", log.Fields{
					"device-id": dh.DeviceID, "error": err})
				return
			}
		} else {
			logger.Errorw(ctx, "onu upgrade fsm could not be created", log.Fields{
				"device-id": dh.DeviceID, "error": err})
		}
		return
	}
	logger.Errorw(ctx, "start Onu SW upgrade rejected: no inactive image", log.Fields{
		"device-id": dh.DeviceID, "error": err})
}

//onuSwActivateRequest ensures activation of the requested image with commit options
func (dh *deviceHandler) onuSwActivateRequest(ctx context.Context,
	aVersion string, aCommitRequest bool) (*voltha.ImageState, error) {
	var err error
	//SW activation for the ONU image may have two use cases, one of them is selected here according to following prioritization:
	//  1.) activation of the image for a started upgrade process (in case the running upgrade runs on the requested image)
	//  2.) activation of the inactive image

	pDevEntry := dh.GetOnuDeviceEntry(ctx, true)
	if pDevEntry == nil {
		logger.Errorw(ctx, "Onu image activation rejected: no valid OnuDevice", log.Fields{"device-id": dh.DeviceID})
		return nil, fmt.Errorf("no valid OnuDevice for device-id: %s", dh.DeviceID)
	}
	dh.lockUpgradeFsm.RLock()
	if dh.pOnuUpradeFsm != nil {
		dh.lockUpgradeFsm.RUnlock()
		onuVolthaDevice, getErr := dh.getDeviceFromCore(ctx, dh.DeviceID)
		if getErr != nil || onuVolthaDevice == nil {
			logger.Errorw(ctx, "Failed to fetch Onu device for image activation", log.Fields{"device-id": dh.DeviceID, "err": getErr})
			return nil, fmt.Errorf("could not fetch device for device-id: %s", dh.DeviceID)
		}
		if dh.upgradeCanceled { //avoid starting some new action in case it is already doing the cancelation
			logger.Errorw(ctx, "Some upgrade procedure still runs cancelation - abort", log.Fields{"device-id": dh.DeviceID})
			return nil, fmt.Errorf("request collides with some ongoing cancelation for device-id: %s", dh.DeviceID)
		}
		//  use the OnuVendor identification from this device for the internal unique name
		imageIdentifier := onuVolthaDevice.VendorId + aVersion //head on vendor ID of the ONU
		// 1.) check a started upgrade process and relay the activation request to it
		if err = dh.pOnuUpradeFsm.SetActivationParamsRunning(ctx, imageIdentifier, aCommitRequest); err != nil {
			//if some ONU upgrade is ongoing we do not accept some explicit ONU image-version related activation
			logger.Errorw(ctx, "onu upgrade fsm did not accept activation while running", log.Fields{
				"device-id": dh.DeviceID, "error": err})
			return nil, fmt.Errorf("activation not accepted for this version for device-id: %s", dh.DeviceID)
		}
		logger.Debugw(ctx, "image activation acknowledged by onu upgrade processing", log.Fields{
			"device-id": dh.DeviceID, "image-id": imageIdentifier})
		pImageStates := dh.pOnuUpradeFsm.GetImageStates(ctx, "", aVersion)
		return pImageStates, nil
	} //else
	dh.lockUpgradeFsm.RUnlock()

	// 2.) check if requested image-version equals the inactive one and start its activation
	//   (image version is not [yet] checked - would be possible, but with increased effort ...)
	var inactiveImageID uint16
	if inactiveImageID, err = pDevEntry.GetInactiveImageMeID(ctx); err != nil || inactiveImageID > 1 {
		logger.Errorw(ctx, "get inactive image failed", log.Fields{
			"device-id": dh.DeviceID, "err": err, "image-id": inactiveImageID})
		return nil, fmt.Errorf("no valid inactive image found for device-id: %s", dh.DeviceID)
	}
	dh.lockUpgradeFsm.Lock() //lock again for following creation
	err = dh.createOnuUpgradeFsm(ctx, pDevEntry, cmn.OmciOnuSwUpgradeDone)
	dh.lockUpgradeFsm.Unlock()
	if err == nil {
		if err = dh.pOnuUpradeFsm.SetActivationParamsStart(ctx, aVersion,
			inactiveImageID, aCommitRequest); err != nil {
			logger.Errorw(ctx, "onu upgrade fsm did not accept activation to start", log.Fields{
				"device-id": dh.DeviceID, "error": err})
			return nil, fmt.Errorf("activation to start from scratch not accepted for device-id: %s", dh.DeviceID)
		}
		logger.Debugw(ctx, "inactive image activation acknowledged by onu upgrade", log.Fields{
			"device-id": dh.DeviceID, "image-version": aVersion})
		pImageStates := dh.pOnuUpradeFsm.GetImageStates(ctx, "", aVersion)
		return pImageStates, nil
	} //else
	logger.Errorw(ctx, "onu upgrade fsm could not be created", log.Fields{
		"device-id": dh.DeviceID, "error": err})
	return nil, fmt.Errorf("could not start upgradeFsm for device-id: %s", dh.DeviceID)
}

//onuSwCommitRequest ensures commitment of the requested image
func (dh *deviceHandler) onuSwCommitRequest(ctx context.Context,
	aVersion string) (*voltha.ImageState, error) {
	var err error
	//SW commitment for the ONU image may have two use cases, one of them is selected here according to following prioritization:
	//  1.) commitment of the image for a started upgrade process (in case the running upgrade runs on the requested image)
	//  2.) commitment of the active image

	pDevEntry := dh.GetOnuDeviceEntry(ctx, true)
	if pDevEntry == nil {
		logger.Errorw(ctx, "Onu image commitment rejected: no valid OnuDevice", log.Fields{"device-id": dh.DeviceID})
		return nil, fmt.Errorf("no valid OnuDevice for device-id: %s", dh.DeviceID)
	}
	dh.lockUpgradeFsm.RLock()
	if dh.pOnuUpradeFsm != nil {
		dh.lockUpgradeFsm.RUnlock()
		onuVolthaDevice, getErr := dh.getDeviceFromCore(ctx, dh.DeviceID)
		if getErr != nil || onuVolthaDevice == nil {
			logger.Errorw(ctx, "Failed to fetch Onu device for image commitment", log.Fields{"device-id": dh.DeviceID, "err": getErr})
			return nil, fmt.Errorf("could not fetch device for device-id: %s", dh.DeviceID)
		}
		if dh.upgradeCanceled { //avoid starting some new action in case it is already doing the cancelation
			logger.Errorw(ctx, "Some upgrade procedure still runs cancelation - abort", log.Fields{"device-id": dh.DeviceID})
			return nil, fmt.Errorf("request collides with some ongoing cancelation for device-id: %s", dh.DeviceID)
		}
		//  use the OnuVendor identification from this device for the internal unique name
		imageIdentifier := onuVolthaDevice.VendorId + aVersion //head on vendor ID of the ONU
		// 1.) check a started upgrade process and relay the commitment request to it
		// the running upgrade may be based either on the imageIdentifier (started from download)
		//   or on the imageVersion (started from pure activation)
		if err = dh.pOnuUpradeFsm.SetCommitmentParamsRunning(ctx, imageIdentifier, aVersion); err != nil {
			//if some ONU upgrade is ongoing we do not accept some explicit different ONU image-version related commitment
			logger.Errorw(ctx, "onu upgrade fsm did not accept commitment while running", log.Fields{
				"device-id": dh.DeviceID, "error": err})
			return nil, fmt.Errorf("commitment not accepted for this version for device-id: %s", dh.DeviceID)
		}
		logger.Debugw(ctx, "image commitment acknowledged by onu upgrade processing", log.Fields{
			"device-id": dh.DeviceID, "image-id": imageIdentifier})
		pImageStates := dh.pOnuUpradeFsm.GetImageStates(ctx, "", aVersion)
		return pImageStates, nil
	} //else
	dh.lockUpgradeFsm.RUnlock()

	// 2.) use the active image to directly commit
	var activeImageID uint16
	if activeImageID, err = pDevEntry.GetActiveImageMeID(ctx); err != nil || activeImageID > 1 {
		logger.Errorw(ctx, "get active image failed", log.Fields{
			"device-id": dh.DeviceID, "err": err, "image-id": activeImageID})
		return nil, fmt.Errorf("no valid active image found for device-id: %s", dh.DeviceID)
	}
	dh.lockUpgradeFsm.Lock() //lock again for following creation
	err = dh.createOnuUpgradeFsm(ctx, pDevEntry, cmn.OmciOnuSwUpgradeDone)
	dh.lockUpgradeFsm.Unlock()
	if err == nil {
		if err = dh.pOnuUpradeFsm.SetCommitmentParamsStart(ctx, aVersion, activeImageID); err != nil {
			logger.Errorw(ctx, "onu upgrade fsm did not accept commitment to start", log.Fields{
				"device-id": dh.DeviceID, "error": err})
			return nil, fmt.Errorf("commitment to start from scratch not accepted for device-id: %s", dh.DeviceID)
		}
		logger.Debugw(ctx, "active image commitment acknowledged by onu upgrade", log.Fields{
			"device-id": dh.DeviceID, "image-version": aVersion})
		pImageStates := dh.pOnuUpradeFsm.GetImageStates(ctx, "", aVersion)
		return pImageStates, nil
	} //else
	logger.Errorw(ctx, "onu upgrade fsm could not be created", log.Fields{
		"device-id": dh.DeviceID, "error": err})
	return nil, fmt.Errorf("could not start upgradeFsm for device-id: %s", dh.DeviceID)
}

func (dh *deviceHandler) requestOnuSwUpgradeState(ctx context.Context, aImageIdentifier string,
	aVersion string) *voltha.ImageState {
	var pImageState *voltha.ImageState
	dh.lockUpgradeFsm.RLock()
	defer dh.lockUpgradeFsm.RUnlock()
	if dh.pOnuUpradeFsm != nil {
		pImageState = dh.pOnuUpradeFsm.GetImageStates(ctx, aImageIdentifier, aVersion)
	} else { //use the last stored ImageState (if the requested Imageversion coincides)
		if aVersion == dh.pLastUpgradeImageState.Version {
			pImageState = dh.pLastUpgradeImageState
		} else { //state request for an image version different from last processed image version
			pImageState = &voltha.ImageState{
				Version: aVersion,
				//we cannot state something concerning this version
				DownloadState: voltha.ImageState_DOWNLOAD_UNKNOWN,
				Reason:        voltha.ImageState_NO_ERROR,
				ImageState:    voltha.ImageState_IMAGE_UNKNOWN,
			}
		}
	}
	return pImageState
}

func (dh *deviceHandler) cancelOnuSwUpgrade(ctx context.Context, aImageIdentifier string,
	aVersion string, pDeviceImageState *voltha.DeviceImageState) {
	pDeviceImageState.DeviceId = dh.DeviceID
	pDeviceImageState.ImageState.Version = aVersion
	dh.lockUpgradeFsm.RLock()
	if dh.pOnuUpradeFsm != nil {
		dh.lockUpgradeFsm.RUnlock()
		// so then we cancel the upgrade operation
		// but before we still request the actual upgrade states for the direct response
		pImageState := dh.pOnuUpradeFsm.GetImageStates(ctx, aImageIdentifier, aVersion)
		pDeviceImageState.ImageState.DownloadState = pImageState.DownloadState
		pDeviceImageState.ImageState.Reason = voltha.ImageState_CANCELLED_ON_REQUEST
		pDeviceImageState.ImageState.ImageState = pImageState.ImageState
		if pImageState.DownloadState != voltha.ImageState_DOWNLOAD_UNKNOWN {
			//so here the imageIdentifier or version equals to what is used in the upgrade FSM
			if !dh.upgradeCanceled { //avoid double cancelation in case it is already doing the cancelation
				dh.upgradeCanceled = true
				dh.pOnuUpradeFsm.CancelProcessing(ctx, true, voltha.ImageState_CANCELLED_ON_REQUEST) //complete abort
			}
		} //nothing to cancel (upgrade FSM for different image stays alive)
	} else {
		dh.lockUpgradeFsm.RUnlock()
		// if no upgrade is ongoing, nothing is canceled and accordingly the states of the requested image are unknown
		// reset also the dh handler LastUpgradeImageState (not relevant anymore/cleared)
		(*dh.pLastUpgradeImageState).DownloadState = voltha.ImageState_DOWNLOAD_UNKNOWN
		(*dh.pLastUpgradeImageState).Reason = voltha.ImageState_NO_ERROR
		(*dh.pLastUpgradeImageState).ImageState = voltha.ImageState_IMAGE_UNKNOWN
		(*dh.pLastUpgradeImageState).Version = "" //reset to 'no (relevant) upgrade done' (like initial state)
		pDeviceImageState.ImageState.DownloadState = voltha.ImageState_DOWNLOAD_UNKNOWN
		pDeviceImageState.ImageState.Reason = voltha.ImageState_NO_ERROR
		pDeviceImageState.ImageState.ImageState = voltha.ImageState_IMAGE_UNKNOWN
		//an abort request to a not active upgrade processing can be used to reset the device upgrade states completely
	}
}

func (dh *deviceHandler) getOnuImages(ctx context.Context) (*voltha.OnuImages, error) {

	var onuImageStatus *swupg.OnuImageStatus

	pDevEntry := dh.GetOnuDeviceEntry(ctx, false)
	if pDevEntry != nil {
		onuImageStatus = swupg.NewOnuImageStatus(dh, pDevEntry)
		pDevEntry.MutexOnuImageStatus.Lock()
		pDevEntry.POnuImageStatus = onuImageStatus
		pDevEntry.MutexOnuImageStatus.Unlock()

	} else {
		logger.Errorw(ctx, "No valid OnuDevice - aborting", log.Fields{"device-id": dh.DeviceID})
		return nil, fmt.Errorf("no-valid-OnuDevice-aborting")
	}
	images, err := onuImageStatus.GetOnuImageStatus(ctx)
	pDevEntry.MutexOnuImageStatus.Lock()
	pDevEntry.POnuImageStatus = nil
	pDevEntry.MutexOnuImageStatus.Unlock()
	return images, err
}

//  deviceHandler methods that implement the adapters interface requests## end #########
// #####################################################################################

// ################  to be updated acc. needs of ONU Device ########################
// deviceHandler StateMachine related state transition methods ##### begin #########

func (dh *deviceHandler) logStateChange(ctx context.Context, e *fsm.Event) {
	logger.Debugw(ctx, "Device FSM: ", log.Fields{"event name": string(e.Event),
		"src state": string(e.Src), "dst state": string(e.Dst), "device-id": dh.DeviceID})
}

// doStateInit provides the device update to the core
func (dh *deviceHandler) doStateInit(ctx context.Context, e *fsm.Event) {

	logger.Debugw(ctx, "doStateInit-started", log.Fields{"device-id": dh.DeviceID})
	var err error

	// populate what we know.  rest comes later after mib sync
	dh.device.Root = false
	dh.device.Vendor = "OpenONU"
	dh.device.Model = "go"
	dh.device.Reason = cmn.DeviceReasonMap[cmn.DrActivatingOnu]
	_ = dh.ReasonUpdate(ctx, cmn.DrActivatingOnu, false)

	dh.logicalDeviceID = dh.DeviceID // really needed - what for ??? //TODO!!!

	if !dh.IsReconciling() {
		logger.Infow(ctx, "DeviceUpdate", log.Fields{"deviceReason": dh.device.Reason, "device-id": dh.DeviceID})
		if err := dh.updateDeviceInCore(ctx, dh.device); err != nil {
			logger.Errorw(ctx, "device-update-failed", log.Fields{"device-id": dh.device.Id, "error": err})
		}
		//TODO Need to Update Device Reason To CORE as part of device update userstory
	} else {
		logger.Debugw(ctx, "reconciling - don't notify core about DeviceUpdate",
			log.Fields{"device-id": dh.DeviceID})
	}

	dh.parentID = dh.device.ParentId
	dh.ponPortNumber = dh.device.ParentPortNo

	// store proxy parameters for later communication - assumption: invariant, else they have to be requested dynamically!!
	dh.ProxyAddressID = dh.device.ProxyAddress.GetDeviceId()
	dh.ProxyAddressType = dh.device.ProxyAddress.GetDeviceType()
	logger.Debugw(ctx, "device-updated", log.Fields{"device-id": dh.DeviceID, "proxyAddressID": dh.ProxyAddressID,
		"proxyAddressType": dh.ProxyAddressType, "SNR": dh.device.SerialNumber,
		"ParentId": dh.parentID, "ParentPortNo": dh.ponPortNumber})

	/*
		self._pon = PonPort.create(self, self._pon_port_number)
		self._pon.add_peer(self.parent_id, self._pon_port_number)
		self.logger.debug('adding-pon-port-to-agent',
				   type=self._pon.get_port().type,
				   admin_state=self._pon.get_port().admin_state,
				   oper_status=self._pon.get_port().oper_status,
				   )
	*/
	if !dh.IsReconciling() {
		logger.Debugw(ctx, "adding-pon-port", log.Fields{"device-id": dh.DeviceID, "ponPortNo": dh.ponPortNumber})
		var ponPortNo uint32 = 1
		if dh.ponPortNumber != 0 {
			ponPortNo = dh.ponPortNumber
		}

		pPonPort := &voltha.Port{
			DeviceId:   dh.DeviceID,
			PortNo:     ponPortNo,
			Label:      fmt.Sprintf("pon-%d", ponPortNo),
			Type:       voltha.Port_PON_ONU,
			OperStatus: voltha.OperStatus_ACTIVE,
			Peers: []*voltha.Port_PeerPort{{DeviceId: dh.parentID, // Peer device  is OLT
				PortNo: ponPortNo}}, // Peer port is parent's port number
		}
		if err = dh.CreatePortInCore(ctx, pPonPort); err != nil {
			logger.Fatalf(ctx, "Device FSM: PortCreated-failed-%s:%s", err, dh.DeviceID)
			e.Cancel(err)
			return
		}
	} else {
		logger.Debugw(ctx, "reconciling - pon-port already added", log.Fields{"device-id": dh.DeviceID})
	}
	logger.Debugw(ctx, "doStateInit-done", log.Fields{"device-id": dh.DeviceID})
}

// postInit setups the DeviceEntry for the conerned device
func (dh *deviceHandler) postInit(ctx context.Context, e *fsm.Event) {

	logger.Debugw(ctx, "postInit-started", log.Fields{"device-id": dh.DeviceID})
	var err error
	/*
		dh.Client = oop.NewOpenoltClient(dh.clientCon)
		dh.pTransitionMap.Handle(ctx, GrpcConnected)
		return nil
	*/
	if err = dh.addOnuDeviceEntry(log.WithSpanFromContext(context.TODO(), ctx)); err != nil {
		logger.Fatalf(ctx, "Device FSM: addOnuDeviceEntry-failed-%s:%s", err, dh.DeviceID)
		e.Cancel(err)
		return
	}

	if dh.IsReconciling() {
		go dh.reconcileDeviceOnuInd(ctx)
		// reconcilement will be continued after mib download is done
	}

	/*
			############################################################################
			# Setup Alarm handler
			self.events = AdapterEvents(self.core_proxy, device.id, self.logical_device_id,
										device.serial_number)
			############################################################################
			# Setup PM configuration for this device
			# Pass in ONU specific options
			kwargs = {
				OnuPmMetrics.DEFAULT_FREQUENCY_KEY: OnuPmMetrics.DEFAULT_ONU_COLLECTION_FREQUENCY,
				'heartbeat': self.heartbeat,
				OnuOmciPmMetrics.OMCI_DEV_KEY: self._onu_omci_device
			}
			self.logger.debug('create-pm-metrics', device_id=device.id, serial_number=device.serial_number)
			self._pm_metrics = OnuPmMetrics(self.events, self.core_proxy, self.device_id,
										   self.logical_device_id, device.serial_number,
										   grouped=True, freq_override=False, **kwargs)
			pm_config = self._pm_metrics.make_proto()
			self._onu_omci_device.set_pm_config(self._pm_metrics.omci_pm.openomci_interval_pm)
			self.logger.info("initial-pm-config", device_id=device.id, serial_number=device.serial_number)
			yield self.core_proxy.device_pm_config_update(pm_config, init=True)

			# Note, ONU ID and UNI intf set in add_uni_port method
			self._onu_omci_device.alarm_synchronizer.set_alarm_params(mgr=self.events,
																	  ani_ports=[self._pon])

			# Code to Run OMCI Test Action
			kwargs_omci_test_action = {
				OmciTestRequest.DEFAULT_FREQUENCY_KEY:
					OmciTestRequest.DEFAULT_COLLECTION_FREQUENCY
			}
			serial_number = device.serial_number
			self._test_request = OmciTestRequest(self.core_proxy,
										   self.omci_agent, self.device_id,
										   AniG, serial_number,
										   self.logical_device_id,
										   exclusive=False,
										   **kwargs_omci_test_action)

			self.Enabled = True
		else:
			self.logger.info('onu-already-activated')
	*/