VOL-4147
Add support of setting operation state to RECONCILE_FAILED from adapters
Change-Id: I419517fc741abb5d625862740c91e291f02df89f
diff --git a/rw_core/core/device/agent.go b/rw_core/core/device/agent.go
index 22ed4c4..c2705e9 100755
--- a/rw_core/core/device/agent.go
+++ b/rw_core/core/device/agent.go
@@ -416,7 +416,7 @@
if !agent.proceedWithRequestNoLock() {
agent.requestQueue.RequestComplete()
- desc = fmt.Sprintf("deviceId:%s, Device deletion or reconciling is in progress.", agent.deviceID)
+ desc = fmt.Sprintf("deviceId:%s, Cannot complete operation as Device deletion/reconciling is in progress or reconcile failed.", agent.deviceID)
return status.Error(codes.FailedPrecondition, desc)
}
// First figure out which adapter will handle this device type. We do it at this stage as allow devices to be
@@ -588,8 +588,8 @@
if !agent.proceedWithRequestNoLock() {
agent.requestQueue.RequestComplete()
- desc = fmt.Sprintf("deviceId:%s, Device deletion or reconciling is in progress.", agent.deviceID)
- return status.Errorf(codes.FailedPrecondition, "deviceId:%s, Device reconciling is in progress.", agent.deviceID)
+ desc = fmt.Sprintf("deviceId:%s,Cannot complete operation as Device deletion/reconciling is in progress or reconcile failed.", agent.deviceID)
+ return status.Errorf(codes.FailedPrecondition, desc)
}
// Update the Admin State and operational state before sending the request out
@@ -634,8 +634,8 @@
device := agent.getDeviceReadOnlyWithoutLock()
if !agent.proceedWithRequestNoLock() {
- desc = fmt.Sprintf("deviceId:%s, Device delection or reconciling is in progress.", agent.deviceID)
- return status.Errorf(codes.FailedPrecondition, "deviceId:%s, Device reconciling is in progress.", agent.deviceID)
+ desc = fmt.Sprintf("deviceId:%s, Cannot complete operation as Device deletion/reconciling is in progress or reconcile failed.", agent.deviceID)
+ return status.Errorf(codes.FailedPrecondition, desc)
}
subCtx, cancel := context.WithTimeout(log.WithSpanFromContext(context.Background(), ctx), agent.defaultTimeout)
subCtx = coreutils.WithRPCMetadataFromContext(subCtx, ctx)
@@ -719,9 +719,9 @@
return err
}
- if agent.isReconcileInProgress() {
+ if agent.isInReconcileState() {
agent.requestQueue.RequestComplete()
- desc = fmt.Sprintf("deviceId:%s, Device Reconciling is in progress", agent.deviceID)
+ desc = fmt.Sprintf("deviceId:%s, Cannot complete operation as Reconciling is in progress or failed", agent.deviceID)
return status.Error(codes.FailedPrecondition, desc)
}
@@ -1264,7 +1264,7 @@
// The device lock MUST be held by the caller.
func (agent *Agent) proceedWithRequestNoLock() bool {
- return !agent.isDeletionInProgress() && !agent.isReconcileInProgress()
+ return !agent.isDeletionInProgress() && !agent.isInReconcileState()
}
func (agent *Agent) stopReconcile() {
@@ -1287,7 +1287,7 @@
if !agent.proceedWithRequestNoLock() {
agent.requestQueue.RequestComplete()
- desc = fmt.Sprintf("Either device is in deletion or reconcile is already in progress for device : %s", device.Id)
+ desc = fmt.Sprintf("Cannot complete operation as Device deletion/reconciling is in progress or reconcile failed for device : %s", device.Id)
logger.Errorf(ctx, desc)
agent.logDeviceUpdate(ctx, "Reconciling", nil, nil, operStatus, &desc)
return
diff --git a/rw_core/core/device/agent_image.go b/rw_core/core/device/agent_image.go
index 075e2bf..fc56c02 100644
--- a/rw_core/core/device/agent_image.go
+++ b/rw_core/core/device/agent_image.go
@@ -44,7 +44,7 @@
}
if !agent.proceedWithRequestNoLock() {
agent.requestQueue.RequestComplete()
- return nil, status.Errorf(codes.FailedPrecondition, "deviceId:%s, Device reconciling or deletion is in progress.",
+ return nil, status.Errorf(codes.FailedPrecondition, "deviceId:%s, Cannot complete operation as Device deletion/reconciling is in progress or reconcile failed.",
agent.deviceID)
}
@@ -107,7 +107,7 @@
if !agent.proceedWithRequestNoLock() {
agent.requestQueue.RequestComplete()
- return nil, status.Errorf(codes.FailedPrecondition, "deviceId:%s, Device reconciling or deletion is in progress.",
+ return nil, status.Errorf(codes.FailedPrecondition, "deviceId:%s, Cannot complete operation as Device deletion/reconciling is in progress or reconcile failed.",
agent.deviceID)
}
@@ -151,7 +151,7 @@
if !agent.proceedWithRequestNoLock() {
agent.requestQueue.RequestComplete()
- return nil, status.Errorf(codes.FailedPrecondition, "deviceId:%s, Device reconciling or deletion is in progress.",
+ return nil, status.Errorf(codes.FailedPrecondition, "deviceId:%s, Cannot complete operation as Device deletion/reconciling is in progress or reconcile failed.",
agent.deviceID)
}
@@ -270,7 +270,7 @@
if !agent.proceedWithRequestNoLock() {
agent.requestQueue.RequestComplete()
- return status.Errorf(codes.FailedPrecondition, "deviceId:%s, Device reconciling or deletion is in progress.",
+ return status.Errorf(codes.FailedPrecondition, "deviceId:%s, Cannot complete operation as Device deletion/reconciling is in progress or reconcile failed.",
agent.deviceID)
}
@@ -333,6 +333,8 @@
}
if !agent.proceedWithRequestNoLock() {
agent.requestQueue.RequestComplete()
+ logger.Errorw(subCtx, "Cannot complete operation as Device deletion/reconciling is in progress or reconcile failed.",
+ log.Fields{"rpc": rpc, "device-id": agent.deviceID})
return
}
if res, ok := response.(error); ok {
@@ -383,6 +385,8 @@
}
if !agent.proceedWithRequestNoLock() {
agent.requestQueue.RequestComplete()
+ logger.Errorw(ctx, "Cannot complete operation as Device deletion/reconciling is in progress or reconcile failed.",
+ log.Fields{"rpc": rpc, "device-id": agent.deviceID})
return
}
logger.Infow(ctx, "rpc-successful", log.Fields{"rpc": rpc, "device-id": agent.deviceID, "response": response, "args": reqArgs})
diff --git a/rw_core/core/device/agent_transient_state.go b/rw_core/core/device/agent_transient_state.go
index 6b5b100..6f75ecc 100644
--- a/rw_core/core/device/agent_transient_state.go
+++ b/rw_core/core/device/agent_transient_state.go
@@ -74,8 +74,8 @@
return nil
}
-func (agent *Agent) isReconcileInProgress() bool {
+func (agent *Agent) isInReconcileState() bool {
device := agent.getDeviceReadOnlyWithoutLock()
- return device.OperStatus == common.OperStatus_RECONCILING ||
+ return device.OperStatus == common.OperStatus_RECONCILING || device.OperStatus == common.OperStatus_RECONCILING_FAILED ||
agent.matchTransientState(voltha.DeviceTransientState_RECONCILE_IN_PROGRESS)
}
diff --git a/rw_core/core/device/state/transitions.go b/rw_core/core/device/state/transitions.go
index 103697f..b843c70 100644
--- a/rw_core/core/device/state/transitions.go
+++ b/rw_core/core/device/state/transitions.go
@@ -325,6 +325,24 @@
previousState: deviceState{Admin: voltha.AdminState_DOWNLOADING_IMAGE, Connection: voltha.ConnectStatus_UNKNOWN, Operational: voltha.OperStatus_RECONCILING, Transient: voltha.DeviceTransientState_RECONCILE_IN_PROGRESS},
currentState: deviceState{Admin: voltha.AdminState_DOWNLOADING_IMAGE, Connection: voltha.ConnectStatus_UNKNOWN, Operational: voltha.OperStatus_UNKNOWN, Transient: voltha.DeviceTransientState_RECONCILE_IN_PROGRESS},
handlers: []transitionHandler{dMgr.ReconcilingCleanup}})
+ transitionMap.transitions = append(transitionMap.transitions,
+ transition{
+ deviceType: any,
+ previousState: deviceState{Admin: voltha.AdminState_ENABLED, Connection: voltha.ConnectStatus_UNKNOWN, Operational: voltha.OperStatus_RECONCILING, Transient: voltha.DeviceTransientState_RECONCILE_IN_PROGRESS},
+ currentState: deviceState{Admin: voltha.AdminState_ENABLED, Connection: voltha.ConnectStatus_UNKNOWN, Operational: voltha.OperStatus_RECONCILING_FAILED, Transient: voltha.DeviceTransientState_RECONCILE_IN_PROGRESS},
+ handlers: []transitionHandler{dMgr.ReconcilingCleanup}})
+ transitionMap.transitions = append(transitionMap.transitions,
+ transition{
+ deviceType: any,
+ previousState: deviceState{Admin: voltha.AdminState_DISABLED, Connection: voltha.ConnectStatus_UNKNOWN, Operational: voltha.OperStatus_RECONCILING, Transient: voltha.DeviceTransientState_RECONCILE_IN_PROGRESS},
+ currentState: deviceState{Admin: voltha.AdminState_DISABLED, Connection: voltha.ConnectStatus_UNKNOWN, Operational: voltha.OperStatus_RECONCILING_FAILED, Transient: voltha.DeviceTransientState_RECONCILE_IN_PROGRESS},
+ handlers: []transitionHandler{dMgr.ReconcilingCleanup}})
+ transitionMap.transitions = append(transitionMap.transitions,
+ transition{
+ deviceType: any,
+ previousState: deviceState{Admin: voltha.AdminState_DOWNLOADING_IMAGE, Connection: voltha.ConnectStatus_UNKNOWN, Operational: voltha.OperStatus_RECONCILING, Transient: voltha.DeviceTransientState_RECONCILE_IN_PROGRESS},
+ currentState: deviceState{Admin: voltha.AdminState_DOWNLOADING_IMAGE, Connection: voltha.ConnectStatus_UNKNOWN, Operational: voltha.OperStatus_RECONCILING_FAILED, Transient: voltha.DeviceTransientState_RECONCILE_IN_PROGRESS},
+ handlers: []transitionHandler{dMgr.ReconcilingCleanup}})
return &transitionMap
}
diff --git a/rw_core/core/device/state/transitions_test.go b/rw_core/core/device/state/transitions_test.go
index ee4986b..ad53526 100644
--- a/rw_core/core/device/state/transitions_test.go
+++ b/rw_core/core/device/state/transitions_test.go
@@ -271,6 +271,41 @@
assert.Equal(t, 1, len(handlers))
assert.True(t, reflect.ValueOf(tdm.ReconcilingCleanup).Pointer() == reflect.ValueOf(handlers[0]).Pointer())
+ previousDevice = getDevice(false, voltha.AdminState_ENABLED, voltha.ConnectStatus_UNREACHABLE, voltha.OperStatus_RECONCILING)
+ device = getDevice(false, voltha.AdminState_ENABLED, voltha.ConnectStatus_UNREACHABLE, voltha.OperStatus_RECONCILING_FAILED)
+ handlers = transitionMap.getTransitionHandler(ctx, device, previousDevice, voltha.DeviceTransientState_RECONCILE_IN_PROGRESS,
+ voltha.DeviceTransientState_RECONCILE_IN_PROGRESS)
+ assert.Equal(t, 1, len(handlers))
+ assert.True(t, reflect.ValueOf(tdm.ReconcilingCleanup).Pointer() == reflect.ValueOf(handlers[0]).Pointer())
+
+ previousDevice = getDevice(false, voltha.AdminState_DISABLED, voltha.ConnectStatus_UNREACHABLE, voltha.OperStatus_RECONCILING)
+ device = getDevice(false, voltha.AdminState_DISABLED, voltha.ConnectStatus_UNREACHABLE, voltha.OperStatus_RECONCILING_FAILED)
+ handlers = transitionMap.getTransitionHandler(ctx, device, previousDevice, voltha.DeviceTransientState_RECONCILE_IN_PROGRESS,
+ voltha.DeviceTransientState_RECONCILE_IN_PROGRESS)
+ assert.Equal(t, 1, len(handlers))
+ assert.True(t, reflect.ValueOf(tdm.ReconcilingCleanup).Pointer() == reflect.ValueOf(handlers[0]).Pointer())
+
+ previousDevice = getDevice(false, voltha.AdminState_DOWNLOADING_IMAGE, voltha.ConnectStatus_UNREACHABLE, voltha.OperStatus_RECONCILING)
+ device = getDevice(false, voltha.AdminState_DOWNLOADING_IMAGE, voltha.ConnectStatus_UNREACHABLE, voltha.OperStatus_RECONCILING_FAILED)
+ handlers = transitionMap.getTransitionHandler(ctx, device, previousDevice, voltha.DeviceTransientState_RECONCILE_IN_PROGRESS,
+ voltha.DeviceTransientState_RECONCILE_IN_PROGRESS)
+ assert.Equal(t, 1, len(handlers))
+ assert.True(t, reflect.ValueOf(tdm.ReconcilingCleanup).Pointer() == reflect.ValueOf(handlers[0]).Pointer())
+
+ previousDevice = getDevice(false, voltha.AdminState_ENABLED, voltha.ConnectStatus_REACHABLE, voltha.OperStatus_RECONCILING)
+ device = getDevice(false, voltha.AdminState_ENABLED, voltha.ConnectStatus_REACHABLE, voltha.OperStatus_RECONCILING_FAILED)
+ handlers = transitionMap.getTransitionHandler(ctx, device, previousDevice, voltha.DeviceTransientState_RECONCILE_IN_PROGRESS,
+ voltha.DeviceTransientState_RECONCILE_IN_PROGRESS)
+ assert.Equal(t, 1, len(handlers))
+ assert.True(t, reflect.ValueOf(tdm.ReconcilingCleanup).Pointer() == reflect.ValueOf(handlers[0]).Pointer())
+
+ previousDevice = getDevice(false, voltha.AdminState_ENABLED, voltha.ConnectStatus_REACHABLE, voltha.OperStatus_RECONCILING)
+ device = getDevice(false, voltha.AdminState_ENABLED, voltha.ConnectStatus_UNKNOWN, voltha.OperStatus_RECONCILING_FAILED)
+ handlers = transitionMap.getTransitionHandler(ctx, device, previousDevice, voltha.DeviceTransientState_RECONCILE_IN_PROGRESS,
+ voltha.DeviceTransientState_RECONCILE_IN_PROGRESS)
+ assert.Equal(t, 1, len(handlers))
+ assert.True(t, reflect.ValueOf(tdm.ReconcilingCleanup).Pointer() == reflect.ValueOf(handlers[0]).Pointer())
+
var deleteDeviceTest = struct {
previousDevices []*voltha.Device
devices []*voltha.Device