[VOL-5433] Added changes to fix the parent-id missing issue after OLT reboot
Signed-off-by: bseeniva <balaji.seenivasan@radisys.com>
Change-Id: I7de343c19401ea79f24c47e1f9acf9bf08504829
diff --git a/VERSION b/VERSION
index cff2619..c47e8b5 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-3.6.9
+3.6.10
diff --git a/go.mod b/go.mod
index 2e88ada..18cc57d 100644
--- a/go.mod
+++ b/go.mod
@@ -19,7 +19,7 @@
github.com/google/uuid v1.3.0
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
github.com/opencord/voltha-lib-go/v7 v7.6.6
- github.com/opencord/voltha-protos/v5 v5.6.2
+ github.com/opencord/voltha-protos/v5 v5.6.3
github.com/opentracing/opentracing-go v1.2.0
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2
github.com/stretchr/testify v1.7.0
diff --git a/go.sum b/go.sum
index c4ecfda..ec161a8 100644
--- a/go.sum
+++ b/go.sum
@@ -208,8 +208,9 @@
github.com/onsi/gomega v1.14.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0=
github.com/opencord/voltha-lib-go/v7 v7.6.6 h1:TrLo0nykH2MXPngKzTtmL9/u2gjxs98m1/F0m3FWY2U=
github.com/opencord/voltha-lib-go/v7 v7.6.6/go.mod h1:uGmArLg+nSZd49YXv7ZaD48FA5c+siEFxnyRuldwv6Y=
-github.com/opencord/voltha-protos/v5 v5.6.2 h1:evT3MYShV8uzLQAfF+qXS6nbdKRNewxifDK49XzqtVM=
github.com/opencord/voltha-protos/v5 v5.6.2/go.mod h1:E/Jn3DNu8VGRBCgIWSSg4sWtTBiNuQGSFvHyNH1XlyM=
+github.com/opencord/voltha-protos/v5 v5.6.3 h1:q0WxcH5JGDBKX4xDLZhxif4Hlg+oEXWpvdChay8MMRo=
+github.com/opencord/voltha-protos/v5 v5.6.3/go.mod h1:E/Jn3DNu8VGRBCgIWSSg4sWtTBiNuQGSFvHyNH1XlyM=
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
diff --git a/rw_core/core/device/agent.go b/rw_core/core/device/agent.go
index 649eb3f..e6292a2 100755
--- a/rw_core/core/device/agent.go
+++ b/rw_core/core/device/agent.go
@@ -1010,13 +1010,20 @@
_ = agent.deviceMgr.Agent.SendDeviceStateChangeEvent(ctx, prevDevice.OperStatus, prevDevice.ConnectStatus, prevDevice.AdminState, device, time.Now().Unix())
}
deviceTransientState := agent.getTransientState()
-
+ prevDeviceTransientState := deviceTransientState
+ // Update the transient state to olt reboot in progress if the device is root and oper status is rebooted
+ if device.Root && device.OperStatus == common.OperStatus_REBOOTED && prevDevice.OperStatus != common.OperStatus_REBOOTED {
+ if err := agent.updateTransientState(ctx, core.DeviceTransientState_REBOOT_IN_PROGRESS); err != nil {
+ return err
+ }
+ deviceTransientState = core.DeviceTransientState_REBOOT_IN_PROGRESS
+ }
// release lock before processing transition
agent.requestQueue.RequestComplete()
subCtx := coreutils.WithSpanAndRPCMetadataFromContext(ctx)
if err := agent.deviceMgr.stateTransitions.ProcessTransition(subCtx,
- device, prevDevice, deviceTransientState, deviceTransientState); err != nil {
+ device, prevDevice, deviceTransientState, prevDeviceTransientState); err != nil {
logger.Errorw(ctx, "failed-process-transition", log.Fields{"device-id": device.Id, "previous-admin-state": prevDevice.AdminState, "current-admin-state": device.AdminState})
// Sending RPC EVENT here
rpce := agent.deviceMgr.NewRPCEvent(ctx, agent.deviceID, err.Error(), nil)
@@ -1387,6 +1394,7 @@
case core.DeviceTransientState_DELETING_FROM_ADAPTER:
updatedState = core.DeviceTransientState_DELETE_FAILED
case core.DeviceTransientState_DELETE_FAILED:
+ case core.DeviceTransientState_REBOOT_IN_PROGRESS:
// do not change state
return nil
default:
@@ -1486,6 +1494,7 @@
func (agent *Agent) StartReconcileWithRetry(ctx context.Context) {
+ var reconcileErr error
state := agent.getTransientState()
logger.Debugw(ctx, "starting-reconcile", log.Fields{"device-id": agent.deviceID, "state": state})
@@ -1569,17 +1578,32 @@
// Release lock before sending request to adapter
agent.requestQueue.RequestComplete()
- // Send a reconcile request to the adapter.
- err := agent.sendReconcileRequestToAdapter(ctx, device)
+ if state != core.DeviceTransientState_REBOOT_IN_PROGRESS {
+ if state != core.DeviceTransientState_RECONCILE_IN_PROGRESS {
+ // set transient state to RECONCILE IN PROGRESS
+ if err := agent.UpdateTransientStateToReconcile(ctx); err != nil {
+ logger.Errorw(ctx, "setting-transient-state-failed", log.Fields{"error": err})
+ agent.stopReconcilingMutex.Lock()
+ if agent.stopReconciling != nil {
+ agent.stopReconciling = nil
+ }
+ desc = "Failed-to-update-transient-state"
+ agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc)
+ agent.stopReconcilingMutex.Unlock()
+ break retry
+ }
+ }
+ reconcileErr = agent.sendReconcileRequestToAdapter(ctx, device)
+ }
- if errors.Is(err, errContextExpired) || errors.Is(err, errReconcileAborted) {
- logger.Errorw(ctx, "reconcile-aborted", log.Fields{"error": err})
+ if errors.Is(reconcileErr, errContextExpired) || errors.Is(reconcileErr, errReconcileAborted) {
+ logger.Errorw(ctx, "reconcile-aborted", log.Fields{"error": reconcileErr})
requestStatus = &common.OperationResp{Code: common.OperationResp_OperationReturnCode(common.OperStatus_FAILED)}
desc = "aborted"
- agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc)
+ agent.logDeviceUpdate(ctx, nil, nil, requestStatus, reconcileErr, desc)
break retry
}
- st, ok := status.FromError(err)
+ st, ok := status.FromError(reconcileErr)
if ok {
// Decode the error code and error message
errorCode := st.Code()
@@ -1596,7 +1620,7 @@
Either of the cases we retry for retryinterval and assume that the device is already reconciled in the new adapter.
*/
if errorCode == codes.AlreadyExists {
- logger.Warnw(ctx, "device already reconciled", log.Fields{"error": err})
+ logger.Warnw(ctx, "device already reconciled", log.Fields{"error": reconcileErr})
// Reset the backoff current interval when we receive the device reconciled error for the first time
// so that we can retry for one full backoff cycle and then break
if !isDeviceReconciledErr {
@@ -1607,17 +1631,19 @@
isDeviceReconciledErr = false
}
}
- if err != nil {
- agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc)
+ // If the reconcile request fails or the device transient state is in OLT reboot in progress state, retry the reconcile
+ if reconcileErr != nil || state == core.DeviceTransientState_REBOOT_IN_PROGRESS {
+ agent.logDeviceUpdate(ctx, nil, nil, requestStatus, reconcileErr, desc)
select {
case <-backoffTimer.C:
// backoffTimer expired continue
// Take lock back before retrying
- if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
+ if reconcileErr = agent.requestQueue.WaitForGreenLight(ctx); reconcileErr != nil {
desc = "failed-to-acquire-lock"
- agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc)
+ agent.logDeviceUpdate(ctx, nil, nil, requestStatus, reconcileErr, desc)
break retry
}
+ state = agent.getTransientState()
continue
case _, ok := (<-agent.exitChannel):
@@ -1625,7 +1651,7 @@
if !ok {
logger.Warnw(ctx, "device agent stopped, aborting reconcile", log.Fields{"device-id": agent.deviceID})
desc = "device-agent-stopped"
- agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc)
+ agent.logDeviceUpdate(ctx, nil, nil, requestStatus, reconcileErr, desc)
break retry
}
}
@@ -1633,7 +1659,7 @@
// Success
requestStatus = &common.OperationResp{Code: common.OperationResp_OPERATION_SUCCESS}
desc = "adapter-response"
- agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc)
+ agent.logDeviceUpdate(ctx, nil, nil, requestStatus, reconcileErr, desc)
break retry
}
@@ -1659,13 +1685,6 @@
func (agent *Agent) ReconcileDevice(ctx context.Context) {
- // set transient state to RECONCILE IN PROGRESS
- err := agent.UpdateTransientStateToReconcile(ctx)
- if err != nil {
- logger.Errorw(ctx, "check-and-update-transient-state-failed", log.Fields{"error": err})
- return
- }
-
agent.StartReconcileWithRetry(ctx)
}
diff --git a/rw_core/core/device/manager.go b/rw_core/core/device/manager.go
index bf68509..1a2e5f6 100755
--- a/rw_core/core/device/manager.go
+++ b/rw_core/core/device/manager.go
@@ -401,11 +401,6 @@
"device-type": deviceAgent.deviceType,
"adapter-type": adapter.Type,
})
- // set transient state to RECONCILE IN PROGRESS
- err = deviceAgent.UpdateTransientStateToReconcile(ctx)
- if err != nil {
- logger.Errorw(ctx, "setting-transient-state-failed", log.Fields{"error": err})
- }
} else {
logger.Errorw(ctx, "failed-aborting-exisiting-processing", log.Fields{"error": err})
}
diff --git a/rw_core/core/device/manager_state_callback.go b/rw_core/core/device/manager_state_callback.go
index f4f6754..62f79e6 100644
--- a/rw_core/core/device/manager_state_callback.go
+++ b/rw_core/core/device/manager_state_callback.go
@@ -211,3 +211,16 @@
return nil
}
+
+// ClearTransientState updates the parent device transient state to none
+func (dMgr *Manager) ClearDeviceTransientState(ctx context.Context, parentDevice *voltha.Device) error {
+ logger.Debugw(ctx, "clear-device-transient-state", log.Fields{"parent-device-id": parentDevice.Id})
+ if agent := dMgr.getDeviceAgent(ctx, parentDevice.Id); agent != nil {
+ // Update the transient state to none if the device is root and oper status is active and transient state is olt reboot in progress
+ if err := agent.updateTransientState(ctx, core.DeviceTransientState_NONE); err != nil {
+ return err
+ }
+ return nil
+ }
+ return status.Errorf(codes.NotFound, "%s", parentDevice.Id)
+}
diff --git a/rw_core/core/device/state/transitions.go b/rw_core/core/device/state/transitions.go
index ebc720c..8b5da90 100644
--- a/rw_core/core/device/state/transitions.go
+++ b/rw_core/core/device/state/transitions.go
@@ -104,6 +104,7 @@
DeleteAllLogicalMeters(ctx context.Context, curr *voltha.Device) error
DeleteAllDeviceFlows(ctx context.Context, curr *voltha.Device) error
ReconcilingCleanup(ctx context.Context, curr *voltha.Device) error
+ ClearDeviceTransientState(ctx context.Context, curr *voltha.Device) error
}
// NewTransitionMap creates transition map
@@ -182,8 +183,8 @@
transition{
deviceType: parent,
previousState: deviceState{Admin: voltha.AdminState_ENABLED, Connection: voltha.ConnectStatus_UNREACHABLE, Operational: voltha.OperStatus_UNKNOWN, Transient: core.DeviceTransientState_NONE},
- currentState: deviceState{Admin: voltha.AdminState_ENABLED, Connection: voltha.ConnectStatus_REACHABLE, Operational: voltha.OperStatus_REBOOTED, Transient: core.DeviceTransientState_NONE},
- handlers: []transitionHandler{dMgr.DeleteAllLogicalPorts, dMgr.DeleteAllChildDevices, dMgr.DeleteAllLogicalMeters, dMgr.DeleteLogicalDevice, dMgr.DeleteAllDeviceFlows}})
+ currentState: deviceState{Admin: voltha.AdminState_ENABLED, Connection: voltha.ConnectStatus_REACHABLE, Operational: voltha.OperStatus_REBOOTED, Transient: core.DeviceTransientState_REBOOT_IN_PROGRESS},
+ handlers: []transitionHandler{dMgr.DeleteAllLogicalPorts, dMgr.DeleteAllChildDevices, dMgr.DeleteAllLogicalMeters, dMgr.DeleteLogicalDevice, dMgr.DeleteAllDeviceFlows, dMgr.ClearDeviceTransientState}})
transitionMap.transitions = append(transitionMap.transitions,
transition{
deviceType: parent,
diff --git a/vendor/github.com/opencord/voltha-protos/v5/go/core/core.pb.go b/vendor/github.com/opencord/voltha-protos/v5/go/core/core.pb.go
index e01c94e..dab601d 100644
--- a/vendor/github.com/opencord/voltha-protos/v5/go/core/core.pb.go
+++ b/vendor/github.com/opencord/voltha-protos/v5/go/core/core.pb.go
@@ -39,6 +39,8 @@
DeviceTransientState_DELETE_FAILED DeviceTransientState_Types = 5
// State to represent that reconcile is in progress
DeviceTransientState_RECONCILE_IN_PROGRESS DeviceTransientState_Types = 6
+ // State to represent that reboot is in progress
+ DeviceTransientState_REBOOT_IN_PROGRESS DeviceTransientState_Types = 7
)
var DeviceTransientState_Types_name = map[int32]string{
@@ -49,6 +51,7 @@
4: "DELETING_POST_ADAPTER_RESPONSE",
5: "DELETE_FAILED",
6: "RECONCILE_IN_PROGRESS",
+ 7: "REBOOT_IN_PROGRESS",
}
var DeviceTransientState_Types_value = map[string]int32{
@@ -59,6 +62,7 @@
"DELETING_POST_ADAPTER_RESPONSE": 4,
"DELETE_FAILED": 5,
"RECONCILE_IN_PROGRESS": 6,
+ "REBOOT_IN_PROGRESS": 7,
}
func (x DeviceTransientState_Types) String() string {
@@ -116,23 +120,24 @@
func init() { proto.RegisterFile("voltha_protos/core.proto", fileDescriptor_39634f15fb8a505e) }
var fileDescriptor_39634f15fb8a505e = []byte{
- // 286 bytes of a gzipped FileDescriptorProto
- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x90, 0xdf, 0x4a, 0xc3, 0x30,
- 0x18, 0xc5, 0xad, 0xfb, 0xa3, 0x04, 0x9c, 0x31, 0x28, 0xcc, 0x1b, 0x19, 0xbb, 0xf2, 0x42, 0x53,
- 0x50, 0x7c, 0x80, 0xba, 0x7e, 0x9b, 0x85, 0x99, 0x94, 0xa4, 0x37, 0x7a, 0x13, 0xba, 0x1a, 0xba,
- 0x82, 0x36, 0xa5, 0x8d, 0x05, 0x9f, 0xc5, 0x87, 0x55, 0x9a, 0x59, 0x41, 0xf0, 0xee, 0xe4, 0x77,
- 0x4e, 0x0e, 0x7c, 0x07, 0x4d, 0x5b, 0xf3, 0x6a, 0xb7, 0xa9, 0xaa, 0x6a, 0x63, 0x4d, 0xe3, 0x67,
- 0xa6, 0xd6, 0xd4, 0x69, 0x32, 0xec, 0xf4, 0xfc, 0xcb, 0x43, 0xa7, 0xa1, 0x6e, 0x8b, 0x4c, 0x27,
- 0x75, 0x5a, 0x36, 0x85, 0x2e, 0xad, 0xb4, 0xa9, 0xd5, 0x24, 0x42, 0xc7, 0xb6, 0x27, 0xaa, 0xe9,
- 0xd0, 0xd4, 0x9b, 0x79, 0x97, 0x93, 0x9b, 0x19, 0x75, 0x25, 0xff, 0x7d, 0xa2, 0xc9, 0x47, 0xa5,
- 0x1b, 0x31, 0xb1, 0x7f, 0xe8, 0xfc, 0xd3, 0x43, 0x23, 0xe7, 0x90, 0x43, 0x34, 0x64, 0x9c, 0x01,
- 0xde, 0x23, 0x07, 0x68, 0x10, 0xb0, 0x27, 0xec, 0x11, 0x82, 0x26, 0x4b, 0x2e, 0x16, 0xa0, 0x42,
- 0x58, 0x43, 0x12, 0xb1, 0x15, 0xde, 0x27, 0xe7, 0xe8, 0xac, 0x7f, 0xa9, 0xa5, 0xe0, 0x8f, 0x2a,
- 0x08, 0x83, 0x38, 0x01, 0x81, 0x07, 0x64, 0x8e, 0x2e, 0x7e, 0xad, 0x98, 0xcb, 0xa4, 0xb7, 0x94,
- 0x00, 0x19, 0x73, 0x26, 0x01, 0x0f, 0xc9, 0x09, 0x3a, 0x72, 0x19, 0x50, 0xcb, 0x20, 0x5a, 0x43,
- 0x88, 0x47, 0x5d, 0xa3, 0x80, 0x05, 0x67, 0x8b, 0x68, 0x0d, 0x2a, 0x62, 0x2a, 0x16, 0x7c, 0x25,
- 0x40, 0x4a, 0x3c, 0xbe, 0x7f, 0x40, 0x53, 0x53, 0xe7, 0xd4, 0x54, 0xba, 0xcc, 0x4c, 0xfd, 0x42,
- 0x77, 0x83, 0xb9, 0x23, 0x9f, 0xaf, 0xf2, 0xc2, 0x6e, 0xdf, 0x37, 0x34, 0x33, 0x6f, 0x7e, 0x1f,
- 0xf0, 0x77, 0x81, 0xeb, 0x9f, 0x45, 0xdb, 0x3b, 0x3f, 0x37, 0x6e, 0xd7, 0xcd, 0xd8, 0xa1, 0xdb,
- 0xef, 0x00, 0x00, 0x00, 0xff, 0xff, 0x1c, 0x32, 0x24, 0x57, 0x74, 0x01, 0x00, 0x00,
+ // 299 bytes of a gzipped FileDescriptorProto
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x90, 0xcf, 0x4a, 0x33, 0x31,
+ 0x14, 0xc5, 0xbf, 0xe9, 0xdf, 0x8f, 0x0b, 0xd6, 0x18, 0x54, 0xea, 0x46, 0x4a, 0x57, 0x2e, 0x74,
+ 0x06, 0x14, 0x1f, 0x60, 0xda, 0xb9, 0xad, 0x03, 0x35, 0x19, 0x92, 0x6c, 0x74, 0x13, 0xda, 0x31,
+ 0xb4, 0x05, 0x6d, 0xca, 0x4c, 0x2c, 0xf8, 0x24, 0x3e, 0x88, 0x2f, 0x28, 0x93, 0x3a, 0x42, 0xc1,
+ 0xdd, 0xc9, 0xef, 0x9c, 0x1c, 0xb8, 0x07, 0xfa, 0x3b, 0xfb, 0xea, 0x56, 0x73, 0xbd, 0x2d, 0xac,
+ 0xb3, 0x65, 0x94, 0xdb, 0xc2, 0x84, 0x5e, 0xd3, 0x56, 0xa5, 0x87, 0x9f, 0x0d, 0x38, 0x4d, 0xcc,
+ 0x6e, 0x9d, 0x1b, 0x55, 0xcc, 0x37, 0xe5, 0xda, 0x6c, 0x9c, 0x74, 0x73, 0x67, 0x68, 0x0a, 0xc7,
+ 0xae, 0x26, 0xba, 0xac, 0x50, 0x3f, 0x18, 0x04, 0x57, 0xbd, 0xdb, 0x41, 0xe8, 0x4b, 0xfe, 0xfa,
+ 0x14, 0xaa, 0x8f, 0xad, 0x29, 0x45, 0xcf, 0x1d, 0xd0, 0xe1, 0x57, 0x00, 0x6d, 0xef, 0xd0, 0xff,
+ 0xd0, 0x62, 0x9c, 0x21, 0xf9, 0x47, 0xbb, 0xd0, 0x8c, 0xd9, 0x13, 0x09, 0x28, 0x85, 0xde, 0x84,
+ 0x8b, 0x31, 0xea, 0x04, 0x67, 0xa8, 0x52, 0x36, 0x25, 0x0d, 0x7a, 0x01, 0x67, 0xf5, 0x4b, 0x4f,
+ 0x04, 0x7f, 0xd4, 0x71, 0x12, 0x67, 0x0a, 0x05, 0x69, 0xd2, 0x21, 0x5c, 0xfe, 0x5a, 0x19, 0x97,
+ 0xaa, 0xb6, 0xb4, 0x40, 0x99, 0x71, 0x26, 0x91, 0xb4, 0xe8, 0x09, 0x1c, 0xf9, 0x0c, 0xea, 0x49,
+ 0x9c, 0xce, 0x30, 0x21, 0xed, 0xaa, 0x51, 0xe0, 0x98, 0xb3, 0x71, 0x3a, 0x43, 0x9d, 0x32, 0x9d,
+ 0x09, 0x3e, 0x15, 0x28, 0x25, 0xe9, 0xd0, 0x73, 0xa0, 0x02, 0x47, 0x9c, 0xab, 0x03, 0xde, 0x1d,
+ 0x3d, 0x40, 0xdf, 0x16, 0xcb, 0xd0, 0x6e, 0xcd, 0x26, 0xb7, 0xc5, 0x4b, 0xb8, 0x1f, 0xd2, 0x1f,
+ 0xff, 0x7c, 0xbd, 0x5c, 0xbb, 0xd5, 0xfb, 0x22, 0xcc, 0xed, 0x5b, 0x54, 0x07, 0xa2, 0x7d, 0xe0,
+ 0xe6, 0x67, 0xe9, 0xdd, 0x7d, 0xb4, 0xb4, 0x7e, 0xef, 0x45, 0xc7, 0xa3, 0xbb, 0xef, 0x00, 0x00,
+ 0x00, 0xff, 0xff, 0xfc, 0xd7, 0xdf, 0xa2, 0x8c, 0x01, 0x00, 0x00,
}
diff --git a/vendor/modules.txt b/vendor/modules.txt
index 65132f5..a48dfe5 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -286,7 +286,7 @@
github.com/opencord/voltha-lib-go/v7/pkg/mocks/kafka
github.com/opencord/voltha-lib-go/v7/pkg/probe
github.com/opencord/voltha-lib-go/v7/pkg/version
-# github.com/opencord/voltha-protos/v5 v5.6.2
+# github.com/opencord/voltha-protos/v5 v5.6.3
## explicit; go 1.16
github.com/opencord/voltha-protos/v5/go/adapter_service
github.com/opencord/voltha-protos/v5/go/common