VOL-1774 Etcd Crash Handling
Change-Id: I1eeb726654c3972fd0a4fafae134607e5a810415
diff --git a/rw_core/core/adapter_manager.go b/rw_core/core/adapter_manager.go
index b5f0131..3155ab8 100644
--- a/rw_core/core/adapter_manager.go
+++ b/rw_core/core/adapter_manager.go
@@ -110,42 +110,65 @@
return &adapterMgr
}
-func (aMgr *AdapterManager) start(ctx context.Context) {
+func (aMgr *AdapterManager) start(ctx context.Context) error {
log.Info("starting-adapter-manager")
// Load the existing adapterAgents and device types - this will also ensure the correct paths have been
// created if there are no data in the dB to start
- aMgr.loadAdaptersAndDevicetypesInMemory()
+ err := aMgr.loadAdaptersAndDevicetypesInMemory()
+ if err != nil {
+ log.Errorw("Failed-to-load-adapters-and-device-types-in-memeory", log.Fields{"error": err})
+ return err
+ }
//// Create the proxies
- aMgr.adapterProxy = aMgr.clusterDataProxy.CreateProxy(context.Background(), "/adapters", false)
- aMgr.deviceTypeProxy = aMgr.clusterDataProxy.CreateProxy(context.Background(), "/device_types", false)
+ aMgr.adapterProxy, err = aMgr.clusterDataProxy.CreateProxy(context.Background(), "/adapters", false)
+ if err != nil {
+ log.Errorw("Failed-to-create-adapter-proxy", log.Fields{"error": err})
+ return err
+ }
+ aMgr.deviceTypeProxy, err = aMgr.clusterDataProxy.CreateProxy(context.Background(), "/device_types", false)
+ if err != nil {
+ log.Errorw("Failed-to-create-device-proxy", log.Fields{"error": err})
+ return err
+ }
// Register the callbacks
aMgr.adapterProxy.RegisterCallback(model.POST_UPDATE, aMgr.adapterUpdated)
aMgr.deviceTypeProxy.RegisterCallback(model.POST_UPDATE, aMgr.deviceTypesUpdated)
probe.UpdateStatusFromContext(ctx, "adapter-manager", probe.ServiceStatusRunning)
log.Info("adapter-manager-started")
+ return nil
}
//loadAdaptersAndDevicetypesInMemory loads the existing set of adapters and device types in memory
-func (aMgr *AdapterManager) loadAdaptersAndDevicetypesInMemory() {
+func (aMgr *AdapterManager) loadAdaptersAndDevicetypesInMemory() error {
// Load the adapters
- if adaptersIf := aMgr.clusterDataProxy.List(context.Background(), "/adapters", 0, false, ""); adaptersIf != nil {
+ adaptersIf, err := aMgr.clusterDataProxy.List(context.Background(), "/adapters", 0, false, "")
+ if err != nil {
+ log.Errorw("Failed-to-list-adapters-from-cluster-data-proxy", log.Fields{"error": err})
+ return err
+ }
+ if adaptersIf != nil {
for _, adapterIf := range adaptersIf.([]interface{}) {
if adapter, ok := adapterIf.(*voltha.Adapter); ok {
log.Debugw("found-existing-adapter", log.Fields{"adapterId": adapter.Id})
- aMgr.addAdapter(adapter, false)
+ return aMgr.addAdapter(adapter, false)
}
}
} else {
log.Debug("no-existing-adapter-found")
// No adapter data. In order to have a proxy setup for that path let's create a fake adapter
- aMgr.addAdapter(&voltha.Adapter{Id: SentinelAdapterID}, true)
+ return aMgr.addAdapter(&voltha.Adapter{Id: SentinelAdapterID}, true)
}
// Load the device types
- if deviceTypesIf := aMgr.clusterDataProxy.List(context.Background(), "/device_types", 0, false, ""); deviceTypesIf != nil {
+ deviceTypesIf, err := aMgr.clusterDataProxy.List(context.Background(), "/device_types", 0, false, "")
+ if err != nil {
+ log.Errorw("Failed-to-list-device-types-from-cluster-data-proxy", log.Fields{"error": err})
+ return err
+ }
+ if deviceTypesIf != nil {
dTypes := &voltha.DeviceTypes{Items: []*voltha.DeviceType{}}
for _, deviceTypeIf := range deviceTypesIf.([]interface{}) {
if dType, ok := deviceTypeIf.(*voltha.DeviceType); ok {
@@ -153,12 +176,12 @@
dTypes.Items = append(dTypes.Items, dType)
}
}
- aMgr.addDeviceTypes(dTypes, false)
- } else {
- log.Debug("no-existing-device-type-found")
- // No device types data. In order to have a proxy setup for that path let's create a fake device type
- aMgr.addDeviceTypes(&voltha.DeviceTypes{Items: []*voltha.DeviceType{{Id: SentinelDevicetypeID, Adapter: SentinelAdapterID}}}, true)
+ return aMgr.addDeviceTypes(dTypes, false)
}
+
+ log.Debug("no-existing-device-type-found")
+ // No device types data. In order to have a proxy setup for that path let's create a fake device type
+ return aMgr.addDeviceTypes(&voltha.DeviceTypes{Items: []*voltha.DeviceType{{Id: SentinelDevicetypeID, Adapter: SentinelAdapterID}}}, true)
}
//updateAdaptersAndDevicetypesInMemory loads the existing set of adapters and device types in memory
@@ -180,7 +203,12 @@
}
// Update the adapters
- if adaptersIf := aMgr.clusterDataProxy.List(context.Background(), "/adapters", 0, false, ""); adaptersIf != nil {
+ adaptersIf, err := aMgr.clusterDataProxy.List(context.Background(), "/adapters", 0, false, "")
+ if err != nil {
+ log.Errorw("failed-to-list-adapters-from-cluster-proxy", log.Fields{"error": err})
+ return
+ }
+ if adaptersIf != nil {
for _, adapterIf := range adaptersIf.([]interface{}) {
if adapter, ok := adapterIf.(*voltha.Adapter); ok {
log.Debugw("found-existing-adapter", log.Fields{"adapterId": adapter.Id})
@@ -191,7 +219,12 @@
aMgr.lockdDeviceTypeToAdapterMap.Lock()
defer aMgr.lockdDeviceTypeToAdapterMap.Unlock()
// Update the device types
- if deviceTypesIf := aMgr.clusterDataProxy.List(context.Background(), "/device_types", 0, false, ""); deviceTypesIf != nil {
+ deviceTypesIf, err := aMgr.clusterDataProxy.List(context.Background(), "/device_types", 0, false, "")
+ if err != nil {
+ log.Errorw("Failed-to-list-device-types-in-cluster-data-proxy", log.Fields{"error": err})
+ return
+ }
+ if deviceTypesIf != nil {
dTypes := &voltha.DeviceTypes{Items: []*voltha.DeviceType{}}
for _, deviceTypeIf := range deviceTypesIf.([]interface{}) {
if dType, ok := deviceTypeIf.(*voltha.DeviceType); ok {
@@ -202,7 +235,7 @@
}
}
-func (aMgr *AdapterManager) addAdapter(adapter *voltha.Adapter, saveToDb bool) {
+func (aMgr *AdapterManager) addAdapter(adapter *voltha.Adapter, saveToDb bool) error {
aMgr.lockAdaptersMap.Lock()
defer aMgr.lockAdaptersMap.Unlock()
log.Debugw("adding-adapter", log.Fields{"adapter": adapter})
@@ -211,8 +244,18 @@
aMgr.adapterAgents[adapter.Id] = newAdapterAgent(clonedAdapter, nil)
if saveToDb {
// Save the adapter to the KV store - first check if it already exist
- if kvAdapter := aMgr.clusterDataProxy.Get(context.Background(), "/adapters/"+adapter.Id, 0, false, ""); kvAdapter == nil {
- if added := aMgr.clusterDataProxy.AddWithID(context.Background(), "/adapters", adapter.Id, clonedAdapter, ""); added == nil {
+ kvAdapter, err := aMgr.clusterDataProxy.Get(context.Background(), "/adapters/"+adapter.Id, 0, false, "")
+ if err != nil {
+ log.Errorw("failed-to-get-adapters-from-cluster-proxy", log.Fields{"error": err})
+ return err
+ }
+ if kvAdapter == nil {
+ added, err := aMgr.clusterDataProxy.AddWithID(context.Background(), "/adapters", adapter.Id, clonedAdapter, "")
+ if err != nil {
+ log.Errorw("failed-to-save-adapter-to-cluster-proxy", log.Fields{"error": err})
+ return err
+ }
+ if added == nil {
//TODO: Errors when saving to KV would require a separate go routine to be launched and try the saving again
log.Errorw("failed-to-save-adapter", log.Fields{"adapter": adapter})
} else {
@@ -221,11 +264,12 @@
}
}
}
+ return nil
}
-func (aMgr *AdapterManager) addDeviceTypes(deviceTypes *voltha.DeviceTypes, saveToDb bool) {
+func (aMgr *AdapterManager) addDeviceTypes(deviceTypes *voltha.DeviceTypes, saveToDb bool) error {
if deviceTypes == nil {
- return
+ return fmt.Errorf("no-device-type")
}
log.Debugw("adding-device-types", log.Fields{"deviceTypes": deviceTypes})
aMgr.lockAdaptersMap.Lock()
@@ -245,10 +289,20 @@
if saveToDb {
// Save the device types to the KV store as well
for _, deviceType := range deviceTypes.Items {
- if dType := aMgr.clusterDataProxy.Get(context.Background(), "/device_types/"+deviceType.Id, 0, false, ""); dType == nil {
+ dType, err := aMgr.clusterDataProxy.Get(context.Background(), "/device_types/"+deviceType.Id, 0, false, "")
+ if err != nil {
+ log.Errorw("Failed-to--device-types-from-cluster-data-proxy", log.Fields{"error": err})
+ return err
+ }
+ if dType == nil {
// Does not exist - save it
clonedDType := (proto.Clone(deviceType)).(*voltha.DeviceType)
- if added := aMgr.clusterDataProxy.AddWithID(context.Background(), "/device_types", deviceType.Id, clonedDType, ""); added == nil {
+ added, err := aMgr.clusterDataProxy.AddWithID(context.Background(), "/device_types", deviceType.Id, clonedDType, "")
+ if err != nil {
+ log.Errorw("Failed-to-add-device-types-to-cluster-data-proxy", log.Fields{"error": err})
+ return err
+ }
+ if added == nil {
log.Errorw("failed-to-save-deviceType", log.Fields{"deviceType": deviceType})
} else {
log.Debugw("device-type-saved-to-KV-Store", log.Fields{"deviceType": deviceType})
@@ -256,6 +310,7 @@
}
}
}
+ return nil
}
func (aMgr *AdapterManager) listAdapters(ctx context.Context) (*voltha.Adapters, error) {
@@ -315,7 +370,7 @@
aMgr.deviceTypeToAdapterMap[deviceType.Id] = deviceType.Adapter
}
-func (aMgr *AdapterManager) registerAdapter(adapter *voltha.Adapter, deviceTypes *voltha.DeviceTypes) *voltha.CoreInstance {
+func (aMgr *AdapterManager) registerAdapter(adapter *voltha.Adapter, deviceTypes *voltha.DeviceTypes) (*voltha.CoreInstance, error) {
log.Debugw("registerAdapter", log.Fields{"adapter": adapter, "deviceTypes": deviceTypes.Items})
if aMgr.getAdapter(adapter.Id) != nil {
@@ -326,15 +381,21 @@
log.Errorw("unable-to-restart-adapter", log.Fields{"error": err})
}
}()
- return &voltha.CoreInstance{InstanceId: aMgr.coreInstanceID}
+ return &voltha.CoreInstance{InstanceId: aMgr.coreInstanceID}, nil
}
// Save the adapter and the device types
- aMgr.addAdapter(adapter, true)
- aMgr.addDeviceTypes(deviceTypes, true)
+ if err := aMgr.addAdapter(adapter, true); err != nil {
+ log.Errorw("failed-to-add-adapter", log.Fields{"error": err})
+ return nil, err
+ }
+ if err := aMgr.addDeviceTypes(deviceTypes, true); err != nil {
+ log.Errorw("failed-to-add-device-types", log.Fields{"error": err})
+ return nil, err
+ }
log.Debugw("adapter-registered", log.Fields{"adapter": adapter.Id})
- return &voltha.CoreInstance{InstanceId: aMgr.coreInstanceID}
+ return &voltha.CoreInstance{InstanceId: aMgr.coreInstanceID}, nil
}
//getAdapterName returns the name of the device adapter that service this device type