blob: 2c69b2806875b610355f06c194d9c16d720a0f51 [file] [log] [blame]
khenaidoob9203542018-09-17 22:56:37 -04001/*
2 * Copyright 2018-present Open Networking Foundation
3
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7
8 * http://www.apache.org/licenses/LICENSE-2.0
9
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
npujar1d86a522019-11-14 17:11:16 +053016
Kent Hagerman2b216042020-04-03 18:28:56 -040017package device
khenaidoob9203542018-09-17 22:56:37 -040018
19import (
20 "context"
Matteo Scandolo360605d2019-11-05 18:29:17 -080021 "encoding/hex"
Kent Hagerman4f355f52020-03-30 16:01:33 -040022 "errors"
khenaidoo3ab34882019-05-02 21:33:30 -040023 "fmt"
Mahir Gunyelb0343bf2021-05-11 14:14:26 -070024 "reflect"
25 "sync"
26 "time"
27
khenaidoo9beaaf12021-10-19 17:32:01 -040028 "github.com/opencord/voltha-protos/v5/go/adapter_service"
khenaidood948f772021-08-11 17:49:24 -040029 "github.com/opencord/voltha-protos/v5/go/core"
khenaidoo9beaaf12021-10-19 17:32:01 -040030 "github.com/opencord/voltha-protos/v5/go/omci"
khenaidood948f772021-08-11 17:49:24 -040031
Maninder0aabf0c2021-03-17 14:55:14 +053032 "github.com/cenkalti/backoff/v3"
Maninder9a1bc0d2020-10-26 11:34:02 +053033 "github.com/gogo/protobuf/proto"
Maninder9a1bc0d2020-10-26 11:34:02 +053034 "github.com/golang/protobuf/ptypes/empty"
Maninder0aabf0c2021-03-17 14:55:14 +053035 "github.com/opencord/voltha-go/rw_core/config"
khenaidoo68a5e0c2021-11-06 13:08:03 -040036 "github.com/opencord/voltha-go/rw_core/utils"
Maninder9a1bc0d2020-10-26 11:34:02 +053037 "google.golang.org/grpc/codes"
38 "google.golang.org/grpc/status"
Mahir Gunyeladdb66a2020-04-29 18:08:50 -070039
Himani Chawla2ba1c9c2020-10-07 13:19:03 +053040 "github.com/opencord/voltha-go/db/model"
Kent Hagerman2b216042020-04-03 18:28:56 -040041 "github.com/opencord/voltha-go/rw_core/core/adapter"
Mahir Gunyel03de0d32020-06-03 01:36:59 -070042 "github.com/opencord/voltha-go/rw_core/core/device/flow"
43 "github.com/opencord/voltha-go/rw_core/core/device/group"
Kent Hagerman2a07b862020-06-19 15:23:07 -040044 "github.com/opencord/voltha-go/rw_core/core/device/port"
Himani Chawla2ba1c9c2020-10-07 13:19:03 +053045 "github.com/opencord/voltha-go/rw_core/core/device/transientstate"
Scott Bakerb671a862019-10-24 10:53:40 -070046 coreutils "github.com/opencord/voltha-go/rw_core/utils"
khenaidood948f772021-08-11 17:49:24 -040047 "github.com/opencord/voltha-lib-go/v7/pkg/log"
48 "github.com/opencord/voltha-protos/v5/go/common"
khenaidoo9beaaf12021-10-19 17:32:01 -040049 ca "github.com/opencord/voltha-protos/v5/go/core_adapter"
khenaidood948f772021-08-11 17:49:24 -040050 "github.com/opencord/voltha-protos/v5/go/extension"
khenaidood948f772021-08-11 17:49:24 -040051 ofp "github.com/opencord/voltha-protos/v5/go/openflow_13"
52 "github.com/opencord/voltha-protos/v5/go/voltha"
khenaidoob9203542018-09-17 22:56:37 -040053)
54
khenaidood948f772021-08-11 17:49:24 -040055var errReconcileAborted = errors.New("reconcile aborted")
56var errContextExpired = errors.New("context expired")
khenaidoo68a5e0c2021-11-06 13:08:03 -040057var errNoConnection = errors.New("no connection")
khenaidood948f772021-08-11 17:49:24 -040058
Kent Hagerman2b216042020-04-03 18:28:56 -040059// Agent represents device agent attributes
60type Agent struct {
Maninder0aabf0c2021-03-17 14:55:14 +053061 deviceID string
62 parentID string
63 deviceType string
khenaidood948f772021-08-11 17:49:24 -040064 adapterEndpoint string
Maninder0aabf0c2021-03-17 14:55:14 +053065 isRootDevice bool
Maninder0aabf0c2021-03-17 14:55:14 +053066 adapterMgr *adapter.Manager
67 deviceMgr *Manager
68 dbProxy *model.Proxy
69 exitChannel chan int
70 device *voltha.Device
71 requestQueue *coreutils.RequestQueue
khenaidood948f772021-08-11 17:49:24 -040072 internalTimeout time.Duration
73 rpcTimeout time.Duration
Himani Chawla4b4bd252021-11-08 15:59:40 +053074 flowTimeout time.Duration
Maninder0aabf0c2021-03-17 14:55:14 +053075 startOnce sync.Once
76 stopOnce sync.Once
77 stopped bool
78 stopReconciling chan int
79 stopReconcilingMutex sync.RWMutex
80 config *config.RWCoreFlags
Mahir Gunyel03de0d32020-06-03 01:36:59 -070081
khenaidoo7585a962021-06-10 16:15:38 -040082 flowCache *flow.Cache
83 groupCache *group.Cache
Himani Chawla2ba1c9c2020-10-07 13:19:03 +053084 portLoader *port.Loader
85 transientStateLoader *transientstate.Loader
khenaidoob9203542018-09-17 22:56:37 -040086}
87
Kent Hagerman2b216042020-04-03 18:28:56 -040088//newAgent creates a new device agent. The device will be initialized when start() is called.
Himani Chawla4b4bd252021-11-08 15:59:40 +053089func newAgent(device *voltha.Device, deviceMgr *Manager, dbPath *model.Path, deviceProxy *model.Proxy, internalTimeout, rpcTimeout, flowTimeout time.Duration) *Agent {
Kent Hagerman2a07b862020-06-19 15:23:07 -040090 deviceID := device.Id
91 if deviceID == "" {
92 deviceID = coreutils.CreateDeviceID()
Stephane Barbarie1ab43272018-12-08 21:42:13 -050093 }
Scott Baker80678602019-11-14 16:57:36 -080094
Kent Hagerman2a07b862020-06-19 15:23:07 -040095 return &Agent{
Himani Chawla2ba1c9c2020-10-07 13:19:03 +053096 deviceID: deviceID,
Himani Chawla2ba1c9c2020-10-07 13:19:03 +053097 isRootDevice: device.Root,
98 parentID: device.ParentId,
99 deviceType: device.Type,
khenaidood948f772021-08-11 17:49:24 -0400100 adapterEndpoint: device.AdapterEndpoint,
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530101 deviceMgr: deviceMgr,
102 adapterMgr: deviceMgr.adapterMgr,
103 exitChannel: make(chan int, 1),
104 dbProxy: deviceProxy,
khenaidood948f772021-08-11 17:49:24 -0400105 internalTimeout: internalTimeout,
106 rpcTimeout: rpcTimeout,
Himani Chawla4b4bd252021-11-08 15:59:40 +0530107 flowTimeout: flowTimeout,
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530108 device: proto.Clone(device).(*voltha.Device),
109 requestQueue: coreutils.NewRequestQueue(),
Maninder0aabf0c2021-03-17 14:55:14 +0530110 config: deviceMgr.config,
khenaidoo7585a962021-06-10 16:15:38 -0400111 flowCache: flow.NewCache(),
112 groupCache: group.NewCache(),
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530113 portLoader: port.NewLoader(dbPath.SubPath("ports").Proxy(deviceID)),
114 transientStateLoader: transientstate.NewLoader(dbPath.SubPath("core").Proxy("transientstate"), deviceID),
Kent Hagerman2a07b862020-06-19 15:23:07 -0400115 }
khenaidoob9203542018-09-17 22:56:37 -0400116}
117
khenaidoo442e7c72020-03-10 16:13:48 -0400118// start() saves the device to the data model and registers for callbacks on that device if deviceToCreate!=nil.
119// Otherwise, it will load the data from the dB and setup the necessary callbacks and proxies. Returns the device that
Scott Baker80678602019-11-14 16:57:36 -0800120// was started.
khenaidoo7585a962021-06-10 16:15:38 -0400121func (agent *Agent) start(ctx context.Context, deviceExist bool, deviceToCreate *voltha.Device) (*voltha.Device, error) {
khenaidoo442e7c72020-03-10 16:13:48 -0400122 needToStart := false
123 if agent.startOnce.Do(func() { needToStart = true }); !needToStart {
Kent Hagermancba2f302020-07-28 13:37:36 -0400124 return agent.getDeviceReadOnly(ctx)
khenaidoo442e7c72020-03-10 16:13:48 -0400125 }
126 var startSucceeded bool
127 defer func() {
128 if !startSucceeded {
129 if err := agent.stop(ctx); err != nil {
Rohan Agrawal31f21802020-06-12 05:38:46 +0000130 logger.Errorw(ctx, "failed-to-cleanup-after-unsuccessful-start", log.Fields{"device-id": agent.deviceID, "error": err})
khenaidoo442e7c72020-03-10 16:13:48 -0400131 }
132 }
133 }()
khenaidoo7585a962021-06-10 16:15:38 -0400134 if deviceExist {
135 device := deviceToCreate
136 if device == nil {
137 // Load from dB
138 device = &voltha.Device{}
139 have, err := agent.dbProxy.Get(ctx, agent.deviceID, device)
140 if err != nil {
141 return nil, err
142 } else if !have {
143 return nil, status.Errorf(codes.NotFound, "device-%s", agent.deviceID)
144 }
khenaidood948f772021-08-11 17:49:24 -0400145 logger.Infow(ctx, "device-loaded-from-db", log.Fields{"device-id": agent.deviceID, "adapter-endpoint": device.AdapterEndpoint, "type": device.Type})
Thomas Lee Se5a44012019-11-07 20:32:24 +0530146 }
khenaidood948f772021-08-11 17:49:24 -0400147 agent.deviceType = device.Type
148 agent.adapterEndpoint = device.AdapterEndpoint
Kent Hagerman4f355f52020-03-30 16:01:33 -0400149 agent.device = proto.Clone(device).(*voltha.Device)
khenaidoo7585a962021-06-10 16:15:38 -0400150 // load the ports from KV to cache
Kent Hagerman2a07b862020-06-19 15:23:07 -0400151 agent.portLoader.Load(ctx)
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530152 agent.transientStateLoader.Load(ctx)
khenaidoo297cd252019-02-07 22:10:23 -0500153 } else {
Scott Baker80678602019-11-14 16:57:36 -0800154 // Create a new device
Maninder9a1bc0d2020-10-26 11:34:02 +0530155 var desc string
khenaidood948f772021-08-11 17:49:24 -0400156 var err error
Maninder9a1bc0d2020-10-26 11:34:02 +0530157 prevState := common.AdminState_UNKNOWN
158 currState := common.AdminState_UNKNOWN
khenaidood948f772021-08-11 17:49:24 -0400159 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
Maninder9a1bc0d2020-10-26 11:34:02 +0530160
khenaidood948f772021-08-11 17:49:24 -0400161 defer func() { agent.logDeviceUpdate(ctx, &prevState, &currState, requestStatus, err, desc) }()
Maninder9a1bc0d2020-10-26 11:34:02 +0530162
Kent Hagermanf5a67352020-04-30 15:15:26 -0400163 // Assumption is that AdminState, FlowGroups, and Flows are uninitialized since this
164 // is a new device, so populate them here before passing the device to ldProxy.Set.
Kent Hagerman2b216042020-04-03 18:28:56 -0400165 // agent.deviceId will also have been set during newAgent().
khenaidoo7585a962021-06-10 16:15:38 -0400166 device := (proto.Clone(deviceToCreate)).(*voltha.Device)
npujar1d86a522019-11-14 17:11:16 +0530167 device.Id = agent.deviceID
Scott Baker80678602019-11-14 16:57:36 -0800168 device.AdminState = voltha.AdminState_PREPROVISIONED
Maninder9a1bc0d2020-10-26 11:34:02 +0530169 currState = device.AdminState
Scott Baker80678602019-11-14 16:57:36 -0800170 if !deviceToCreate.GetRoot() && deviceToCreate.ProxyAddress != nil {
171 // Set the default vlan ID to the one specified by the parent adapter. It can be
172 // overwritten by the child adapter during a device update request
173 device.Vlan = deviceToCreate.ProxyAddress.ChannelId
174 }
175
khenaidood948f772021-08-11 17:49:24 -0400176 // Save the device to the model
177 if err = agent.dbProxy.Set(ctx, agent.deviceID, device); err != nil {
178 err = status.Errorf(codes.Aborted, "failed-adding-device-%s: %s", agent.deviceID, err)
179 return nil, err
khenaidoo297cd252019-02-07 22:10:23 -0500180 }
Mahir Gunyelb0343bf2021-05-11 14:14:26 -0700181 _ = agent.deviceMgr.Agent.SendDeviceStateChangeEvent(ctx, device.OperStatus, device.ConnectStatus, prevState, device, time.Now().Unix())
khenaidood948f772021-08-11 17:49:24 -0400182 requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
khenaidoo442e7c72020-03-10 16:13:48 -0400183 agent.device = device
khenaidoob9203542018-09-17 22:56:37 -0400184 }
khenaidoo442e7c72020-03-10 16:13:48 -0400185 startSucceeded = true
Rohan Agrawalcf12f202020-08-03 04:42:01 +0000186 log.EnrichSpan(ctx, log.Fields{"device-id": agent.deviceID})
Rohan Agrawal31f21802020-06-12 05:38:46 +0000187 logger.Debugw(ctx, "device-agent-started", log.Fields{"device-id": agent.deviceID})
khenaidoo442e7c72020-03-10 16:13:48 -0400188
Kent Hagermancba2f302020-07-28 13:37:36 -0400189 return agent.getDeviceReadOnly(ctx)
khenaidoob9203542018-09-17 22:56:37 -0400190}
191
khenaidoo4d4802d2018-10-04 21:59:49 -0400192// stop stops the device agent. Not much to do for now
Kent Hagerman2b216042020-04-03 18:28:56 -0400193func (agent *Agent) stop(ctx context.Context) error {
khenaidoo442e7c72020-03-10 16:13:48 -0400194 needToStop := false
195 if agent.stopOnce.Do(func() { needToStop = true }); !needToStop {
196 return nil
197 }
198 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
199 return err
200 }
201 defer agent.requestQueue.RequestComplete()
khenaidoo49085352020-01-13 19:15:43 -0500202
Himani Chawlab4c25912020-11-12 17:16:38 +0530203 logger.Infow(ctx, "stopping-device-agent", log.Fields{"device-id": agent.deviceID, "parent-id": agent.parentID})
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530204 // Remove the device transient loader
205 if err := agent.deleteTransientState(ctx); err != nil {
206 return err
207 }
khenaidoo0a822f92019-05-08 15:15:57 -0400208 // Remove the device from the KV store
Kent Hagermanf5a67352020-04-30 15:15:26 -0400209 if err := agent.dbProxy.Remove(ctx, agent.deviceID); err != nil {
khenaidoo442e7c72020-03-10 16:13:48 -0400210 return err
Thomas Lee Se5a44012019-11-07 20:32:24 +0530211 }
Abhilash Laxmeshwar75517ea2021-11-24 18:38:10 +0530212 //send the device event to the message bus
213 _ = agent.deviceMgr.Agent.SendDeviceDeletedEvent(ctx, agent.device, time.Now().Unix())
khenaidoo442e7c72020-03-10 16:13:48 -0400214
khenaidoo442e7c72020-03-10 16:13:48 -0400215 close(agent.exitChannel)
216
217 agent.stopped = true
218
Rohan Agrawal31f21802020-06-12 05:38:46 +0000219 logger.Infow(ctx, "device-agent-stopped", log.Fields{"device-id": agent.deviceID, "parent-id": agent.parentID})
khenaidoo442e7c72020-03-10 16:13:48 -0400220
221 return nil
khenaidoob9203542018-09-17 22:56:37 -0400222}
223
Scott Baker80678602019-11-14 16:57:36 -0800224// Load the most recent state from the KVStore for the device.
Kent Hagerman2b216042020-04-03 18:28:56 -0400225func (agent *Agent) reconcileWithKVStore(ctx context.Context) {
khenaidoo442e7c72020-03-10 16:13:48 -0400226 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
Rohan Agrawal31f21802020-06-12 05:38:46 +0000227 logger.Warnw(ctx, "request-aborted", log.Fields{"device-id": agent.deviceID, "error": err})
khenaidoo442e7c72020-03-10 16:13:48 -0400228 return
229 }
230 defer agent.requestQueue.RequestComplete()
Rohan Agrawal31f21802020-06-12 05:38:46 +0000231 logger.Debug(ctx, "reconciling-device-agent-devicetype")
Scott Baker80678602019-11-14 16:57:36 -0800232 // TODO: context timeout
Kent Hagerman4f355f52020-03-30 16:01:33 -0400233 device := &voltha.Device{}
Kent Hagermanf5a67352020-04-30 15:15:26 -0400234 if have, err := agent.dbProxy.Get(ctx, agent.deviceID, device); err != nil {
Rohan Agrawal31f21802020-06-12 05:38:46 +0000235 logger.Errorw(ctx, "kv-get-failed", log.Fields{"device-id": agent.deviceID, "error": err})
Thomas Lee Se5a44012019-11-07 20:32:24 +0530236 return
Kent Hagerman4f355f52020-03-30 16:01:33 -0400237 } else if !have {
238 return // not found in kv
Thomas Lee Se5a44012019-11-07 20:32:24 +0530239 }
Kent Hagerman4f355f52020-03-30 16:01:33 -0400240
khenaidood948f772021-08-11 17:49:24 -0400241 agent.deviceType = device.Type
Kent Hagerman4f355f52020-03-30 16:01:33 -0400242 agent.device = device
khenaidood948f772021-08-11 17:49:24 -0400243 agent.adapterEndpoint = device.AdapterEndpoint
Kent Hagerman2a07b862020-06-19 15:23:07 -0400244 agent.portLoader.Load(ctx)
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530245 agent.transientStateLoader.Load(ctx)
246
Rohan Agrawal31f21802020-06-12 05:38:46 +0000247 logger.Debugw(ctx, "reconciled-device-agent-devicetype", log.Fields{"device-id": agent.deviceID, "type": agent.deviceType})
Scott Baker80678602019-11-14 16:57:36 -0800248}
249
khenaidoo442e7c72020-03-10 16:13:48 -0400250// onSuccess is a common callback for scenarios where we receive a nil response following a request to an adapter
khenaidood948f772021-08-11 17:49:24 -0400251func (agent *Agent) onSuccess(ctx context.Context, prevState, currState *common.AdminState_Types, deviceUpdateLog bool) {
252 if deviceUpdateLog {
253 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_SUCCESS}
254 desc := "adapter-response"
255 agent.logDeviceUpdate(ctx, prevState, currState, requestStatus, nil, desc)
256 return
257 }
258 logger.Debugw(ctx, "successful-operation", log.Fields{"device-id": agent.deviceID, "rpc": coreutils.GetRPCMetadataFromContext(ctx)})
khenaidoo442e7c72020-03-10 16:13:48 -0400259}
260
261// onFailure is a common callback for scenarios where we receive an error response following a request to an adapter
262// and the only action required is to publish the failed result on kafka
khenaidood948f772021-08-11 17:49:24 -0400263func (agent *Agent) onFailure(ctx context.Context, err error, prevState, currState *common.AdminState_Types, deviceUpdateLog bool) {
264 // Send an event on kafka
265 rpce := agent.deviceMgr.NewRPCEvent(ctx, agent.deviceID, err.Error(), nil)
266 go agent.deviceMgr.SendRPCEvent(ctx, "RPC_ERROR_RAISE_EVENT", rpce,
267 voltha.EventCategory_COMMUNICATION, nil, time.Now().Unix())
khenaidoo442e7c72020-03-10 16:13:48 -0400268
khenaidood948f772021-08-11 17:49:24 -0400269 // Log the device update event
270 if deviceUpdateLog {
271 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
272 desc := "adapter-response"
273 agent.logDeviceUpdate(ctx, prevState, currState, requestStatus, err, desc)
274 return
khenaidoo442e7c72020-03-10 16:13:48 -0400275 }
khenaidood948f772021-08-11 17:49:24 -0400276 logger.Errorw(ctx, "failed-operation", log.Fields{"error": err, "device-id": agent.deviceID, "rpc": coreutils.GetRPCMetadataFromContext(ctx)})
khenaidoo442e7c72020-03-10 16:13:48 -0400277}
278
khenaidoo6cd8e862021-11-12 16:33:43 -0500279// onForceDeleteResponse is invoked following a force delete request to an adapter.
280func (agent *Agent) onForceDeleteResponse(ctx context.Context, prevState, currState *common.AdminState_Types, dErr error) {
281 // Log the status
282 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_SUCCESS}
283 if dErr != nil {
284 requestStatus = &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
285 }
286 agent.logDeviceUpdate(ctx, prevState, currState, requestStatus, dErr, "adapter-force-delete-response")
287
288 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
289 logger.Errorw(ctx, "failed-getting-device-request-lock", log.Fields{"device-id": agent.deviceID, "error": err})
290 }
291 previousDeviceTransientState := agent.getTransientState()
292 newDevice := agent.cloneDeviceWithoutLock()
293
294 // Even on a delete error response, cleaup the device in the core
295 requestStatus = &common.OperationResp{Code: common.OperationResp_OPERATION_SUCCESS}
296 err := agent.updateDeviceWithTransientStateAndReleaseLock(ctx, newDevice,
297 core.DeviceTransientState_DELETING_POST_ADAPTER_RESPONSE, previousDeviceTransientState)
298 if err != nil {
299 requestStatus = &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
300 }
301 agent.logDeviceUpdate(ctx, prevState, currState, requestStatus, err, "transient-state-update")
302}
303
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530304// onDeleteSuccess is a common callback for scenarios where we receive a nil response following a delete request
305// to an adapter.
khenaidood948f772021-08-11 17:49:24 -0400306func (agent *Agent) onDeleteSuccess(ctx context.Context, prevState, currState *common.AdminState_Types) {
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530307 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
khenaidood948f772021-08-11 17:49:24 -0400308 logger.Errorw(ctx, "delete-device-failure", log.Fields{"device-id": agent.deviceID, "error": err})
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530309 }
310 previousDeviceTransientState := agent.getTransientState()
311 newDevice := agent.cloneDeviceWithoutLock()
312 if err := agent.updateDeviceWithTransientStateAndReleaseLock(ctx, newDevice,
khenaidood948f772021-08-11 17:49:24 -0400313 core.DeviceTransientState_DELETING_POST_ADAPTER_RESPONSE, previousDeviceTransientState); err != nil {
314 logger.Errorw(ctx, "delete-device-failure", log.Fields{"device-id": agent.deviceID, "error": err})
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530315 }
khenaidood948f772021-08-11 17:49:24 -0400316 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_SUCCESS}
317 desc := "adapter-response"
318 agent.logDeviceUpdate(ctx, prevState, currState, requestStatus, nil, desc)
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530319}
320
321// onDeleteFailure is a common callback for scenarios where we receive an error response following a delete request
322// to an adapter and the only action required is to return the error response.
khenaidood948f772021-08-11 17:49:24 -0400323func (agent *Agent) onDeleteFailure(ctx context.Context, err error, prevState, currState *common.AdminState_Types) {
324 logger.Errorw(ctx, "rpc-failed", log.Fields{"rpc": coreutils.GetRPCMetadataFromContext(ctx), "device-id": agent.deviceID, "error": err})
325
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530326 //Only updating of transient state is required, no transition.
khenaidood948f772021-08-11 17:49:24 -0400327 if er := agent.updateTransientState(ctx, core.DeviceTransientState_DELETE_FAILED); er != nil {
328 logger.Errorw(ctx, "failed-to-update-transient-state-as-delete-failed", log.Fields{"device-id": agent.deviceID, "error": er})
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530329 }
khenaidood948f772021-08-11 17:49:24 -0400330 rpce := agent.deviceMgr.NewRPCEvent(ctx, agent.deviceID, err.Error(), nil)
331 go agent.deviceMgr.SendRPCEvent(ctx, "RPC_ERROR_RAISE_EVENT", rpce,
332 voltha.EventCategory_COMMUNICATION, nil, time.Now().Unix())
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530333
khenaidood948f772021-08-11 17:49:24 -0400334 // Log the device update event
335 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
336 desc := "adapter-response"
337 agent.logDeviceUpdate(ctx, prevState, currState, requestStatus, err, desc)
Maninder9a1bc0d2020-10-26 11:34:02 +0530338}
339
Kent Hagermancba2f302020-07-28 13:37:36 -0400340// getDeviceReadOnly returns a device which MUST NOT be modified, but is safe to keep forever.
341func (agent *Agent) getDeviceReadOnly(ctx context.Context) (*voltha.Device, error) {
khenaidoo442e7c72020-03-10 16:13:48 -0400342 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
343 return nil, err
344 }
345 defer agent.requestQueue.RequestComplete()
Kent Hagermancba2f302020-07-28 13:37:36 -0400346 return agent.device, nil
khenaidoo92e62c52018-10-03 14:02:54 -0400347}
348
Kent Hagermancba2f302020-07-28 13:37:36 -0400349// getDeviceReadOnlyWithoutLock returns a device which MUST NOT be modified, but is safe to keep forever. This is very efficient.
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400350// The device lock MUST be held by the caller.
Kent Hagermancba2f302020-07-28 13:37:36 -0400351func (agent *Agent) getDeviceReadOnlyWithoutLock() *voltha.Device {
khenaidoo0db4c812020-05-27 15:27:30 -0400352 return agent.device
khenaidoo92e62c52018-10-03 14:02:54 -0400353}
354
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400355// cloneDeviceWithoutLock returns a copy of the device which is safe to modify.
356// The device lock MUST be held by the caller.
357func (agent *Agent) cloneDeviceWithoutLock() *voltha.Device {
358 return proto.Clone(agent.device).(*voltha.Device)
359}
360
khenaidood948f772021-08-11 17:49:24 -0400361func (agent *Agent) updateDeviceTypeAndEndpoint(ctx context.Context) error {
362 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
363 return err
364 }
365 changed := false
366 cloned := agent.cloneDeviceWithoutLock()
367 if cloned.Type == "" {
368 adapterType, err := agent.adapterMgr.GetAdapterType(cloned.Type)
369 if err != nil {
370 agent.requestQueue.RequestComplete()
371 return err
372 }
373 cloned.Type = adapterType
374 changed = true
375 }
376
377 if cloned.AdapterEndpoint == "" {
378 var err error
379 if cloned.AdapterEndpoint, err = agent.adapterMgr.GetAdapterEndpoint(ctx, cloned.Id, cloned.Type); err != nil {
380 agent.requestQueue.RequestComplete()
381 return err
382 }
383 agent.adapterEndpoint = cloned.AdapterEndpoint
384 changed = true
385 }
386
387 if changed {
388 return agent.updateDeviceAndReleaseLock(ctx, cloned)
389 }
390 agent.requestQueue.RequestComplete()
391 return nil
392}
393
khenaidoo3ab34882019-05-02 21:33:30 -0400394// enableDevice activates a preprovisioned or a disable device
Kent Hagerman2b216042020-04-03 18:28:56 -0400395func (agent *Agent) enableDevice(ctx context.Context) error {
Maninder9a1bc0d2020-10-26 11:34:02 +0530396 //To preserve and use oldDevice state as prev state in new device
khenaidood948f772021-08-11 17:49:24 -0400397 var err error
Maninder9a1bc0d2020-10-26 11:34:02 +0530398 var desc string
khenaidood948f772021-08-11 17:49:24 -0400399 var prevAdminState, currAdminState common.AdminState_Types
400 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
Maninder9a1bc0d2020-10-26 11:34:02 +0530401
khenaidood948f772021-08-11 17:49:24 -0400402 defer func() { agent.logDeviceUpdate(ctx, &prevAdminState, &currAdminState, requestStatus, err, desc) }()
Maninder9a1bc0d2020-10-26 11:34:02 +0530403
khenaidood948f772021-08-11 17:49:24 -0400404 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
khenaidoo442e7c72020-03-10 16:13:48 -0400405 return err
406 }
Himani Chawlab4c25912020-11-12 17:16:38 +0530407 logger.Debugw(ctx, "enable-device", log.Fields{"device-id": agent.deviceID})
khenaidoo21d51152019-02-01 13:48:37 -0500408
Kent Hagermancba2f302020-07-28 13:37:36 -0400409 oldDevice := agent.getDeviceReadOnlyWithoutLock()
khenaidood948f772021-08-11 17:49:24 -0400410 prevAdminState = oldDevice.AdminState
Maninder9a1bc0d2020-10-26 11:34:02 +0530411
Maninder2195ccc2021-06-23 20:23:01 +0530412 if !agent.proceedWithRequest(oldDevice) {
413 agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -0400414 err = status.Errorf(codes.FailedPrecondition, "cannot complete operation as device deletion is in progress or reconciling is in progress/failed: %s", agent.deviceID)
415 return err
Maninder2195ccc2021-06-23 20:23:01 +0530416 }
Mahir Gunyel92dd1212021-10-22 11:42:56 -0700417 //vol-4275 TST meeting 08/04/2021: Let EnableDevice to be called again if device is in FAILED operational state,
418 //even the admin state is ENABLED.
419 if oldDevice.AdminState == voltha.AdminState_ENABLED && oldDevice.OperStatus != voltha.OperStatus_FAILED {
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400420 logger.Warnw(ctx, "device-already-enabled", log.Fields{"device-id": agent.deviceID})
421 agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -0400422 err = status.Errorf(codes.FailedPrecondition, fmt.Sprintf("cannot-enable-an-already-enabled-device: %s", oldDevice.Id))
Matteo Scandolod525ae32020-04-02 17:27:29 -0700423 return err
npujar1d86a522019-11-14 17:11:16 +0530424 }
425
khenaidood948f772021-08-11 17:49:24 -0400426 // Verify whether there is a device type that supports this device type
427 _, err = agent.adapterMgr.GetAdapterType(oldDevice.Type)
428 if err != nil {
429 agent.requestQueue.RequestComplete()
430 return err
431 }
432
433 // Update device adapter endpoint if not set. This is set once by the Core and use as is by the adapters. E.g if this is a
434 // child device then the parent adapter will use this device's adapter endpoint (set here) to communicate with it.
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400435 newDevice := agent.cloneDeviceWithoutLock()
khenaidood948f772021-08-11 17:49:24 -0400436 if newDevice.AdapterEndpoint == "" {
437 if newDevice.AdapterEndpoint, err = agent.adapterMgr.GetAdapterEndpoint(ctx, newDevice.Id, newDevice.Type); err != nil {
438 agent.requestQueue.RequestComplete()
439 return err
440 }
441 agent.adapterEndpoint = newDevice.AdapterEndpoint
442 }
npujar1d86a522019-11-14 17:11:16 +0530443
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400444 // Update the Admin State and set the operational state to activating before sending the request to the Adapters
445 newDevice.AdminState = voltha.AdminState_ENABLED
446 newDevice.OperStatus = voltha.OperStatus_ACTIVATING
Maninder9a1bc0d2020-10-26 11:34:02 +0530447
khenaidoo442e7c72020-03-10 16:13:48 -0400448 // Adopt the device if it was in pre-provision state. In all other cases, try to re-enable it.
khenaidood948f772021-08-11 17:49:24 -0400449 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
khenaidoo442e7c72020-03-10 16:13:48 -0400450 if err != nil {
khenaidood948f772021-08-11 17:49:24 -0400451 logger.Errorw(ctx, "grpc-client-nil",
452 log.Fields{
453 "error": err,
454 "device-id": agent.deviceID,
455 "device-type": agent.deviceType,
456 "adapter-endpoint": newDevice.AdapterEndpoint,
457 })
458 agent.requestQueue.RequestComplete()
khenaidoo442e7c72020-03-10 16:13:48 -0400459 return err
460 }
khenaidood948f772021-08-11 17:49:24 -0400461 subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
462 requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS
463 go func() {
464 defer cancel()
465 var err error
466 if oldDevice.AdminState == voltha.AdminState_PREPROVISIONED {
467 _, err = client.AdoptDevice(subCtx, newDevice)
468 } else {
469 _, err = client.ReEnableDevice(subCtx, newDevice)
470 }
471 if err == nil {
472 agent.onSuccess(subCtx, nil, nil, true)
473 } else {
474 agent.onFailure(subCtx, err, nil, nil, true)
Himani Chawlab4c25912020-11-12 17:16:38 +0530475 }
476 }()
khenaidood948f772021-08-11 17:49:24 -0400477
478 // Update device
479 if err = agent.updateDeviceAndReleaseLock(ctx, newDevice); err != nil {
480 return err
khenaidoo2c6a0992019-04-29 13:46:56 -0400481 }
khenaidood948f772021-08-11 17:49:24 -0400482 currAdminState = newDevice.AdminState
483 return nil
khenaidoo2c6a0992019-04-29 13:46:56 -0400484}
485
A R Karthick5c28f552019-12-11 22:47:44 -0800486//addFlowsAndGroups adds the "newFlows" and "newGroups" from the existing flows/groups and sends the update to the
487//adapters
khenaidoo9beaaf12021-10-19 17:32:01 -0400488func (agent *Agent) addFlowsAndGroups(ctx context.Context, newFlows []*ofp.OfpFlowStats, newGroups []*ofp.OfpGroupEntry, flowMetadata *ofp.FlowMetadata) error {
Mahir Gunyel03de0d32020-06-03 01:36:59 -0700489 var flwResponse, grpResponse coreutils.Response
490 var err error
491 //if new flow list is empty then the called function returns quickly
492 if flwResponse, err = agent.addFlowsToAdapter(ctx, newFlows, flowMetadata); err != nil {
A R Karthick5c28f552019-12-11 22:47:44 -0800493 return err
494 }
Mahir Gunyel03de0d32020-06-03 01:36:59 -0700495 //if new group list is empty then the called function returns quickly
496 if grpResponse, err = agent.addGroupsToAdapter(ctx, newGroups, flowMetadata); err != nil {
497 return err
498 }
Himani Chawla4b4bd252021-11-08 15:59:40 +0530499 if errs := coreutils.WaitForNilOrErrorResponses(agent.flowTimeout, flwResponse, grpResponse); errs != nil {
khenaidood948f772021-08-11 17:49:24 -0400500 logger.Warnw(ctx, "adapter-response", log.Fields{"device-id": agent.deviceID, "result": errs})
khenaidoo442e7c72020-03-10 16:13:48 -0400501 return status.Errorf(codes.Aborted, "flow-failure-device-%s", agent.deviceID)
khenaidoo0458db62019-06-20 08:50:36 -0400502 }
khenaidoo0458db62019-06-20 08:50:36 -0400503 return nil
504}
505
A R Karthick5c28f552019-12-11 22:47:44 -0800506//deleteFlowsAndGroups removes the "flowsToDel" and "groupsToDel" from the existing flows/groups and sends the update to the
507//adapters
khenaidoo9beaaf12021-10-19 17:32:01 -0400508func (agent *Agent) deleteFlowsAndGroups(ctx context.Context, flowsToDel []*ofp.OfpFlowStats, groupsToDel []*ofp.OfpGroupEntry, flowMetadata *ofp.FlowMetadata) error {
Mahir Gunyel03de0d32020-06-03 01:36:59 -0700509 var flwResponse, grpResponse coreutils.Response
510 var err error
511 if flwResponse, err = agent.deleteFlowsFromAdapter(ctx, flowsToDel, flowMetadata); err != nil {
A R Karthick5c28f552019-12-11 22:47:44 -0800512 return err
513 }
Mahir Gunyel03de0d32020-06-03 01:36:59 -0700514 if grpResponse, err = agent.deleteGroupsFromAdapter(ctx, groupsToDel, flowMetadata); err != nil {
515 return err
516 }
517
Himani Chawla4b4bd252021-11-08 15:59:40 +0530518 if res := coreutils.WaitForNilOrErrorResponses(agent.flowTimeout, flwResponse, grpResponse); res != nil {
khenaidoo0458db62019-06-20 08:50:36 -0400519 return status.Errorf(codes.Aborted, "errors-%s", res)
520 }
521 return nil
khenaidoo0458db62019-06-20 08:50:36 -0400522}
523
A R Karthick5c28f552019-12-11 22:47:44 -0800524//updateFlowsAndGroups replaces the existing flows and groups with "updatedFlows" and "updatedGroups" respectively. It
525//also sends the updates to the adapters
khenaidoo9beaaf12021-10-19 17:32:01 -0400526func (agent *Agent) updateFlowsAndGroups(ctx context.Context, updatedFlows []*ofp.OfpFlowStats, updatedGroups []*ofp.OfpGroupEntry, flowMetadata *ofp.FlowMetadata) error {
Mahir Gunyel03de0d32020-06-03 01:36:59 -0700527 var flwResponse, grpResponse coreutils.Response
528 var err error
529 if flwResponse, err = agent.updateFlowsToAdapter(ctx, updatedFlows, flowMetadata); err != nil {
A R Karthick5c28f552019-12-11 22:47:44 -0800530 return err
531 }
Mahir Gunyel03de0d32020-06-03 01:36:59 -0700532 if grpResponse, err = agent.updateGroupsToAdapter(ctx, updatedGroups, flowMetadata); err != nil {
533 return err
534 }
535
Himani Chawla4b4bd252021-11-08 15:59:40 +0530536 if res := coreutils.WaitForNilOrErrorResponses(agent.flowTimeout, flwResponse, grpResponse); res != nil {
khenaidoo0458db62019-06-20 08:50:36 -0400537 return status.Errorf(codes.Aborted, "errors-%s", res)
538 }
539 return nil
khenaidoo19d7b632018-10-30 10:49:50 -0400540}
541
khenaidoo4d4802d2018-10-04 21:59:49 -0400542//disableDevice disable a device
Kent Hagerman2b216042020-04-03 18:28:56 -0400543func (agent *Agent) disableDevice(ctx context.Context) error {
khenaidood948f772021-08-11 17:49:24 -0400544 var err error
Maninder9a1bc0d2020-10-26 11:34:02 +0530545 var desc string
khenaidood948f772021-08-11 17:49:24 -0400546 var prevAdminState, currAdminState common.AdminState_Types
547 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
548 defer func() { agent.logDeviceUpdate(ctx, &prevAdminState, &currAdminState, requestStatus, err, desc) }()
Maninder9a1bc0d2020-10-26 11:34:02 +0530549
khenaidood948f772021-08-11 17:49:24 -0400550 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
khenaidoo442e7c72020-03-10 16:13:48 -0400551 return err
552 }
Himani Chawlab4c25912020-11-12 17:16:38 +0530553 logger.Debugw(ctx, "disable-device", log.Fields{"device-id": agent.deviceID})
khenaidoo6e55d9e2019-12-12 18:26:26 -0500554
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400555 cloned := agent.cloneDeviceWithoutLock()
khenaidood948f772021-08-11 17:49:24 -0400556 prevAdminState = agent.device.AdminState
khenaidoo6e55d9e2019-12-12 18:26:26 -0500557
Maninder2195ccc2021-06-23 20:23:01 +0530558 if !agent.proceedWithRequest(cloned) {
khenaidood948f772021-08-11 17:49:24 -0400559 err = status.Errorf(codes.FailedPrecondition, "cannot complete operation as device deletion is in progress or reconciling is in progress/failed: %s", agent.deviceID)
Maninder2195ccc2021-06-23 20:23:01 +0530560 agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -0400561 return err
Maninder2195ccc2021-06-23 20:23:01 +0530562 }
563
khenaidoo6e55d9e2019-12-12 18:26:26 -0500564 if cloned.AdminState == voltha.AdminState_DISABLED {
Maninder9a1bc0d2020-10-26 11:34:02 +0530565 desc = "device-already-disabled"
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400566 agent.requestQueue.RequestComplete()
npujar1d86a522019-11-14 17:11:16 +0530567 return nil
568 }
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530569 if cloned.AdminState == voltha.AdminState_PREPROVISIONED {
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400570 agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -0400571 err = status.Errorf(codes.FailedPrecondition, "deviceId:%s, invalid-admin-state:%s", agent.deviceID, cloned.AdminState)
572 return err
npujar1d86a522019-11-14 17:11:16 +0530573 }
Maninder0aabf0c2021-03-17 14:55:14 +0530574
npujar1d86a522019-11-14 17:11:16 +0530575 // Update the Admin State and operational state before sending the request out
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400576 cloned.AdminState = voltha.AdminState_DISABLED
577 cloned.OperStatus = voltha.OperStatus_UNKNOWN
Maninder9a1bc0d2020-10-26 11:34:02 +0530578
khenaidood948f772021-08-11 17:49:24 -0400579 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
khenaidoo442e7c72020-03-10 16:13:48 -0400580 if err != nil {
khenaidood948f772021-08-11 17:49:24 -0400581 logger.Errorw(ctx, "grpc-client-nil",
582 log.Fields{
583 "error": err,
584 "device-id": agent.deviceID,
585 "device-type": agent.deviceType,
586 "adapter-endpoint": cloned.AdapterEndpoint,
587 })
588 agent.requestQueue.RequestComplete()
npujar1d86a522019-11-14 17:11:16 +0530589 return err
khenaidoo0a822f92019-05-08 15:15:57 -0400590 }
khenaidood948f772021-08-11 17:49:24 -0400591 subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
592 requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS
593 go func() {
594 defer cancel()
595 _, err := client.DisableDevice(subCtx, cloned)
596 if err == nil {
597 agent.onSuccess(subCtx, nil, nil, true)
598 } else {
599 agent.onFailure(subCtx, err, nil, nil, true)
600 }
601 }()
Maninder9a1bc0d2020-10-26 11:34:02 +0530602
khenaidood948f772021-08-11 17:49:24 -0400603 // Update device
604 if err = agent.updateDeviceAndReleaseLock(ctx, cloned); err != nil {
605 return err
606 }
607 currAdminState = cloned.AdminState
khenaidoo0a822f92019-05-08 15:15:57 -0400608
khenaidoo92e62c52018-10-03 14:02:54 -0400609 return nil
610}
611
Kent Hagerman2b216042020-04-03 18:28:56 -0400612func (agent *Agent) rebootDevice(ctx context.Context) error {
Maninder9a1bc0d2020-10-26 11:34:02 +0530613 var desc string
khenaidood948f772021-08-11 17:49:24 -0400614 var err error
615 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
616 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
Maninder9a1bc0d2020-10-26 11:34:02 +0530617
khenaidood948f772021-08-11 17:49:24 -0400618 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
Maninder9a1bc0d2020-10-26 11:34:02 +0530619 desc = err.Error()
npujar1d86a522019-11-14 17:11:16 +0530620 return err
khenaidoo4d4802d2018-10-04 21:59:49 -0400621 }
khenaidoo442e7c72020-03-10 16:13:48 -0400622 defer agent.requestQueue.RequestComplete()
Himani Chawlab4c25912020-11-12 17:16:38 +0530623 logger.Debugw(ctx, "reboot-device", log.Fields{"device-id": agent.deviceID})
khenaidoo442e7c72020-03-10 16:13:48 -0400624
Kent Hagermancba2f302020-07-28 13:37:36 -0400625 device := agent.getDeviceReadOnlyWithoutLock()
Maninder2195ccc2021-06-23 20:23:01 +0530626
627 if !agent.proceedWithRequest(device) {
khenaidood948f772021-08-11 17:49:24 -0400628 err = status.Errorf(codes.FailedPrecondition, "cannot complete operation as device deletion is in progress or reconciling is in progress/failed:%s", agent.deviceID)
khenaidoo442e7c72020-03-10 16:13:48 -0400629 return err
630 }
Maninder9a1bc0d2020-10-26 11:34:02 +0530631
khenaidood948f772021-08-11 17:49:24 -0400632 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
633 if err != nil {
634 logger.Errorw(ctx, "grpc-client-nil",
635 log.Fields{
636 "error": err,
637 "device-id": agent.deviceID,
638 "device-type": agent.deviceType,
639 "adapter-endpoint": device.AdapterEndpoint,
640 })
641 return err
642 }
643 subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
644 requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS
645 go func() {
646 defer cancel()
647 _, err := client.RebootDevice(subCtx, device)
648 if err == nil {
649 agent.onSuccess(subCtx, nil, nil, true)
650 } else {
651 agent.onFailure(subCtx, err, nil, nil, true)
652 }
653 }()
khenaidoo4d4802d2018-10-04 21:59:49 -0400654 return nil
655}
656
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530657func (agent *Agent) deleteDeviceForce(ctx context.Context) error {
Himani Chawlab4c25912020-11-12 17:16:38 +0530658 logger.Debugw(ctx, "delete-device-force", log.Fields{"device-id": agent.deviceID})
Maninder9a1bc0d2020-10-26 11:34:02 +0530659
660 var desc string
khenaidood948f772021-08-11 17:49:24 -0400661 var err error
662 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
663 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
Maninder9a1bc0d2020-10-26 11:34:02 +0530664
khenaidood948f772021-08-11 17:49:24 -0400665 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530666 return err
667 }
668 // Get the device Transient state, return err if it is DELETING
669 previousDeviceTransientState := agent.getTransientState()
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530670 device := agent.cloneDeviceWithoutLock()
khenaidood948f772021-08-11 17:49:24 -0400671 if !agent.isForceDeletingAllowed(previousDeviceTransientState, device) {
672 agent.requestQueue.RequestComplete()
673 err = status.Error(codes.FailedPrecondition, fmt.Sprintf("deviceId:%s, force deletion is in progress", agent.deviceID))
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530674 return err
675 }
Himani Chawlab4c25912020-11-12 17:16:38 +0530676
khenaidood948f772021-08-11 17:49:24 -0400677 previousAdminState := device.AdminState
678 if previousAdminState != common.AdminState_PREPROVISIONED {
khenaidoo9beaaf12021-10-19 17:32:01 -0400679 var client adapter_service.AdapterServiceClient
khenaidood948f772021-08-11 17:49:24 -0400680 client, err = agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530681 if err != nil {
khenaidood948f772021-08-11 17:49:24 -0400682 logger.Errorw(ctx, "grpc-client-nil",
683 log.Fields{
684 "error": err,
685 "device-id": agent.deviceID,
686 "device-type": agent.deviceType,
687 "adapter-endpoint": device.AdapterEndpoint,
688 })
689 agent.requestQueue.RequestComplete()
khenaidoo68a5e0c2021-11-06 13:08:03 -0400690 return fmt.Errorf("remote-not-reachable %w", errNoConnection)
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530691 }
khenaidood948f772021-08-11 17:49:24 -0400692 subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
693 requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS
694 go func() {
695 defer cancel()
696 _, err := client.DeleteDevice(subCtx, device)
697 if err == nil {
698 agent.onSuccess(subCtx, nil, nil, true)
699 } else {
700 agent.onFailure(subCtx, err, nil, nil, true)
701 }
702 }()
703 }
704
705 // Update device
706 if err = agent.updateDeviceWithTransientStateAndReleaseLock(ctx, device,
707 core.DeviceTransientState_FORCE_DELETING, previousDeviceTransientState); err != nil {
708 return err
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530709 }
710 return nil
711}
712
Kent Hagerman2b216042020-04-03 18:28:56 -0400713func (agent *Agent) deleteDevice(ctx context.Context) error {
Himani Chawlab4c25912020-11-12 17:16:38 +0530714 logger.Debugw(ctx, "delete-device", log.Fields{"device-id": agent.deviceID})
Maninder9a1bc0d2020-10-26 11:34:02 +0530715
716 var desc string
khenaidood948f772021-08-11 17:49:24 -0400717 var err error
718 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
719 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
Maninder9a1bc0d2020-10-26 11:34:02 +0530720
khenaidood948f772021-08-11 17:49:24 -0400721 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
Maninder9a1bc0d2020-10-26 11:34:02 +0530722 desc = err.Error()
khenaidoo442e7c72020-03-10 16:13:48 -0400723 return err
724 }
Maninder0aabf0c2021-03-17 14:55:14 +0530725
Maninder2195ccc2021-06-23 20:23:01 +0530726 device := agent.cloneDeviceWithoutLock()
727
728 if !agent.proceedWithRequest(device) {
Maninder0aabf0c2021-03-17 14:55:14 +0530729 agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -0400730 err = status.Errorf(codes.FailedPrecondition, "cannot complete operation as device deletion is in progress or reconciling is in progress/failed: %s", agent.deviceID)
731 return err
Maninder0aabf0c2021-03-17 14:55:14 +0530732 }
733
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530734 // Get the device Transient state, return err if it is DELETING
735 previousDeviceTransientState := agent.getTransientState()
khenaidoo6e55d9e2019-12-12 18:26:26 -0500736
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530737 previousAdminState := device.AdminState
738 // Change the device transient state to DELETING_FROM_ADAPTER state till the device is removed from adapters.
khenaidood948f772021-08-11 17:49:24 -0400739 currentDeviceTransientState := core.DeviceTransientState_DELETING_FROM_ADAPTER
khenaidoo442e7c72020-03-10 16:13:48 -0400740
khenaidood948f772021-08-11 17:49:24 -0400741 if previousAdminState == common.AdminState_PREPROVISIONED {
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530742 // Change the state to DELETING POST ADAPTER RESPONSE directly as adapters have no info of the device.
khenaidood948f772021-08-11 17:49:24 -0400743 currentDeviceTransientState = core.DeviceTransientState_DELETING_POST_ADAPTER_RESPONSE
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530744 }
khenaidood948f772021-08-11 17:49:24 -0400745 // If the device was in pre-prov state (only parent device are in that state) then do not send the request to the
746 // adapter
747 if previousAdminState != common.AdminState_PREPROVISIONED {
khenaidoo9beaaf12021-10-19 17:32:01 -0400748 var client adapter_service.AdapterServiceClient
khenaidood948f772021-08-11 17:49:24 -0400749 client, err = agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
750 if err != nil {
751 logger.Errorw(ctx, "grpc-client-nil",
752 log.Fields{
753 "error": err,
754 "device-id": agent.deviceID,
755 "device-type": agent.deviceType,
756 "adapter-endpoint": device.AdapterEndpoint,
757 })
758 agent.requestQueue.RequestComplete()
759 return err
760 }
761 subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
762 requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS
763 go func() {
764 defer cancel()
765 _, err := client.DeleteDevice(subCtx, device)
766 if err == nil {
767 agent.onDeleteSuccess(subCtx, nil, nil)
768 } else {
769 agent.onDeleteFailure(subCtx, err, nil, nil)
770 }
771 }()
772 }
773
774 // Update device and release lock
775 if err = agent.updateDeviceWithTransientStateAndReleaseLock(ctx, device,
Himani Chawlab4c25912020-11-12 17:16:38 +0530776 currentDeviceTransientState, previousDeviceTransientState); err != nil {
Maninder9a1bc0d2020-10-26 11:34:02 +0530777 desc = err.Error()
npujar1d86a522019-11-14 17:11:16 +0530778 return err
779 }
Himani Chawlab4c25912020-11-12 17:16:38 +0530780
khenaidoo4d4802d2018-10-04 21:59:49 -0400781 return nil
782}
783
Kent Hagerman2b216042020-04-03 18:28:56 -0400784func (agent *Agent) setParentID(ctx context.Context, device *voltha.Device, parentID string) error {
khenaidoo442e7c72020-03-10 16:13:48 -0400785 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
786 return err
787 }
Himani Chawlab4c25912020-11-12 17:16:38 +0530788 logger.Debugw(ctx, "set-parent-id", log.Fields{"device-id": device.Id, "parent-id": parentID})
khenaidoo6e55d9e2019-12-12 18:26:26 -0500789
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400790 cloned := agent.cloneDeviceWithoutLock()
npujar1d86a522019-11-14 17:11:16 +0530791 cloned.ParentId = parentID
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400792 return agent.updateDeviceAndReleaseLock(ctx, cloned)
khenaidooad06fd72019-10-28 12:26:05 -0400793}
794
khenaidoo442e7c72020-03-10 16:13:48 -0400795// getSwitchCapability retrieves the switch capability of a parent device
khenaidoo9beaaf12021-10-19 17:32:01 -0400796func (agent *Agent) getSwitchCapability(ctx context.Context) (*ca.SwitchCapability, error) {
Himani Chawlab4c25912020-11-12 17:16:38 +0530797 logger.Debugw(ctx, "get-switch-capability", log.Fields{"device-id": agent.deviceID})
khenaidoo442e7c72020-03-10 16:13:48 -0400798
Kent Hagermancba2f302020-07-28 13:37:36 -0400799 device, err := agent.getDeviceReadOnly(ctx)
khenaidoo442e7c72020-03-10 16:13:48 -0400800 if err != nil {
khenaidoob9203542018-09-17 22:56:37 -0400801 return nil, err
khenaidoob9203542018-09-17 22:56:37 -0400802 }
khenaidood948f772021-08-11 17:49:24 -0400803
804 // Get the gRPC client
805 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
khenaidoo442e7c72020-03-10 16:13:48 -0400806 if err != nil {
807 return nil, err
808 }
809
khenaidood948f772021-08-11 17:49:24 -0400810 return client.GetOfpDeviceInfo(ctx, device)
khenaidoob9203542018-09-17 22:56:37 -0400811}
812
khenaidood948f772021-08-11 17:49:24 -0400813func (agent *Agent) onPacketFailure(ctx context.Context, err error, packet *ofp.OfpPacketOut) {
814 logger.Errorw(ctx, "packet-out-error", log.Fields{
khenaidoo442e7c72020-03-10 16:13:48 -0400815 "device-id": agent.deviceID,
khenaidood948f772021-08-11 17:49:24 -0400816 "error": err.Error(),
817 "packet": hex.EncodeToString(packet.Data),
khenaidoo442e7c72020-03-10 16:13:48 -0400818 })
khenaidood948f772021-08-11 17:49:24 -0400819 rpce := agent.deviceMgr.NewRPCEvent(ctx, agent.deviceID, err.Error(), nil)
820 go agent.deviceMgr.SendRPCEvent(ctx, "RPC_ERROR_RAISE_EVENT", rpce,
821 voltha.EventCategory_COMMUNICATION, nil, time.Now().Unix())
khenaidoo442e7c72020-03-10 16:13:48 -0400822}
823
Kent Hagerman2b216042020-04-03 18:28:56 -0400824func (agent *Agent) packetOut(ctx context.Context, outPort uint32, packet *ofp.OfpPacketOut) error {
Scott Baker80678602019-11-14 16:57:36 -0800825 if agent.deviceType == "" {
npujar467fe752020-01-16 20:17:45 +0530826 agent.reconcileWithKVStore(ctx)
Scott Baker80678602019-11-14 16:57:36 -0800827 }
khenaidoofdbad6e2018-11-06 22:26:38 -0500828 // Send packet to adapter
khenaidood948f772021-08-11 17:49:24 -0400829 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
khenaidoo442e7c72020-03-10 16:13:48 -0400830 if err != nil {
khenaidood948f772021-08-11 17:49:24 -0400831 logger.Errorw(ctx, "grpc-client-nil",
832 log.Fields{
833 "error": err,
834 "device-id": agent.deviceID,
835 "device-type": agent.deviceType,
836 })
837 return err
khenaidoofdbad6e2018-11-06 22:26:38 -0500838 }
khenaidood948f772021-08-11 17:49:24 -0400839 subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
840 go func() {
841 defer cancel()
khenaidoo9beaaf12021-10-19 17:32:01 -0400842 _, err := client.SendPacketOut(subCtx, &ca.PacketOut{
khenaidood948f772021-08-11 17:49:24 -0400843 DeviceId: agent.deviceID,
844 EgressPortNo: outPort,
845 Packet: packet,
846 })
847 if err == nil {
848 agent.onSuccess(subCtx, nil, nil, false)
849 } else {
850 agent.onPacketFailure(subCtx, err, packet)
851 }
852 }()
khenaidoofdbad6e2018-11-06 22:26:38 -0500853 return nil
854}
855
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400856func (agent *Agent) updateDeviceUsingAdapterData(ctx context.Context, device *voltha.Device) error {
khenaidood948f772021-08-11 17:49:24 -0400857 var err error
858 var desc string
859 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
860 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
861
862 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400863 return err
864 }
Himani Chawlab4c25912020-11-12 17:16:38 +0530865 logger.Debugw(ctx, "update-device-using-adapter-data", log.Fields{"device-id": device.Id})
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400866
867 cloned := agent.cloneDeviceWithoutLock()
Mahir Gunyel8e2707d2019-07-25 00:36:21 -0700868 cloned.Root = device.Root
869 cloned.Vendor = device.Vendor
870 cloned.Model = device.Model
871 cloned.SerialNumber = device.SerialNumber
872 cloned.MacAddress = device.MacAddress
873 cloned.Vlan = device.Vlan
874 cloned.Reason = device.Reason
Andrea Campanella025667e2021-01-14 11:50:07 +0100875 cloned.ImageDownloads = device.ImageDownloads
khenaidood948f772021-08-11 17:49:24 -0400876 cloned.OperStatus = device.OperStatus
877 cloned.ConnectStatus = device.ConnectStatus
878 if err = agent.updateDeviceAndReleaseLock(ctx, cloned); err == nil {
879 requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
880 }
881 return err
khenaidoo43c82122018-11-22 18:38:28 -0500882}
883
Kent Hagerman2b216042020-04-03 18:28:56 -0400884func (agent *Agent) updateDeviceStatus(ctx context.Context, operStatus voltha.OperStatus_Types, connStatus voltha.ConnectStatus_Types) error {
khenaidood948f772021-08-11 17:49:24 -0400885 var err error
886 var desc string
887 opStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
888 defer func() { agent.logDeviceUpdate(ctx, nil, nil, opStatus, err, desc) }()
889
890 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
khenaidoo442e7c72020-03-10 16:13:48 -0400891 return err
892 }
khenaidoo6e55d9e2019-12-12 18:26:26 -0500893
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400894 cloned := agent.cloneDeviceWithoutLock()
npujar1d86a522019-11-14 17:11:16 +0530895 // Ensure the enums passed in are valid - they will be invalid if they are not set when this function is invoked
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400896 if s, ok := voltha.ConnectStatus_Types_name[int32(connStatus)]; ok {
khenaidood948f772021-08-11 17:49:24 -0400897 logger.Debugw(ctx, "update-device-conn-status", log.Fields{"ok": ok, "val": s})
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400898 cloned.ConnectStatus = connStatus
npujar1d86a522019-11-14 17:11:16 +0530899 }
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400900 if s, ok := voltha.OperStatus_Types_name[int32(operStatus)]; ok {
khenaidood948f772021-08-11 17:49:24 -0400901 logger.Debugw(ctx, "update-device-oper-status", log.Fields{"ok": ok, "val": s})
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400902 cloned.OperStatus = operStatus
npujar1d86a522019-11-14 17:11:16 +0530903 }
Himani Chawlab4c25912020-11-12 17:16:38 +0530904 logger.Debugw(ctx, "update-device-status", log.Fields{"device-id": cloned.Id, "oper-status": cloned.OperStatus, "connect-status": cloned.ConnectStatus})
npujar1d86a522019-11-14 17:11:16 +0530905 // Store the device
khenaidood948f772021-08-11 17:49:24 -0400906 if err = agent.updateDeviceAndReleaseLock(ctx, cloned); err == nil {
907 opStatus.Code = common.OperationResp_OPERATION_SUCCESS
908 }
909 return err
khenaidoo92e62c52018-10-03 14:02:54 -0400910}
911
khenaidoob9203542018-09-17 22:56:37 -0400912// TODO: A generic device update by attribute
Kent Hagerman2b216042020-04-03 18:28:56 -0400913func (agent *Agent) updateDeviceAttribute(ctx context.Context, name string, value interface{}) {
khenaidoob9203542018-09-17 22:56:37 -0400914 if value == nil {
915 return
916 }
khenaidoo6e55d9e2019-12-12 18:26:26 -0500917
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400918 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
919 logger.Warnw(ctx, "request-aborted", log.Fields{"device-id": agent.deviceID, "name": name, "error": err})
920 return
921 }
922
923 cloned := agent.cloneDeviceWithoutLock()
khenaidoob9203542018-09-17 22:56:37 -0400924 updated := false
khenaidoo6e55d9e2019-12-12 18:26:26 -0500925 s := reflect.ValueOf(cloned).Elem()
khenaidoob9203542018-09-17 22:56:37 -0400926 if s.Kind() == reflect.Struct {
927 // exported field
928 f := s.FieldByName(name)
929 if f.IsValid() && f.CanSet() {
930 switch f.Kind() {
931 case reflect.String:
932 f.SetString(value.(string))
933 updated = true
934 case reflect.Uint32:
935 f.SetUint(uint64(value.(uint32)))
936 updated = true
937 case reflect.Bool:
938 f.SetBool(value.(bool))
939 updated = true
940 }
941 }
942 }
divyadesaicb8b59d2020-08-18 09:55:47 +0000943 logger.Debugw(ctx, "update-field-status", log.Fields{"device-id": cloned.Id, "name": name, "updated": updated})
khenaidoob9203542018-09-17 22:56:37 -0400944 // Save the data
khenaidoo6e55d9e2019-12-12 18:26:26 -0500945
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400946 if err := agent.updateDeviceAndReleaseLock(ctx, cloned); err != nil {
Rohan Agrawal31f21802020-06-12 05:38:46 +0000947 logger.Warnw(ctx, "attribute-update-failed", log.Fields{"attribute": name, "value": value})
khenaidoob9203542018-09-17 22:56:37 -0400948 }
khenaidoob9203542018-09-17 22:56:37 -0400949}
serkant.uluderya334479d2019-04-10 08:26:15 -0700950
Kent Hagerman45a13e42020-04-13 12:23:50 -0400951func (agent *Agent) simulateAlarm(ctx context.Context, simulateReq *voltha.SimulateAlarmRequest) error {
khenaidood948f772021-08-11 17:49:24 -0400952 var err error
953 var desc string
954 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
955 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
956
957 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
khenaidoo442e7c72020-03-10 16:13:48 -0400958 return err
959 }
960 defer agent.requestQueue.RequestComplete()
Himani Chawlab4c25912020-11-12 17:16:38 +0530961 logger.Debugw(ctx, "simulate-alarm", log.Fields{"device-id": agent.deviceID})
khenaidoo6e55d9e2019-12-12 18:26:26 -0500962
Kent Hagermancba2f302020-07-28 13:37:36 -0400963 device := agent.getDeviceReadOnlyWithoutLock()
khenaidoo6e55d9e2019-12-12 18:26:26 -0500964
khenaidood948f772021-08-11 17:49:24 -0400965 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
khenaidoo442e7c72020-03-10 16:13:48 -0400966 if err != nil {
khenaidood948f772021-08-11 17:49:24 -0400967 logger.Errorw(ctx, "grpc-client-nil",
968 log.Fields{
969 "error": err,
970 "device-id": agent.deviceID,
971 "device-type": agent.deviceType,
972 "adapter-endpoint": device.AdapterEndpoint,
973 })
npujar1d86a522019-11-14 17:11:16 +0530974 return err
serkant.uluderya334479d2019-04-10 08:26:15 -0700975 }
khenaidood948f772021-08-11 17:49:24 -0400976 subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
977 requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS
978 go func() {
979 defer cancel()
khenaidoo9beaaf12021-10-19 17:32:01 -0400980 _, err := client.SimulateAlarm(subCtx, &ca.SimulateAlarmMessage{Device: device, Request: simulateReq})
khenaidood948f772021-08-11 17:49:24 -0400981 if err == nil {
982 agent.onSuccess(subCtx, nil, nil, false)
983 } else {
984 agent.onFailure(subCtx, err, nil, nil, false)
985 }
986 }()
serkant.uluderya334479d2019-04-10 08:26:15 -0700987 return nil
988}
Mahir Gunyelb5851672019-07-24 10:46:26 +0300989
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400990// This function updates the device in the DB, releases the device lock, and runs any state transitions.
991// The calling function MUST hold the device lock. The caller MUST NOT modify the device after this is called.
992func (agent *Agent) updateDeviceAndReleaseLock(ctx context.Context, device *voltha.Device) error {
993 // fail early if this agent is no longer valid
Kent Hagerman4f355f52020-03-30 16:01:33 -0400994 if agent.stopped {
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400995 agent.requestQueue.RequestComplete()
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530996 return errors.New("device-agent-stopped")
Thomas Lee Se5a44012019-11-07 20:32:24 +0530997 }
Kent Hagerman4f355f52020-03-30 16:01:33 -0400998
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400999 // update in db
Kent Hagermanf5a67352020-04-30 15:15:26 -04001000 if err := agent.dbProxy.Set(ctx, agent.deviceID, device); err != nil {
Kent Hagermanf6db9f12020-07-22 17:16:19 -04001001 agent.requestQueue.RequestComplete()
Kent Hagerman4f355f52020-03-30 16:01:33 -04001002 return status.Errorf(codes.Internal, "failed-update-device:%s: %s", agent.deviceID, err)
Mahir Gunyelb5851672019-07-24 10:46:26 +03001003 }
divyadesaicb8b59d2020-08-18 09:55:47 +00001004 logger.Debugw(ctx, "updated-device-in-store", log.Fields{"device-id: ": agent.deviceID})
Mahir Gunyelb5851672019-07-24 10:46:26 +03001005
Kent Hagerman6031aad2020-07-29 16:36:33 -04001006 prevDevice := agent.device
Kent Hagermanf6db9f12020-07-22 17:16:19 -04001007 // update the device
khenaidoo0db4c812020-05-27 15:27:30 -04001008 agent.device = device
Mahir Gunyelb0343bf2021-05-11 14:14:26 -07001009 //If any of the states has chenged, send the change event.
1010 if prevDevice.OperStatus != device.OperStatus || prevDevice.ConnectStatus != device.ConnectStatus || prevDevice.AdminState != device.AdminState {
1011 _ = agent.deviceMgr.Agent.SendDeviceStateChangeEvent(ctx, prevDevice.OperStatus, prevDevice.ConnectStatus, prevDevice.AdminState, device, time.Now().Unix())
1012 }
Maninder0aabf0c2021-03-17 14:55:14 +05301013 deviceTransientState := agent.getTransientState()
1014
Kent Hagermanf6db9f12020-07-22 17:16:19 -04001015 // release lock before processing transition
1016 agent.requestQueue.RequestComplete()
Himani Chawlab4c25912020-11-12 17:16:38 +05301017 subCtx := coreutils.WithSpanAndRPCMetadataFromContext(ctx)
Kent Hagermanf6db9f12020-07-22 17:16:19 -04001018
Himani Chawlab4c25912020-11-12 17:16:38 +05301019 if err := agent.deviceMgr.stateTransitions.ProcessTransition(subCtx,
Maninder0aabf0c2021-03-17 14:55:14 +05301020 device, prevDevice, deviceTransientState, deviceTransientState); err != nil {
Himani Chawlab4c25912020-11-12 17:16:38 +05301021 logger.Errorw(ctx, "failed-process-transition", log.Fields{"device-id": device.Id, "previous-admin-state": prevDevice.AdminState, "current-admin-state": device.AdminState})
1022 // Sending RPC EVENT here
1023 rpce := agent.deviceMgr.NewRPCEvent(ctx, agent.deviceID, err.Error(), nil)
Himani Chawla606a4f02021-03-23 19:45:58 +05301024 agent.deviceMgr.SendRPCEvent(ctx, "RPC_ERROR_RAISE_EVENT", rpce, voltha.EventCategory_COMMUNICATION,
1025 nil, time.Now().Unix())
Himani Chawlab4c25912020-11-12 17:16:38 +05301026
Kent Hagermanf6db9f12020-07-22 17:16:19 -04001027 }
Mahir Gunyelb5851672019-07-24 10:46:26 +03001028 return nil
1029}
Mahir Gunyelfdee9212019-10-16 16:52:21 -07001030
Himani Chawla2ba1c9c2020-10-07 13:19:03 +05301031// This function updates the device transient in the DB through loader, releases the device lock, and runs any state transitions.
1032// The calling function MUST hold the device lock. The caller MUST NOT modify the device after this is called.
1033func (agent *Agent) updateDeviceWithTransientStateAndReleaseLock(ctx context.Context, device *voltha.Device,
khenaidood948f772021-08-11 17:49:24 -04001034 transientState, prevTransientState core.DeviceTransientState_Types) error {
Himani Chawla2ba1c9c2020-10-07 13:19:03 +05301035 // fail early if this agent is no longer valid
1036 if agent.stopped {
1037 agent.requestQueue.RequestComplete()
1038 return errors.New("device-agent-stopped")
1039 }
1040 //update device TransientState
1041 if err := agent.updateTransientState(ctx, transientState); err != nil {
1042 agent.requestQueue.RequestComplete()
1043 return err
1044 }
1045 // update in db
1046 if err := agent.dbProxy.Set(ctx, agent.deviceID, device); err != nil {
1047 //Reverting TransientState update
khenaidood948f772021-08-11 17:49:24 -04001048 if errTransient := agent.updateTransientState(ctx, prevTransientState); errTransient != nil {
1049 logger.Errorw(ctx, "failed-to-revert-transient-state-update-on-error", log.Fields{"device-id": device.Id,
1050 "previous-transient-state": prevTransientState, "current-transient-state": transientState, "error": errTransient})
1051 }
Himani Chawla2ba1c9c2020-10-07 13:19:03 +05301052 agent.requestQueue.RequestComplete()
1053 return status.Errorf(codes.Internal, "failed-update-device:%s: %s", agent.deviceID, err)
1054 }
1055
1056 logger.Debugw(ctx, "updated-device-in-store", log.Fields{"device-id: ": agent.deviceID})
1057
1058 prevDevice := agent.device
1059 // update the device
1060 agent.device = device
Mahir Gunyelb0343bf2021-05-11 14:14:26 -07001061 //If any of the states has chenged, send the change event.
1062 if prevDevice.OperStatus != device.OperStatus || prevDevice.ConnectStatus != device.ConnectStatus || prevDevice.AdminState != device.AdminState {
1063 _ = agent.deviceMgr.Agent.SendDeviceStateChangeEvent(ctx, prevDevice.OperStatus, prevDevice.ConnectStatus, prevDevice.AdminState, device, time.Now().Unix())
1064 }
Himani Chawla2ba1c9c2020-10-07 13:19:03 +05301065
1066 // release lock before processing transition
1067 agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -04001068 go func() {
1069 subCtx := coreutils.WithSpanAndRPCMetadataFromContext(ctx)
1070 if err := agent.deviceMgr.stateTransitions.ProcessTransition(subCtx,
1071 device, prevDevice, transientState, prevTransientState); err != nil {
1072 logger.Errorw(ctx, "failed-process-transition", log.Fields{"device-id": device.Id, "previous-admin-state": prevDevice.AdminState, "current-admin-state": device.AdminState})
1073 // Sending RPC EVENT here
1074 rpce := agent.deviceMgr.NewRPCEvent(ctx, agent.deviceID, err.Error(), nil)
1075 agent.deviceMgr.SendRPCEvent(ctx, "RPC_ERROR_RAISE_EVENT", rpce, voltha.EventCategory_COMMUNICATION,
1076 nil, time.Now().Unix())
1077 }
1078 }()
Himani Chawla2ba1c9c2020-10-07 13:19:03 +05301079 return nil
1080}
Kent Hagerman2b216042020-04-03 18:28:56 -04001081func (agent *Agent) updateDeviceReason(ctx context.Context, reason string) error {
khenaidood948f772021-08-11 17:49:24 -04001082 logger.Debugw(ctx, "update-device-reason", log.Fields{"device-id": agent.deviceID, "reason": reason})
1083
1084 var err error
1085 var desc string
1086 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
1087 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
1088
1089 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
khenaidoo442e7c72020-03-10 16:13:48 -04001090 return err
1091 }
Maninder9a1bc0d2020-10-26 11:34:02 +05301092
Kent Hagermanf6db9f12020-07-22 17:16:19 -04001093 cloned := agent.cloneDeviceWithoutLock()
npujar1d86a522019-11-14 17:11:16 +05301094 cloned.Reason = reason
khenaidood948f772021-08-11 17:49:24 -04001095 if err = agent.updateDeviceAndReleaseLock(ctx, cloned); err == nil {
1096 requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
Maninder9a1bc0d2020-10-26 11:34:02 +05301097 }
khenaidood948f772021-08-11 17:49:24 -04001098 return err
Mahir Gunyelfdee9212019-10-16 16:52:21 -07001099}
kesavandbc2d1622020-01-21 00:42:01 -05001100
Kent Hagerman2b216042020-04-03 18:28:56 -04001101func (agent *Agent) ChildDeviceLost(ctx context.Context, device *voltha.Device) error {
Himani Chawlab4c25912020-11-12 17:16:38 +05301102 logger.Debugw(ctx, "child-device-lost", log.Fields{"child-device-id": device.Id, "parent-device-id": agent.deviceID})
Chaitrashree G S543df3e2020-02-24 22:36:54 -05001103
khenaidood948f772021-08-11 17:49:24 -04001104 var err error
1105 var desc string
1106 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
1107 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
1108
Kent Hagerman2a07b862020-06-19 15:23:07 -04001109 // Remove the associated peer ports on the parent device
1110 for portID := range agent.portLoader.ListIDs() {
1111 if portHandle, have := agent.portLoader.Lock(portID); have {
1112 oldPort := portHandle.GetReadOnly()
1113 updatedPeers := make([]*voltha.Port_PeerPort, 0)
1114 for _, peerPort := range oldPort.Peers {
1115 if peerPort.DeviceId != device.Id {
1116 updatedPeers = append(updatedPeers, peerPort)
1117 }
khenaidoo442e7c72020-03-10 16:13:48 -04001118 }
Kent Hagerman2a07b862020-06-19 15:23:07 -04001119 newPort := *oldPort
1120 newPort.Peers = updatedPeers
1121 if err := portHandle.Update(ctx, &newPort); err != nil {
1122 portHandle.Unlock()
1123 return nil
1124 }
1125 portHandle.Unlock()
khenaidoo442e7c72020-03-10 16:13:48 -04001126 }
Chaitrashree G S543df3e2020-02-24 22:36:54 -05001127 }
1128
khenaidoo442e7c72020-03-10 16:13:48 -04001129 //send request to adapter
khenaidood948f772021-08-11 17:49:24 -04001130 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
khenaidoo442e7c72020-03-10 16:13:48 -04001131 if err != nil {
khenaidood948f772021-08-11 17:49:24 -04001132 logger.Errorw(ctx, "grpc-client-nil",
1133 log.Fields{
1134 "error": err,
1135 "device-id": agent.deviceID,
1136 "device-type": agent.deviceType,
1137 "adapter-endpoint": device.AdapterEndpoint,
1138 })
khenaidoo442e7c72020-03-10 16:13:48 -04001139 return err
Chaitrashree G S543df3e2020-02-24 22:36:54 -05001140 }
khenaidood948f772021-08-11 17:49:24 -04001141 subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
1142 requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS
1143 go func() {
1144 defer cancel()
1145 _, err := client.ChildDeviceLost(subCtx, device)
1146 if err == nil {
1147 agent.onSuccess(subCtx, nil, nil, true)
1148 } else {
1149 agent.onFailure(subCtx, err, nil, nil, true)
1150 }
1151 }()
Chaitrashree G S543df3e2020-02-24 22:36:54 -05001152 return nil
Chaitrashree G S543df3e2020-02-24 22:36:54 -05001153}
onkarkundargi87285252020-01-27 11:34:52 +05301154
khenaidoo9beaaf12021-10-19 17:32:01 -04001155func (agent *Agent) startOmciTest(ctx context.Context, omcitestrequest *omci.OmciTestRequest) (*omci.TestResponse, error) {
khenaidood948f772021-08-11 17:49:24 -04001156 var err error
1157 var desc string
1158 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
1159 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
onkarkundargi87285252020-01-27 11:34:52 +05301160
khenaidood948f772021-08-11 17:49:24 -04001161 // OMCI test may be performed on a pre-provisioned device. If a device is in that state both its device type and endpoint
1162 // may not have been set yet.
1163 // First check if we need to update the type or endpoint
1164 cloned, err := agent.getDeviceReadOnly(ctx)
onkarkundargi87285252020-01-27 11:34:52 +05301165 if err != nil {
1166 return nil, err
1167 }
khenaidood948f772021-08-11 17:49:24 -04001168 if cloned.Type == "" || cloned.AdapterEndpoint == "" {
1169 if err = agent.updateDeviceTypeAndEndpoint(ctx); err != nil {
1170 return nil, err
1171 }
1172 cloned, err = agent.getDeviceReadOnly(ctx)
1173 if err != nil {
1174 return nil, err
1175 }
onkarkundargi87285252020-01-27 11:34:52 +05301176 }
1177
khenaidood948f772021-08-11 17:49:24 -04001178 // Send request to the adapter
1179 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
1180 if err != nil {
1181 logger.Errorw(ctx, "grpc-client-nil",
1182 log.Fields{
1183 "error": err,
1184 "device-id": agent.deviceID,
1185 "device-type": agent.deviceType,
1186 "adapter-endpoint": cloned.AdapterEndpoint,
1187 })
1188 return nil, err
onkarkundargi87285252020-01-27 11:34:52 +05301189 }
khenaidood948f772021-08-11 17:49:24 -04001190
khenaidoo9beaaf12021-10-19 17:32:01 -04001191 res, err := client.StartOmciTest(ctx, &ca.OMCITest{
khenaidood948f772021-08-11 17:49:24 -04001192 Device: cloned,
1193 Request: omcitestrequest,
1194 })
1195 if err == nil {
1196 requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
1197 }
1198 return res, err
onkarkundargi87285252020-01-27 11:34:52 +05301199}
Dinesh Belwalkarc1129f12020-02-27 10:41:33 -08001200
khenaidoo9beaaf12021-10-19 17:32:01 -04001201func (agent *Agent) getExtValue(ctx context.Context, pdevice *voltha.Device, cdevice *voltha.Device, valueparam *extension.ValueSpecifier) (*extension.ReturnValues, error) {
Himani Chawlab4c25912020-11-12 17:16:38 +05301202 logger.Debugw(ctx, "get-ext-value", log.Fields{"device-id": agent.deviceID, "onu-id": valueparam.Id, "value-type": valueparam.Value})
khenaidood948f772021-08-11 17:49:24 -04001203 var err error
1204 var desc string
1205 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
1206 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
1207
1208 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
Dinesh Belwalkarc1129f12020-02-27 10:41:33 -08001209 return nil, err
1210 }
1211
khenaidood948f772021-08-11 17:49:24 -04001212 //send request to adapter synchronously
1213 client, err := agent.adapterMgr.GetAdapterClient(ctx, pdevice.AdapterEndpoint)
Dinesh Belwalkarc1129f12020-02-27 10:41:33 -08001214 if err != nil {
khenaidood948f772021-08-11 17:49:24 -04001215 logger.Errorw(ctx, "grpc-client-nil",
1216 log.Fields{
1217 "error": err,
1218 "device-id": agent.deviceID,
1219 "device-type": agent.deviceType,
1220 "adapter-endpoint": pdevice.AdapterEndpoint,
1221 })
1222 agent.requestQueue.RequestComplete()
Dinesh Belwalkarc1129f12020-02-27 10:41:33 -08001223 return nil, err
1224 }
1225
khenaidood948f772021-08-11 17:49:24 -04001226 // Release lock before sending to adapter
1227 agent.requestQueue.RequestComplete()
Dinesh Belwalkarc1129f12020-02-27 10:41:33 -08001228
khenaidoo9beaaf12021-10-19 17:32:01 -04001229 retVal, err := client.GetExtValue(ctx, &ca.GetExtValueMessage{
khenaidood948f772021-08-11 17:49:24 -04001230 ParentDevice: pdevice,
1231 ChildDevice: cdevice,
1232 ValueType: valueparam.Value,
1233 })
1234 if err == nil {
1235 requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
Dinesh Belwalkarc1129f12020-02-27 10:41:33 -08001236 }
khenaidood948f772021-08-11 17:49:24 -04001237 return retVal, err
Dinesh Belwalkarc1129f12020-02-27 10:41:33 -08001238}
dpaul62686312020-06-23 14:17:36 +05301239
khenaidoo9beaaf12021-10-19 17:32:01 -04001240func (agent *Agent) setExtValue(ctx context.Context, device *voltha.Device, value *extension.ValueSet) (*empty.Empty, error) {
Himani Chawlab4c25912020-11-12 17:16:38 +05301241 logger.Debugw(ctx, "set-ext-value", log.Fields{"device-id": value.Id})
khenaidood948f772021-08-11 17:49:24 -04001242
1243 var err error
1244 var desc string
1245 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
1246 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
1247
1248 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
dpaul62686312020-06-23 14:17:36 +05301249 return nil, err
1250 }
1251
1252 //send request to adapter
khenaidood948f772021-08-11 17:49:24 -04001253 //send request to adapter synchronously
1254 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
dpaul62686312020-06-23 14:17:36 +05301255 if err != nil {
khenaidood948f772021-08-11 17:49:24 -04001256 logger.Errorw(ctx, "grpc-client-nil",
1257 log.Fields{
1258 "error": err,
1259 "device-id": agent.deviceID,
1260 "device-type": agent.deviceType,
1261 "adapter-endpoint": device.AdapterEndpoint,
1262 })
1263 agent.requestQueue.RequestComplete()
dpaul62686312020-06-23 14:17:36 +05301264 return nil, err
1265 }
khenaidood948f772021-08-11 17:49:24 -04001266 // Release lock before sending request to adapter
1267 agent.requestQueue.RequestComplete()
dpaul62686312020-06-23 14:17:36 +05301268
khenaidoo9beaaf12021-10-19 17:32:01 -04001269 retVal, err := client.SetExtValue(ctx, &ca.SetExtValueMessage{
khenaidood948f772021-08-11 17:49:24 -04001270 Device: device,
1271 Value: value,
1272 })
1273 if err == nil {
1274 requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
dpaul62686312020-06-23 14:17:36 +05301275 }
khenaidood948f772021-08-11 17:49:24 -04001276 return retVal, err
dpaul62686312020-06-23 14:17:36 +05301277}
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301278
1279func (agent *Agent) getSingleValue(ctx context.Context, request *extension.SingleGetValueRequest) (*extension.SingleGetValueResponse, error) {
Himani Chawlab4c25912020-11-12 17:16:38 +05301280 logger.Debugw(ctx, "get-single-value", log.Fields{"device-id": request.TargetId})
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301281
khenaidood948f772021-08-11 17:49:24 -04001282 var err error
1283 var desc string
1284 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
1285 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
1286
1287 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301288 return nil, err
1289 }
1290
1291 cloned := agent.cloneDeviceWithoutLock()
1292
1293 //send request to adapter
khenaidood948f772021-08-11 17:49:24 -04001294 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301295 if err != nil {
khenaidood948f772021-08-11 17:49:24 -04001296 logger.Errorw(ctx, "grpc-client-nil",
1297 log.Fields{
1298 "error": err,
1299 "device-id": cloned.Id,
1300 "adapter-endpoint": cloned.AdapterEndpoint,
1301 })
1302 agent.requestQueue.RequestComplete()
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301303 return nil, err
1304 }
khenaidood948f772021-08-11 17:49:24 -04001305 // Release lock before sending request to adapter
1306 agent.requestQueue.RequestComplete()
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301307
khenaidood948f772021-08-11 17:49:24 -04001308 resp, err := client.GetSingleValue(ctx, request)
1309 if err == nil {
1310 requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301311 }
khenaidood948f772021-08-11 17:49:24 -04001312 return resp, err
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301313}
1314
1315func (agent *Agent) setSingleValue(ctx context.Context, request *extension.SingleSetValueRequest) (*extension.SingleSetValueResponse, error) {
Himani Chawlab4c25912020-11-12 17:16:38 +05301316 logger.Debugw(ctx, "set-single-value", log.Fields{"device-id": request.TargetId})
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301317
khenaidood948f772021-08-11 17:49:24 -04001318 var err error
1319 var desc string
1320 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
1321 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
1322
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301323 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
1324 return nil, err
1325 }
1326
1327 cloned := agent.cloneDeviceWithoutLock()
1328
1329 //send request to adapter
khenaidood948f772021-08-11 17:49:24 -04001330 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301331 if err != nil {
khenaidood948f772021-08-11 17:49:24 -04001332 logger.Errorw(ctx, "grpc-client-nil",
1333 log.Fields{
1334 "error": err,
1335 "device-id": agent.deviceID,
1336 "device-type": agent.deviceType,
1337 "adapter-endpoint": cloned.AdapterEndpoint,
1338 })
1339 agent.requestQueue.RequestComplete()
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301340 return nil, err
1341 }
khenaidood948f772021-08-11 17:49:24 -04001342 // Release lock before sending request to adapter
1343 agent.requestQueue.RequestComplete()
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301344
khenaidood948f772021-08-11 17:49:24 -04001345 resp, err := client.SetSingleValue(ctx, request)
1346 if err == nil {
1347 requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301348 }
khenaidood948f772021-08-11 17:49:24 -04001349 return resp, err
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301350}
Maninder0aabf0c2021-03-17 14:55:14 +05301351
Maninder2195ccc2021-06-23 20:23:01 +05301352func (agent *Agent) proceedWithRequest(device *voltha.Device) bool {
1353 return !agent.isDeletionInProgress() && !agent.isInReconcileState(device)
Maninder0aabf0c2021-03-17 14:55:14 +05301354}
1355
1356func (agent *Agent) stopReconcile() {
1357 agent.stopReconcilingMutex.Lock()
1358 if agent.stopReconciling != nil {
1359 agent.stopReconciling <- 0
1360 }
1361 agent.stopReconcilingMutex.Unlock()
1362}
1363
khenaidood948f772021-08-11 17:49:24 -04001364// abortAllProcessing is invoked when an adapter managing this device is restarted
1365func (agent *Agent) abortAllProcessing(ctx context.Context) error {
1366 logger.Infow(ctx, "aborting-current-running-requests", log.Fields{"device-id": agent.deviceID})
1367 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
1368 return err
1369 }
khenaidoo25057da2021-12-08 14:40:45 -05001370 logger.Infow(ctx, "aborting-current-running-requests-after-wait", log.Fields{"device-id": agent.deviceID})
1371
khenaidood948f772021-08-11 17:49:24 -04001372 defer agent.requestQueue.RequestComplete()
1373
1374 // If any reconciling is in progress just abort it. The adapter is gone.
1375 agent.stopReconcile()
1376
khenaidoo25057da2021-12-08 14:40:45 -05001377 logger.Infow(ctx, "aborting-current-running-requests-after-sendstop", log.Fields{"device-id": agent.deviceID})
1378
khenaidood948f772021-08-11 17:49:24 -04001379 // Update the Core device transient state accordingly
1380 var updatedState core.DeviceTransientState_Types
1381 switch agent.getTransientState() {
1382 case core.DeviceTransientState_RECONCILE_IN_PROGRESS:
1383 updatedState = core.DeviceTransientState_NONE
1384 case core.DeviceTransientState_FORCE_DELETING:
1385 updatedState = core.DeviceTransientState_DELETE_FAILED
1386 case core.DeviceTransientState_DELETING_FROM_ADAPTER:
1387 updatedState = core.DeviceTransientState_DELETE_FAILED
khenaidoo68a5e0c2021-11-06 13:08:03 -04001388 case core.DeviceTransientState_DELETE_FAILED:
1389 // do not change state
1390 return nil
khenaidood948f772021-08-11 17:49:24 -04001391 default:
1392 updatedState = core.DeviceTransientState_NONE
1393 }
1394 if err := agent.updateTransientState(ctx, updatedState); err != nil {
1395 logger.Errorf(ctx, "transient-state-update-failed", log.Fields{"error": err})
1396 return err
1397 }
1398 return nil
1399}
1400
khenaidoo68a5e0c2021-11-06 13:08:03 -04001401func (agent *Agent) DeleteDevicePostAdapterRestart(ctx context.Context) error {
1402 logger.Debugw(ctx, "delete-post-restart", log.Fields{"device-id": agent.deviceID})
1403 ctx = utils.WithNewSpanAndRPCMetadataContext(ctx, "DelteDevicePostAdapterRestart")
1404
1405 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
1406 return err
1407 }
1408
1409 device := agent.getDeviceReadOnlyWithoutLock()
1410 if device.AdminState == voltha.AdminState_PREPROVISIONED {
1411 logger.Debugw(ctx, "device-in-preprovisioning-state-reconcile-not-needed", log.Fields{"device-id": device.Id})
1412 agent.requestQueue.RequestComplete()
1413 return nil
1414 }
1415 // Change device transient state to FORCE_DELETING
1416 if err := agent.updateTransientState(ctx, core.DeviceTransientState_FORCE_DELETING); err != nil {
1417 logger.Errorw(ctx, "failure-updating-transient-state", log.Fields{"error": err, "device-id": agent.deviceID})
1418 agent.requestQueue.RequestComplete()
1419 return err
1420 }
1421
1422 // Ensure we have a valid grpc client available as we have just restarted
1423 deleteBackoff := backoff.NewExponentialBackOff()
1424 deleteBackoff.InitialInterval = agent.config.BackoffRetryInitialInterval
1425 deleteBackoff.MaxElapsedTime = agent.config.BackoffRetryMaxElapsedTime
1426 deleteBackoff.MaxInterval = agent.config.BackoffRetryMaxInterval
1427 var backoffTimer *time.Timer
1428 var err error
1429 var client adapter_service.AdapterServiceClient
1430retry:
1431 for {
1432 client, err = agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
1433 if err == nil {
1434 break retry
1435 }
1436 duration := deleteBackoff.NextBackOff()
1437 if duration == backoff.Stop {
1438 deleteBackoff.Reset()
1439 duration = deleteBackoff.NextBackOff()
1440 }
1441 backoffTimer = time.NewTimer(duration)
1442 select {
1443 case <-backoffTimer.C:
1444 logger.Debugw(ctx, "backoff-timer-expires", log.Fields{"device-id": agent.deviceID})
1445 case <-ctx.Done():
1446 err = ctx.Err()
1447 break retry
1448 }
1449 }
1450 if backoffTimer != nil && !backoffTimer.Stop() {
1451 select {
1452 case <-backoffTimer.C:
1453 default:
1454 }
1455 }
1456 if err != nil || client == nil {
1457 agent.requestQueue.RequestComplete()
1458 return err
1459 }
1460
1461 // Release the device lock to allow for device state update, if any
1462 agent.requestQueue.RequestComplete()
1463
1464 // Send the delete request to the adapter
1465 subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
1466 defer cancel()
khenaidoo6cd8e862021-11-12 16:33:43 -05001467 _, err = client.DeleteDevice(subCtx, device)
1468 agent.onForceDeleteResponse(subCtx, nil, nil, err)
1469 return err
khenaidoo68a5e0c2021-11-06 13:08:03 -04001470}
1471
khenaidood948f772021-08-11 17:49:24 -04001472func (agent *Agent) ReconcileDevice(ctx context.Context) {
khenaidoo68a5e0c2021-11-06 13:08:03 -04001473 // Do not reconcile if the device was in DELETE_FAILED transient state. Just invoke the force delete on that device.
1474 state := agent.getTransientState()
1475 logger.Debugw(ctx, "starting-reconcile", log.Fields{"device-id": agent.deviceID, "state": state})
1476 if agent.getTransientState() == core.DeviceTransientState_DELETE_FAILED {
1477 if err := agent.DeleteDevicePostAdapterRestart(ctx); err != nil {
1478 logger.Errorw(ctx, "delete-post-restart-failed", log.Fields{"error": err, "device-id": agent.deviceID})
1479 }
1480 return
1481 }
1482
khenaidood948f772021-08-11 17:49:24 -04001483 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
Maninder0aabf0c2021-03-17 14:55:14 +05301484 var desc string
Maninder0aabf0c2021-03-17 14:55:14 +05301485
1486 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
khenaidood948f772021-08-11 17:49:24 -04001487 agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc)
1488 return
1489 }
1490
1491 device := agent.getDeviceReadOnlyWithoutLock()
1492 if device.AdminState == voltha.AdminState_PREPROVISIONED {
1493 agent.requestQueue.RequestComplete()
1494 logger.Debugw(ctx, "device-in-preprovisioning-state-reconcile-not-needed", log.Fields{"device-id": device.Id})
Maninder0aabf0c2021-03-17 14:55:14 +05301495 return
1496 }
1497
Maninder2195ccc2021-06-23 20:23:01 +05301498 if !agent.proceedWithRequest(device) {
Maninder0aabf0c2021-03-17 14:55:14 +05301499 agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -04001500 err := fmt.Errorf("cannot complete operation as device deletion/reconciling is in progress or reconcile failed for device : %s", device.Id)
1501 logger.Errorw(ctx, "reconcile-failed", log.Fields{"error": err})
1502 agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc)
Maninder0aabf0c2021-03-17 14:55:14 +05301503 return
1504 }
1505
1506 //set transient state to RECONCILE IN PROGRESS
khenaidood948f772021-08-11 17:49:24 -04001507 err := agent.updateTransientState(ctx, core.DeviceTransientState_RECONCILE_IN_PROGRESS)
Maninder0aabf0c2021-03-17 14:55:14 +05301508 if err != nil {
1509 agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -04001510 logger.Errorw(ctx, "setting-transient-state-failed", log.Fields{"error": err})
1511 agent.logDeviceUpdate(ctx, nil, nil, requestStatus, nil, desc)
Maninder0aabf0c2021-03-17 14:55:14 +05301512 return
1513 }
1514
Maninder0aabf0c2021-03-17 14:55:14 +05301515 reconcilingBackoff := backoff.NewExponentialBackOff()
1516 reconcilingBackoff.InitialInterval = agent.config.BackoffRetryInitialInterval
1517 reconcilingBackoff.MaxElapsedTime = agent.config.BackoffRetryMaxElapsedTime
1518 reconcilingBackoff.MaxInterval = agent.config.BackoffRetryMaxInterval
1519
1520 //making here to keep lifecycle of this channel within the scope of retryReconcile
1521 agent.stopReconcilingMutex.Lock()
khenaidoo25057da2021-12-08 14:40:45 -05001522 agent.stopReconciling = make(chan int, 1)
Maninder0aabf0c2021-03-17 14:55:14 +05301523 agent.stopReconcilingMutex.Unlock()
1524
David K. Bainbridge482e4422021-06-30 12:23:42 -07001525 // defined outside the retry loop so it can be cleaned
1526 // up when the loop breaks
1527 var backoffTimer *time.Timer
1528
1529retry:
Maninder0aabf0c2021-03-17 14:55:14 +05301530 for {
David K. Bainbridge482e4422021-06-30 12:23:42 -07001531 // If the operations state of the device is RECONCILING_FAILED then we do not
1532 // want to continue to attempt reconciliation.
1533 deviceRef := agent.getDeviceReadOnlyWithoutLock()
1534 if deviceRef.OperStatus == common.OperStatus_RECONCILING_FAILED {
1535 logger.Warnw(ctx, "reconciling-failed-halting-retries",
1536 log.Fields{"device-id": device.Id})
1537 agent.requestQueue.RequestComplete()
1538 break retry
1539 }
1540
Maninder0aabf0c2021-03-17 14:55:14 +05301541 // Use an exponential back off to prevent getting into a tight loop
1542 duration := reconcilingBackoff.NextBackOff()
1543 //This case should never occur in default case as max elapsed time for backoff is 0(by default) , so it will never return stop
1544 if duration == backoff.Stop {
1545 // If we reach a maximum then warn and reset the backoff
1546 // timer and keep attempting.
1547 logger.Warnw(ctx, "maximum-reconciling-backoff-reached--resetting-backoff-timer",
1548 log.Fields{"max-reconciling-backoff": reconcilingBackoff.MaxElapsedTime,
1549 "device-id": device.Id})
1550 reconcilingBackoff.Reset()
1551 duration = reconcilingBackoff.NextBackOff()
1552 }
1553
David K. Bainbridge482e4422021-06-30 12:23:42 -07001554 backoffTimer = time.NewTimer(duration)
Maninder0aabf0c2021-03-17 14:55:14 +05301555
khenaidood948f772021-08-11 17:49:24 -04001556 logger.Debugw(ctx, "retrying-reconciling", log.Fields{"deviceID": device.Id, "endpoint": device.AdapterEndpoint})
1557 // Release lock before sending request to adapter
Maninder0aabf0c2021-03-17 14:55:14 +05301558 agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -04001559
1560 // Send a reconcile request to the adapter.
1561 err := agent.sendReconcileRequestToAdapter(ctx, device)
khenaidoo6cd8e862021-11-12 16:33:43 -05001562
1563 // Check the transient state after a response from the adapter. If a device delete
1564 // request was issued due to a callback during that time and failed then just delete
1565 // the device and stop the reconcile loop and invoke the device deletion
1566 if agent.getTransientState() == core.DeviceTransientState_DELETE_FAILED {
1567 if dErr := agent.DeleteDevicePostAdapterRestart(ctx); dErr != nil {
1568 logger.Errorw(ctx, "delete-post-restart-failed", log.Fields{"error": dErr, "device-id": agent.deviceID})
1569 }
1570 break retry
1571 }
1572
khenaidood948f772021-08-11 17:49:24 -04001573 if errors.Is(err, errContextExpired) || errors.Is(err, errReconcileAborted) {
1574 logger.Errorw(ctx, "reconcile-aborted", log.Fields{"error": err})
1575 requestStatus = &common.OperationResp{Code: common.OperationResp_OperationReturnCode(common.OperStatus_FAILED)}
1576 desc = "aborted"
1577 agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc)
1578 break retry
1579 }
Maninder0aabf0c2021-03-17 14:55:14 +05301580 if err != nil {
khenaidood948f772021-08-11 17:49:24 -04001581 agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc)
Maninder0aabf0c2021-03-17 14:55:14 +05301582 <-backoffTimer.C
1583 // backoffTimer expired continue
1584 // Take lock back before retrying
1585 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
khenaidood948f772021-08-11 17:49:24 -04001586 agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc)
David K. Bainbridge482e4422021-06-30 12:23:42 -07001587 break retry
Maninder0aabf0c2021-03-17 14:55:14 +05301588 }
1589 continue
1590 }
khenaidood948f772021-08-11 17:49:24 -04001591 // Success
1592 requestStatus = &common.OperationResp{Code: common.OperationResp_OPERATION_SUCCESS}
1593 desc = "adapter-response"
1594 agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc)
1595 break retry
David K. Bainbridge482e4422021-06-30 12:23:42 -07001596 }
1597
khenaidoo25057da2021-12-08 14:40:45 -05001598 logger.Debugw(ctx, "reconcile-retry-ends", log.Fields{"adapter-endpoint": agent.adapterEndpoint})
1599
David K. Bainbridge482e4422021-06-30 12:23:42 -07001600 // Retry loop is broken, so stop any timers and drain the channel
1601 if backoffTimer != nil && !backoffTimer.Stop() {
1602
1603 // As per documentation and stack overflow when a timer is stopped its
1604 // channel should be drained. The issue is that Stop returns false
1605 // either if the timer has already been fired "OR" if the timer can be
1606 // stopped before being fired. This means that in some cases the
1607 // channel has already be emptied so attempting to read from it means
1608 // a blocked thread. To get around this use a select so if the
1609 // channel is already empty the default case hits and we are not
1610 // blocked.
1611 select {
1612 case <-backoffTimer.C:
1613 default:
Maninder0aabf0c2021-03-17 14:55:14 +05301614 }
1615 }
1616}
1617
khenaidood948f772021-08-11 17:49:24 -04001618func (agent *Agent) sendReconcileRequestToAdapter(ctx context.Context, device *voltha.Device) error {
1619 logger.Debugw(ctx, "sending-reconcile-to-adapter", log.Fields{"device-id": device.Id, "endpoint": agent.adapterEndpoint})
1620 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
1621 if err != nil {
1622 return err
1623 }
1624 adapterResponse := make(chan error)
1625 go func() {
1626 _, err := client.ReconcileDevice(ctx, device)
1627 adapterResponse <- err
1628 }()
Maninder0aabf0c2021-03-17 14:55:14 +05301629 select {
1630 // wait for response
khenaidood948f772021-08-11 17:49:24 -04001631 case err := <-adapterResponse:
1632 if err != nil {
1633 return err
Maninder0aabf0c2021-03-17 14:55:14 +05301634 }
Maninder0aabf0c2021-03-17 14:55:14 +05301635 //In case of success quit retrying and wait for adapter to reset operation state of device
1636 agent.stopReconcilingMutex.Lock()
1637 agent.stopReconciling = nil
1638 agent.stopReconcilingMutex.Unlock()
1639 return nil
1640
1641 //if reconciling need to be stopped
1642 case _, ok := <-agent.stopReconciling:
1643 agent.stopReconcilingMutex.Lock()
1644 agent.stopReconciling = nil
1645 agent.stopReconcilingMutex.Unlock()
1646 if !ok {
1647 //channel-closed
khenaidood948f772021-08-11 17:49:24 -04001648 return fmt.Errorf("reconcile channel closed:%w", errReconcileAborted)
Maninder0aabf0c2021-03-17 14:55:14 +05301649 }
khenaidood948f772021-08-11 17:49:24 -04001650 return fmt.Errorf("reconciling aborted:%w", errReconcileAborted)
1651 // Context expired
1652 case <-ctx.Done():
1653 return fmt.Errorf("context expired:%s :%w", ctx.Err(), errContextExpired)
Maninder0aabf0c2021-03-17 14:55:14 +05301654 }
Maninder0aabf0c2021-03-17 14:55:14 +05301655}
1656
1657func (agent *Agent) reconcilingCleanup(ctx context.Context) error {
1658 var desc string
khenaidood948f772021-08-11 17:49:24 -04001659 var err error
Maninder0aabf0c2021-03-17 14:55:14 +05301660 operStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
khenaidood948f772021-08-11 17:49:24 -04001661 defer func() { agent.logDeviceUpdate(ctx, nil, nil, operStatus, err, desc) }()
1662
1663 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
1664 desc = "reconcile-cleanup-failed"
Maninder0aabf0c2021-03-17 14:55:14 +05301665 return err
1666 }
1667 defer agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -04001668 err = agent.updateTransientState(ctx, core.DeviceTransientState_NONE)
Maninder0aabf0c2021-03-17 14:55:14 +05301669 if err != nil {
khenaidood948f772021-08-11 17:49:24 -04001670 logger.Errorf(ctx, "transient-state-update-failed", log.Fields{"error": err})
Maninder0aabf0c2021-03-17 14:55:14 +05301671 return err
1672 }
khenaidood948f772021-08-11 17:49:24 -04001673 operStatus.Code = common.OperationResp_OPERATION_SUCCESS
Maninder0aabf0c2021-03-17 14:55:14 +05301674 return nil
1675}
khenaidood948f772021-08-11 17:49:24 -04001676
1677func (agent *Agent) isAdapterConnectionUp(ctx context.Context) bool {
1678 c, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
1679 return c != nil && err == nil
1680}
1681
1682func (agent *Agent) canDeviceRequestProceed(ctx context.Context) error {
1683 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
1684 return err
1685 }
1686 defer agent.requestQueue.RequestComplete()
1687 if agent.proceedWithRequest(agent.device) {
1688 return nil
1689 }
1690 return fmt.Errorf("device-cannot-process-request-%s", agent.deviceID)
1691}