blob: d6430afcd27e8f7cd1d1563758e45a5c523ef6cf [file] [log] [blame]
khenaidoob9203542018-09-17 22:56:37 -04001/*
Joey Armstrong5f51f2e2023-01-17 17:06:26 -05002 * Copyright 2018-2023 Open Networking Foundation (ONF) and the ONF Contributors
khenaidoob9203542018-09-17 22:56:37 -04003
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7
8 * http://www.apache.org/licenses/LICENSE-2.0
9
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
npujar1d86a522019-11-14 17:11:16 +053016
Kent Hagerman2b216042020-04-03 18:28:56 -040017package device
khenaidoob9203542018-09-17 22:56:37 -040018
19import (
20 "context"
Matteo Scandolo360605d2019-11-05 18:29:17 -080021 "encoding/hex"
Kent Hagerman4f355f52020-03-30 16:01:33 -040022 "errors"
khenaidoo3ab34882019-05-02 21:33:30 -040023 "fmt"
Mahir Gunyelb0343bf2021-05-11 14:14:26 -070024 "reflect"
25 "sync"
26 "time"
27
khenaidoo9beaaf12021-10-19 17:32:01 -040028 "github.com/opencord/voltha-protos/v5/go/adapter_service"
khenaidood948f772021-08-11 17:49:24 -040029 "github.com/opencord/voltha-protos/v5/go/core"
khenaidoo9beaaf12021-10-19 17:32:01 -040030 "github.com/opencord/voltha-protos/v5/go/omci"
khenaidood948f772021-08-11 17:49:24 -040031
Maninder0aabf0c2021-03-17 14:55:14 +053032 "github.com/cenkalti/backoff/v3"
Maninder9a1bc0d2020-10-26 11:34:02 +053033 "github.com/gogo/protobuf/proto"
Maninder9a1bc0d2020-10-26 11:34:02 +053034 "github.com/golang/protobuf/ptypes/empty"
Maninder0aabf0c2021-03-17 14:55:14 +053035 "github.com/opencord/voltha-go/rw_core/config"
khenaidoo68a5e0c2021-11-06 13:08:03 -040036 "github.com/opencord/voltha-go/rw_core/utils"
Maninder9a1bc0d2020-10-26 11:34:02 +053037 "google.golang.org/grpc/codes"
38 "google.golang.org/grpc/status"
Mahir Gunyeladdb66a2020-04-29 18:08:50 -070039
Himani Chawla2ba1c9c2020-10-07 13:19:03 +053040 "github.com/opencord/voltha-go/db/model"
Kent Hagerman2b216042020-04-03 18:28:56 -040041 "github.com/opencord/voltha-go/rw_core/core/adapter"
Mahir Gunyel03de0d32020-06-03 01:36:59 -070042 "github.com/opencord/voltha-go/rw_core/core/device/flow"
43 "github.com/opencord/voltha-go/rw_core/core/device/group"
Kent Hagerman2a07b862020-06-19 15:23:07 -040044 "github.com/opencord/voltha-go/rw_core/core/device/port"
Himani Chawla2ba1c9c2020-10-07 13:19:03 +053045 "github.com/opencord/voltha-go/rw_core/core/device/transientstate"
Scott Bakerb671a862019-10-24 10:53:40 -070046 coreutils "github.com/opencord/voltha-go/rw_core/utils"
khenaidood948f772021-08-11 17:49:24 -040047 "github.com/opencord/voltha-lib-go/v7/pkg/log"
48 "github.com/opencord/voltha-protos/v5/go/common"
khenaidoo9beaaf12021-10-19 17:32:01 -040049 ca "github.com/opencord/voltha-protos/v5/go/core_adapter"
khenaidood948f772021-08-11 17:49:24 -040050 "github.com/opencord/voltha-protos/v5/go/extension"
khenaidood948f772021-08-11 17:49:24 -040051 ofp "github.com/opencord/voltha-protos/v5/go/openflow_13"
52 "github.com/opencord/voltha-protos/v5/go/voltha"
khenaidoob9203542018-09-17 22:56:37 -040053)
54
khenaidood948f772021-08-11 17:49:24 -040055var errReconcileAborted = errors.New("reconcile aborted")
56var errContextExpired = errors.New("context expired")
khenaidoo68a5e0c2021-11-06 13:08:03 -040057var errNoConnection = errors.New("no connection")
khenaidood948f772021-08-11 17:49:24 -040058
Kent Hagerman2b216042020-04-03 18:28:56 -040059// Agent represents device agent attributes
60type Agent struct {
Maninder0aabf0c2021-03-17 14:55:14 +053061 deviceID string
62 parentID string
63 deviceType string
khenaidood948f772021-08-11 17:49:24 -040064 adapterEndpoint string
Maninder0aabf0c2021-03-17 14:55:14 +053065 isRootDevice bool
Maninder0aabf0c2021-03-17 14:55:14 +053066 adapterMgr *adapter.Manager
67 deviceMgr *Manager
68 dbProxy *model.Proxy
69 exitChannel chan int
70 device *voltha.Device
71 requestQueue *coreutils.RequestQueue
khenaidood948f772021-08-11 17:49:24 -040072 internalTimeout time.Duration
73 rpcTimeout time.Duration
Himani Chawla4b4bd252021-11-08 15:59:40 +053074 flowTimeout time.Duration
Maninder0aabf0c2021-03-17 14:55:14 +053075 startOnce sync.Once
76 stopOnce sync.Once
77 stopped bool
78 stopReconciling chan int
79 stopReconcilingMutex sync.RWMutex
80 config *config.RWCoreFlags
Mahir Gunyel03de0d32020-06-03 01:36:59 -070081
khenaidoo7585a962021-06-10 16:15:38 -040082 flowCache *flow.Cache
83 groupCache *group.Cache
Himani Chawla2ba1c9c2020-10-07 13:19:03 +053084 portLoader *port.Loader
85 transientStateLoader *transientstate.Loader
khenaidoob9203542018-09-17 22:56:37 -040086}
87
Kent Hagerman2b216042020-04-03 18:28:56 -040088//newAgent creates a new device agent. The device will be initialized when start() is called.
Himani Chawla4b4bd252021-11-08 15:59:40 +053089func newAgent(device *voltha.Device, deviceMgr *Manager, dbPath *model.Path, deviceProxy *model.Proxy, internalTimeout, rpcTimeout, flowTimeout time.Duration) *Agent {
Kent Hagerman2a07b862020-06-19 15:23:07 -040090 deviceID := device.Id
91 if deviceID == "" {
92 deviceID = coreutils.CreateDeviceID()
Stephane Barbarie1ab43272018-12-08 21:42:13 -050093 }
Scott Baker80678602019-11-14 16:57:36 -080094
Kent Hagerman2a07b862020-06-19 15:23:07 -040095 return &Agent{
Himani Chawla2ba1c9c2020-10-07 13:19:03 +053096 deviceID: deviceID,
Himani Chawla2ba1c9c2020-10-07 13:19:03 +053097 isRootDevice: device.Root,
98 parentID: device.ParentId,
99 deviceType: device.Type,
khenaidood948f772021-08-11 17:49:24 -0400100 adapterEndpoint: device.AdapterEndpoint,
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530101 deviceMgr: deviceMgr,
102 adapterMgr: deviceMgr.adapterMgr,
103 exitChannel: make(chan int, 1),
104 dbProxy: deviceProxy,
khenaidood948f772021-08-11 17:49:24 -0400105 internalTimeout: internalTimeout,
106 rpcTimeout: rpcTimeout,
Himani Chawla4b4bd252021-11-08 15:59:40 +0530107 flowTimeout: flowTimeout,
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530108 device: proto.Clone(device).(*voltha.Device),
109 requestQueue: coreutils.NewRequestQueue(),
Maninder0aabf0c2021-03-17 14:55:14 +0530110 config: deviceMgr.config,
khenaidoo7585a962021-06-10 16:15:38 -0400111 flowCache: flow.NewCache(),
112 groupCache: group.NewCache(),
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530113 portLoader: port.NewLoader(dbPath.SubPath("ports").Proxy(deviceID)),
114 transientStateLoader: transientstate.NewLoader(dbPath.SubPath("core").Proxy("transientstate"), deviceID),
Kent Hagerman2a07b862020-06-19 15:23:07 -0400115 }
khenaidoob9203542018-09-17 22:56:37 -0400116}
117
khenaidoo442e7c72020-03-10 16:13:48 -0400118// start() saves the device to the data model and registers for callbacks on that device if deviceToCreate!=nil.
119// Otherwise, it will load the data from the dB and setup the necessary callbacks and proxies. Returns the device that
Scott Baker80678602019-11-14 16:57:36 -0800120// was started.
khenaidoo7585a962021-06-10 16:15:38 -0400121func (agent *Agent) start(ctx context.Context, deviceExist bool, deviceToCreate *voltha.Device) (*voltha.Device, error) {
khenaidoo442e7c72020-03-10 16:13:48 -0400122 needToStart := false
123 if agent.startOnce.Do(func() { needToStart = true }); !needToStart {
Kent Hagermancba2f302020-07-28 13:37:36 -0400124 return agent.getDeviceReadOnly(ctx)
khenaidoo442e7c72020-03-10 16:13:48 -0400125 }
126 var startSucceeded bool
127 defer func() {
128 if !startSucceeded {
129 if err := agent.stop(ctx); err != nil {
Rohan Agrawal31f21802020-06-12 05:38:46 +0000130 logger.Errorw(ctx, "failed-to-cleanup-after-unsuccessful-start", log.Fields{"device-id": agent.deviceID, "error": err})
khenaidoo442e7c72020-03-10 16:13:48 -0400131 }
132 }
133 }()
khenaidoo7585a962021-06-10 16:15:38 -0400134 if deviceExist {
135 device := deviceToCreate
136 if device == nil {
137 // Load from dB
138 device = &voltha.Device{}
139 have, err := agent.dbProxy.Get(ctx, agent.deviceID, device)
140 if err != nil {
141 return nil, err
142 } else if !have {
143 return nil, status.Errorf(codes.NotFound, "device-%s", agent.deviceID)
144 }
khenaidood948f772021-08-11 17:49:24 -0400145 logger.Infow(ctx, "device-loaded-from-db", log.Fields{"device-id": agent.deviceID, "adapter-endpoint": device.AdapterEndpoint, "type": device.Type})
Thomas Lee Se5a44012019-11-07 20:32:24 +0530146 }
khenaidood948f772021-08-11 17:49:24 -0400147 agent.deviceType = device.Type
148 agent.adapterEndpoint = device.AdapterEndpoint
Kent Hagerman4f355f52020-03-30 16:01:33 -0400149 agent.device = proto.Clone(device).(*voltha.Device)
khenaidoo7585a962021-06-10 16:15:38 -0400150 // load the ports from KV to cache
Kent Hagerman2a07b862020-06-19 15:23:07 -0400151 agent.portLoader.Load(ctx)
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530152 agent.transientStateLoader.Load(ctx)
khenaidoo297cd252019-02-07 22:10:23 -0500153 } else {
Scott Baker80678602019-11-14 16:57:36 -0800154 // Create a new device
Maninder9a1bc0d2020-10-26 11:34:02 +0530155 var desc string
khenaidood948f772021-08-11 17:49:24 -0400156 var err error
Maninder9a1bc0d2020-10-26 11:34:02 +0530157 prevState := common.AdminState_UNKNOWN
158 currState := common.AdminState_UNKNOWN
khenaidood948f772021-08-11 17:49:24 -0400159 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
Maninder9a1bc0d2020-10-26 11:34:02 +0530160
khenaidood948f772021-08-11 17:49:24 -0400161 defer func() { agent.logDeviceUpdate(ctx, &prevState, &currState, requestStatus, err, desc) }()
Maninder9a1bc0d2020-10-26 11:34:02 +0530162
Kent Hagermanf5a67352020-04-30 15:15:26 -0400163 // Assumption is that AdminState, FlowGroups, and Flows are uninitialized since this
164 // is a new device, so populate them here before passing the device to ldProxy.Set.
Kent Hagerman2b216042020-04-03 18:28:56 -0400165 // agent.deviceId will also have been set during newAgent().
khenaidoo7585a962021-06-10 16:15:38 -0400166 device := (proto.Clone(deviceToCreate)).(*voltha.Device)
npujar1d86a522019-11-14 17:11:16 +0530167 device.Id = agent.deviceID
Scott Baker80678602019-11-14 16:57:36 -0800168 device.AdminState = voltha.AdminState_PREPROVISIONED
Maninder9a1bc0d2020-10-26 11:34:02 +0530169 currState = device.AdminState
Scott Baker80678602019-11-14 16:57:36 -0800170 if !deviceToCreate.GetRoot() && deviceToCreate.ProxyAddress != nil {
171 // Set the default vlan ID to the one specified by the parent adapter. It can be
172 // overwritten by the child adapter during a device update request
173 device.Vlan = deviceToCreate.ProxyAddress.ChannelId
174 }
175
khenaidood948f772021-08-11 17:49:24 -0400176 // Save the device to the model
177 if err = agent.dbProxy.Set(ctx, agent.deviceID, device); err != nil {
178 err = status.Errorf(codes.Aborted, "failed-adding-device-%s: %s", agent.deviceID, err)
179 return nil, err
khenaidoo297cd252019-02-07 22:10:23 -0500180 }
Mahir Gunyelb0343bf2021-05-11 14:14:26 -0700181 _ = agent.deviceMgr.Agent.SendDeviceStateChangeEvent(ctx, device.OperStatus, device.ConnectStatus, prevState, device, time.Now().Unix())
khenaidood948f772021-08-11 17:49:24 -0400182 requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
khenaidoo442e7c72020-03-10 16:13:48 -0400183 agent.device = device
khenaidoob9203542018-09-17 22:56:37 -0400184 }
khenaidoo442e7c72020-03-10 16:13:48 -0400185 startSucceeded = true
Rohan Agrawalcf12f202020-08-03 04:42:01 +0000186 log.EnrichSpan(ctx, log.Fields{"device-id": agent.deviceID})
Rohan Agrawal31f21802020-06-12 05:38:46 +0000187 logger.Debugw(ctx, "device-agent-started", log.Fields{"device-id": agent.deviceID})
khenaidoo442e7c72020-03-10 16:13:48 -0400188
Kent Hagermancba2f302020-07-28 13:37:36 -0400189 return agent.getDeviceReadOnly(ctx)
khenaidoob9203542018-09-17 22:56:37 -0400190}
191
khenaidoo4d4802d2018-10-04 21:59:49 -0400192// stop stops the device agent. Not much to do for now
Kent Hagerman2b216042020-04-03 18:28:56 -0400193func (agent *Agent) stop(ctx context.Context) error {
khenaidoo442e7c72020-03-10 16:13:48 -0400194 needToStop := false
195 if agent.stopOnce.Do(func() { needToStop = true }); !needToStop {
196 return nil
197 }
198 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
199 return err
200 }
201 defer agent.requestQueue.RequestComplete()
khenaidoo49085352020-01-13 19:15:43 -0500202
Himani Chawlab4c25912020-11-12 17:16:38 +0530203 logger.Infow(ctx, "stopping-device-agent", log.Fields{"device-id": agent.deviceID, "parent-id": agent.parentID})
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530204 // Remove the device transient loader
205 if err := agent.deleteTransientState(ctx); err != nil {
206 return err
207 }
khenaidoo0a822f92019-05-08 15:15:57 -0400208 // Remove the device from the KV store
Kent Hagermanf5a67352020-04-30 15:15:26 -0400209 if err := agent.dbProxy.Remove(ctx, agent.deviceID); err != nil {
khenaidoo442e7c72020-03-10 16:13:48 -0400210 return err
Thomas Lee Se5a44012019-11-07 20:32:24 +0530211 }
Abhilash Laxmeshwar75517ea2021-11-24 18:38:10 +0530212 //send the device event to the message bus
213 _ = agent.deviceMgr.Agent.SendDeviceDeletedEvent(ctx, agent.device, time.Now().Unix())
khenaidoo442e7c72020-03-10 16:13:48 -0400214
khenaidoo442e7c72020-03-10 16:13:48 -0400215 close(agent.exitChannel)
216
217 agent.stopped = true
218
Rohan Agrawal31f21802020-06-12 05:38:46 +0000219 logger.Infow(ctx, "device-agent-stopped", log.Fields{"device-id": agent.deviceID, "parent-id": agent.parentID})
khenaidoo442e7c72020-03-10 16:13:48 -0400220
221 return nil
khenaidoob9203542018-09-17 22:56:37 -0400222}
223
Scott Baker80678602019-11-14 16:57:36 -0800224// Load the most recent state from the KVStore for the device.
Kent Hagerman2b216042020-04-03 18:28:56 -0400225func (agent *Agent) reconcileWithKVStore(ctx context.Context) {
khenaidoo442e7c72020-03-10 16:13:48 -0400226 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
Rohan Agrawal31f21802020-06-12 05:38:46 +0000227 logger.Warnw(ctx, "request-aborted", log.Fields{"device-id": agent.deviceID, "error": err})
khenaidoo442e7c72020-03-10 16:13:48 -0400228 return
229 }
230 defer agent.requestQueue.RequestComplete()
Rohan Agrawal31f21802020-06-12 05:38:46 +0000231 logger.Debug(ctx, "reconciling-device-agent-devicetype")
Scott Baker80678602019-11-14 16:57:36 -0800232 // TODO: context timeout
Kent Hagerman4f355f52020-03-30 16:01:33 -0400233 device := &voltha.Device{}
Kent Hagermanf5a67352020-04-30 15:15:26 -0400234 if have, err := agent.dbProxy.Get(ctx, agent.deviceID, device); err != nil {
Rohan Agrawal31f21802020-06-12 05:38:46 +0000235 logger.Errorw(ctx, "kv-get-failed", log.Fields{"device-id": agent.deviceID, "error": err})
Thomas Lee Se5a44012019-11-07 20:32:24 +0530236 return
Kent Hagerman4f355f52020-03-30 16:01:33 -0400237 } else if !have {
238 return // not found in kv
Thomas Lee Se5a44012019-11-07 20:32:24 +0530239 }
Kent Hagerman4f355f52020-03-30 16:01:33 -0400240
khenaidood948f772021-08-11 17:49:24 -0400241 agent.deviceType = device.Type
Kent Hagerman4f355f52020-03-30 16:01:33 -0400242 agent.device = device
khenaidood948f772021-08-11 17:49:24 -0400243 agent.adapterEndpoint = device.AdapterEndpoint
Kent Hagerman2a07b862020-06-19 15:23:07 -0400244 agent.portLoader.Load(ctx)
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530245 agent.transientStateLoader.Load(ctx)
246
Rohan Agrawal31f21802020-06-12 05:38:46 +0000247 logger.Debugw(ctx, "reconciled-device-agent-devicetype", log.Fields{"device-id": agent.deviceID, "type": agent.deviceType})
Scott Baker80678602019-11-14 16:57:36 -0800248}
249
khenaidoo442e7c72020-03-10 16:13:48 -0400250// onSuccess is a common callback for scenarios where we receive a nil response following a request to an adapter
khenaidood948f772021-08-11 17:49:24 -0400251func (agent *Agent) onSuccess(ctx context.Context, prevState, currState *common.AdminState_Types, deviceUpdateLog bool) {
252 if deviceUpdateLog {
253 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_SUCCESS}
254 desc := "adapter-response"
255 agent.logDeviceUpdate(ctx, prevState, currState, requestStatus, nil, desc)
256 return
257 }
258 logger.Debugw(ctx, "successful-operation", log.Fields{"device-id": agent.deviceID, "rpc": coreutils.GetRPCMetadataFromContext(ctx)})
khenaidoo442e7c72020-03-10 16:13:48 -0400259}
260
261// onFailure is a common callback for scenarios where we receive an error response following a request to an adapter
262// and the only action required is to publish the failed result on kafka
khenaidood948f772021-08-11 17:49:24 -0400263func (agent *Agent) onFailure(ctx context.Context, err error, prevState, currState *common.AdminState_Types, deviceUpdateLog bool) {
264 // Send an event on kafka
265 rpce := agent.deviceMgr.NewRPCEvent(ctx, agent.deviceID, err.Error(), nil)
266 go agent.deviceMgr.SendRPCEvent(ctx, "RPC_ERROR_RAISE_EVENT", rpce,
267 voltha.EventCategory_COMMUNICATION, nil, time.Now().Unix())
khenaidoo442e7c72020-03-10 16:13:48 -0400268
khenaidood948f772021-08-11 17:49:24 -0400269 // Log the device update event
270 if deviceUpdateLog {
271 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
272 desc := "adapter-response"
273 agent.logDeviceUpdate(ctx, prevState, currState, requestStatus, err, desc)
274 return
khenaidoo442e7c72020-03-10 16:13:48 -0400275 }
khenaidood948f772021-08-11 17:49:24 -0400276 logger.Errorw(ctx, "failed-operation", log.Fields{"error": err, "device-id": agent.deviceID, "rpc": coreutils.GetRPCMetadataFromContext(ctx)})
khenaidoo442e7c72020-03-10 16:13:48 -0400277}
278
khenaidoo6cd8e862021-11-12 16:33:43 -0500279// onForceDeleteResponse is invoked following a force delete request to an adapter.
280func (agent *Agent) onForceDeleteResponse(ctx context.Context, prevState, currState *common.AdminState_Types, dErr error) {
281 // Log the status
282 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_SUCCESS}
283 if dErr != nil {
284 requestStatus = &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
285 }
286 agent.logDeviceUpdate(ctx, prevState, currState, requestStatus, dErr, "adapter-force-delete-response")
287
288 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
289 logger.Errorw(ctx, "failed-getting-device-request-lock", log.Fields{"device-id": agent.deviceID, "error": err})
290 }
291 previousDeviceTransientState := agent.getTransientState()
292 newDevice := agent.cloneDeviceWithoutLock()
293
294 // Even on a delete error response, cleaup the device in the core
295 requestStatus = &common.OperationResp{Code: common.OperationResp_OPERATION_SUCCESS}
296 err := agent.updateDeviceWithTransientStateAndReleaseLock(ctx, newDevice,
297 core.DeviceTransientState_DELETING_POST_ADAPTER_RESPONSE, previousDeviceTransientState)
298 if err != nil {
299 requestStatus = &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
300 }
301 agent.logDeviceUpdate(ctx, prevState, currState, requestStatus, err, "transient-state-update")
302}
303
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530304// onDeleteSuccess is a common callback for scenarios where we receive a nil response following a delete request
305// to an adapter.
khenaidood948f772021-08-11 17:49:24 -0400306func (agent *Agent) onDeleteSuccess(ctx context.Context, prevState, currState *common.AdminState_Types) {
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530307 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
khenaidood948f772021-08-11 17:49:24 -0400308 logger.Errorw(ctx, "delete-device-failure", log.Fields{"device-id": agent.deviceID, "error": err})
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530309 }
310 previousDeviceTransientState := agent.getTransientState()
311 newDevice := agent.cloneDeviceWithoutLock()
312 if err := agent.updateDeviceWithTransientStateAndReleaseLock(ctx, newDevice,
khenaidood948f772021-08-11 17:49:24 -0400313 core.DeviceTransientState_DELETING_POST_ADAPTER_RESPONSE, previousDeviceTransientState); err != nil {
314 logger.Errorw(ctx, "delete-device-failure", log.Fields{"device-id": agent.deviceID, "error": err})
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530315 }
khenaidood948f772021-08-11 17:49:24 -0400316 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_SUCCESS}
317 desc := "adapter-response"
318 agent.logDeviceUpdate(ctx, prevState, currState, requestStatus, nil, desc)
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530319}
320
321// onDeleteFailure is a common callback for scenarios where we receive an error response following a delete request
322// to an adapter and the only action required is to return the error response.
khenaidood948f772021-08-11 17:49:24 -0400323func (agent *Agent) onDeleteFailure(ctx context.Context, err error, prevState, currState *common.AdminState_Types) {
324 logger.Errorw(ctx, "rpc-failed", log.Fields{"rpc": coreutils.GetRPCMetadataFromContext(ctx), "device-id": agent.deviceID, "error": err})
325
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530326 //Only updating of transient state is required, no transition.
khenaidood948f772021-08-11 17:49:24 -0400327 if er := agent.updateTransientState(ctx, core.DeviceTransientState_DELETE_FAILED); er != nil {
328 logger.Errorw(ctx, "failed-to-update-transient-state-as-delete-failed", log.Fields{"device-id": agent.deviceID, "error": er})
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530329 }
khenaidood948f772021-08-11 17:49:24 -0400330 rpce := agent.deviceMgr.NewRPCEvent(ctx, agent.deviceID, err.Error(), nil)
331 go agent.deviceMgr.SendRPCEvent(ctx, "RPC_ERROR_RAISE_EVENT", rpce,
332 voltha.EventCategory_COMMUNICATION, nil, time.Now().Unix())
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530333
khenaidood948f772021-08-11 17:49:24 -0400334 // Log the device update event
335 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
336 desc := "adapter-response"
337 agent.logDeviceUpdate(ctx, prevState, currState, requestStatus, err, desc)
Maninder9a1bc0d2020-10-26 11:34:02 +0530338}
339
Kent Hagermancba2f302020-07-28 13:37:36 -0400340// getDeviceReadOnly returns a device which MUST NOT be modified, but is safe to keep forever.
341func (agent *Agent) getDeviceReadOnly(ctx context.Context) (*voltha.Device, error) {
khenaidoo442e7c72020-03-10 16:13:48 -0400342 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
343 return nil, err
344 }
345 defer agent.requestQueue.RequestComplete()
Kent Hagermancba2f302020-07-28 13:37:36 -0400346 return agent.device, nil
khenaidoo92e62c52018-10-03 14:02:54 -0400347}
348
Kent Hagermancba2f302020-07-28 13:37:36 -0400349// getDeviceReadOnlyWithoutLock returns a device which MUST NOT be modified, but is safe to keep forever. This is very efficient.
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400350// The device lock MUST be held by the caller.
Kent Hagermancba2f302020-07-28 13:37:36 -0400351func (agent *Agent) getDeviceReadOnlyWithoutLock() *voltha.Device {
khenaidoo0db4c812020-05-27 15:27:30 -0400352 return agent.device
khenaidoo92e62c52018-10-03 14:02:54 -0400353}
354
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400355// cloneDeviceWithoutLock returns a copy of the device which is safe to modify.
356// The device lock MUST be held by the caller.
357func (agent *Agent) cloneDeviceWithoutLock() *voltha.Device {
358 return proto.Clone(agent.device).(*voltha.Device)
359}
360
khenaidood948f772021-08-11 17:49:24 -0400361func (agent *Agent) updateDeviceTypeAndEndpoint(ctx context.Context) error {
362 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
363 return err
364 }
365 changed := false
366 cloned := agent.cloneDeviceWithoutLock()
367 if cloned.Type == "" {
368 adapterType, err := agent.adapterMgr.GetAdapterType(cloned.Type)
369 if err != nil {
370 agent.requestQueue.RequestComplete()
371 return err
372 }
373 cloned.Type = adapterType
374 changed = true
375 }
376
377 if cloned.AdapterEndpoint == "" {
378 var err error
379 if cloned.AdapterEndpoint, err = agent.adapterMgr.GetAdapterEndpoint(ctx, cloned.Id, cloned.Type); err != nil {
380 agent.requestQueue.RequestComplete()
381 return err
382 }
383 agent.adapterEndpoint = cloned.AdapterEndpoint
384 changed = true
385 }
386
387 if changed {
388 return agent.updateDeviceAndReleaseLock(ctx, cloned)
389 }
390 agent.requestQueue.RequestComplete()
391 return nil
392}
393
khenaidoo3ab34882019-05-02 21:33:30 -0400394// enableDevice activates a preprovisioned or a disable device
Kent Hagerman2b216042020-04-03 18:28:56 -0400395func (agent *Agent) enableDevice(ctx context.Context) error {
Maninder9a1bc0d2020-10-26 11:34:02 +0530396 //To preserve and use oldDevice state as prev state in new device
khenaidood948f772021-08-11 17:49:24 -0400397 var err error
Maninder9a1bc0d2020-10-26 11:34:02 +0530398 var desc string
khenaidood948f772021-08-11 17:49:24 -0400399 var prevAdminState, currAdminState common.AdminState_Types
400 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
Maninder9a1bc0d2020-10-26 11:34:02 +0530401
khenaidood948f772021-08-11 17:49:24 -0400402 defer func() { agent.logDeviceUpdate(ctx, &prevAdminState, &currAdminState, requestStatus, err, desc) }()
Maninder9a1bc0d2020-10-26 11:34:02 +0530403
khenaidood948f772021-08-11 17:49:24 -0400404 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
khenaidoo442e7c72020-03-10 16:13:48 -0400405 return err
406 }
Himani Chawlab4c25912020-11-12 17:16:38 +0530407 logger.Debugw(ctx, "enable-device", log.Fields{"device-id": agent.deviceID})
khenaidoo21d51152019-02-01 13:48:37 -0500408
Kent Hagermancba2f302020-07-28 13:37:36 -0400409 oldDevice := agent.getDeviceReadOnlyWithoutLock()
khenaidood948f772021-08-11 17:49:24 -0400410 prevAdminState = oldDevice.AdminState
Maninder9a1bc0d2020-10-26 11:34:02 +0530411
Maninder2195ccc2021-06-23 20:23:01 +0530412 if !agent.proceedWithRequest(oldDevice) {
413 agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -0400414 err = status.Errorf(codes.FailedPrecondition, "cannot complete operation as device deletion is in progress or reconciling is in progress/failed: %s", agent.deviceID)
415 return err
Maninder2195ccc2021-06-23 20:23:01 +0530416 }
Mahir Gunyel92dd1212021-10-22 11:42:56 -0700417 //vol-4275 TST meeting 08/04/2021: Let EnableDevice to be called again if device is in FAILED operational state,
418 //even the admin state is ENABLED.
419 if oldDevice.AdminState == voltha.AdminState_ENABLED && oldDevice.OperStatus != voltha.OperStatus_FAILED {
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400420 logger.Warnw(ctx, "device-already-enabled", log.Fields{"device-id": agent.deviceID})
421 agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -0400422 err = status.Errorf(codes.FailedPrecondition, fmt.Sprintf("cannot-enable-an-already-enabled-device: %s", oldDevice.Id))
Matteo Scandolod525ae32020-04-02 17:27:29 -0700423 return err
npujar1d86a522019-11-14 17:11:16 +0530424 }
425
khenaidood948f772021-08-11 17:49:24 -0400426 // Verify whether there is a device type that supports this device type
427 _, err = agent.adapterMgr.GetAdapterType(oldDevice.Type)
428 if err != nil {
429 agent.requestQueue.RequestComplete()
430 return err
431 }
432
433 // Update device adapter endpoint if not set. This is set once by the Core and use as is by the adapters. E.g if this is a
434 // child device then the parent adapter will use this device's adapter endpoint (set here) to communicate with it.
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400435 newDevice := agent.cloneDeviceWithoutLock()
khenaidood948f772021-08-11 17:49:24 -0400436 if newDevice.AdapterEndpoint == "" {
437 if newDevice.AdapterEndpoint, err = agent.adapterMgr.GetAdapterEndpoint(ctx, newDevice.Id, newDevice.Type); err != nil {
438 agent.requestQueue.RequestComplete()
439 return err
440 }
441 agent.adapterEndpoint = newDevice.AdapterEndpoint
442 }
npujar1d86a522019-11-14 17:11:16 +0530443
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400444 // Update the Admin State and set the operational state to activating before sending the request to the Adapters
445 newDevice.AdminState = voltha.AdminState_ENABLED
446 newDevice.OperStatus = voltha.OperStatus_ACTIVATING
Maninder9a1bc0d2020-10-26 11:34:02 +0530447
khenaidoo442e7c72020-03-10 16:13:48 -0400448 // Adopt the device if it was in pre-provision state. In all other cases, try to re-enable it.
khenaidood948f772021-08-11 17:49:24 -0400449 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
khenaidoo442e7c72020-03-10 16:13:48 -0400450 if err != nil {
khenaidood948f772021-08-11 17:49:24 -0400451 logger.Errorw(ctx, "grpc-client-nil",
452 log.Fields{
453 "error": err,
454 "device-id": agent.deviceID,
455 "device-type": agent.deviceType,
456 "adapter-endpoint": newDevice.AdapterEndpoint,
457 })
458 agent.requestQueue.RequestComplete()
khenaidoo442e7c72020-03-10 16:13:48 -0400459 return err
460 }
khenaidood948f772021-08-11 17:49:24 -0400461 subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
462 requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS
463 go func() {
464 defer cancel()
465 var err error
466 if oldDevice.AdminState == voltha.AdminState_PREPROVISIONED {
467 _, err = client.AdoptDevice(subCtx, newDevice)
468 } else {
469 _, err = client.ReEnableDevice(subCtx, newDevice)
470 }
471 if err == nil {
472 agent.onSuccess(subCtx, nil, nil, true)
473 } else {
474 agent.onFailure(subCtx, err, nil, nil, true)
Himani Chawlab4c25912020-11-12 17:16:38 +0530475 }
476 }()
khenaidood948f772021-08-11 17:49:24 -0400477
478 // Update device
479 if err = agent.updateDeviceAndReleaseLock(ctx, newDevice); err != nil {
480 return err
khenaidoo2c6a0992019-04-29 13:46:56 -0400481 }
khenaidood948f772021-08-11 17:49:24 -0400482 currAdminState = newDevice.AdminState
483 return nil
khenaidoo2c6a0992019-04-29 13:46:56 -0400484}
485
A R Karthick5c28f552019-12-11 22:47:44 -0800486//addFlowsAndGroups adds the "newFlows" and "newGroups" from the existing flows/groups and sends the update to the
487//adapters
khenaidoo9beaaf12021-10-19 17:32:01 -0400488func (agent *Agent) addFlowsAndGroups(ctx context.Context, newFlows []*ofp.OfpFlowStats, newGroups []*ofp.OfpGroupEntry, flowMetadata *ofp.FlowMetadata) error {
Mahir Gunyel03de0d32020-06-03 01:36:59 -0700489 var flwResponse, grpResponse coreutils.Response
490 var err error
491 //if new flow list is empty then the called function returns quickly
492 if flwResponse, err = agent.addFlowsToAdapter(ctx, newFlows, flowMetadata); err != nil {
A R Karthick5c28f552019-12-11 22:47:44 -0800493 return err
494 }
Mahir Gunyel03de0d32020-06-03 01:36:59 -0700495 //if new group list is empty then the called function returns quickly
496 if grpResponse, err = agent.addGroupsToAdapter(ctx, newGroups, flowMetadata); err != nil {
497 return err
498 }
Himani Chawla4b4bd252021-11-08 15:59:40 +0530499 if errs := coreutils.WaitForNilOrErrorResponses(agent.flowTimeout, flwResponse, grpResponse); errs != nil {
khenaidood948f772021-08-11 17:49:24 -0400500 logger.Warnw(ctx, "adapter-response", log.Fields{"device-id": agent.deviceID, "result": errs})
khenaidoo442e7c72020-03-10 16:13:48 -0400501 return status.Errorf(codes.Aborted, "flow-failure-device-%s", agent.deviceID)
khenaidoo0458db62019-06-20 08:50:36 -0400502 }
khenaidoo0458db62019-06-20 08:50:36 -0400503 return nil
504}
505
A R Karthick5c28f552019-12-11 22:47:44 -0800506//deleteFlowsAndGroups removes the "flowsToDel" and "groupsToDel" from the existing flows/groups and sends the update to the
507//adapters
khenaidoo9beaaf12021-10-19 17:32:01 -0400508func (agent *Agent) deleteFlowsAndGroups(ctx context.Context, flowsToDel []*ofp.OfpFlowStats, groupsToDel []*ofp.OfpGroupEntry, flowMetadata *ofp.FlowMetadata) error {
Mahir Gunyel03de0d32020-06-03 01:36:59 -0700509 var flwResponse, grpResponse coreutils.Response
510 var err error
511 if flwResponse, err = agent.deleteFlowsFromAdapter(ctx, flowsToDel, flowMetadata); err != nil {
A R Karthick5c28f552019-12-11 22:47:44 -0800512 return err
513 }
Mahir Gunyel03de0d32020-06-03 01:36:59 -0700514 if grpResponse, err = agent.deleteGroupsFromAdapter(ctx, groupsToDel, flowMetadata); err != nil {
515 return err
516 }
517
Himani Chawla4b4bd252021-11-08 15:59:40 +0530518 if res := coreutils.WaitForNilOrErrorResponses(agent.flowTimeout, flwResponse, grpResponse); res != nil {
khenaidoo0458db62019-06-20 08:50:36 -0400519 return status.Errorf(codes.Aborted, "errors-%s", res)
520 }
521 return nil
khenaidoo0458db62019-06-20 08:50:36 -0400522}
523
A R Karthick5c28f552019-12-11 22:47:44 -0800524//updateFlowsAndGroups replaces the existing flows and groups with "updatedFlows" and "updatedGroups" respectively. It
525//also sends the updates to the adapters
khenaidoo9beaaf12021-10-19 17:32:01 -0400526func (agent *Agent) updateFlowsAndGroups(ctx context.Context, updatedFlows []*ofp.OfpFlowStats, updatedGroups []*ofp.OfpGroupEntry, flowMetadata *ofp.FlowMetadata) error {
Mahir Gunyel03de0d32020-06-03 01:36:59 -0700527 var flwResponse, grpResponse coreutils.Response
528 var err error
529 if flwResponse, err = agent.updateFlowsToAdapter(ctx, updatedFlows, flowMetadata); err != nil {
A R Karthick5c28f552019-12-11 22:47:44 -0800530 return err
531 }
Mahir Gunyel03de0d32020-06-03 01:36:59 -0700532 if grpResponse, err = agent.updateGroupsToAdapter(ctx, updatedGroups, flowMetadata); err != nil {
533 return err
534 }
535
Himani Chawla4b4bd252021-11-08 15:59:40 +0530536 if res := coreutils.WaitForNilOrErrorResponses(agent.flowTimeout, flwResponse, grpResponse); res != nil {
khenaidoo0458db62019-06-20 08:50:36 -0400537 return status.Errorf(codes.Aborted, "errors-%s", res)
538 }
539 return nil
khenaidoo19d7b632018-10-30 10:49:50 -0400540}
541
khenaidoo4d4802d2018-10-04 21:59:49 -0400542//disableDevice disable a device
Kent Hagerman2b216042020-04-03 18:28:56 -0400543func (agent *Agent) disableDevice(ctx context.Context) error {
khenaidood948f772021-08-11 17:49:24 -0400544 var err error
Maninder9a1bc0d2020-10-26 11:34:02 +0530545 var desc string
khenaidood948f772021-08-11 17:49:24 -0400546 var prevAdminState, currAdminState common.AdminState_Types
547 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
548 defer func() { agent.logDeviceUpdate(ctx, &prevAdminState, &currAdminState, requestStatus, err, desc) }()
Maninder9a1bc0d2020-10-26 11:34:02 +0530549
khenaidood948f772021-08-11 17:49:24 -0400550 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
khenaidoo442e7c72020-03-10 16:13:48 -0400551 return err
552 }
Himani Chawlab4c25912020-11-12 17:16:38 +0530553 logger.Debugw(ctx, "disable-device", log.Fields{"device-id": agent.deviceID})
khenaidoo6e55d9e2019-12-12 18:26:26 -0500554
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400555 cloned := agent.cloneDeviceWithoutLock()
khenaidood948f772021-08-11 17:49:24 -0400556 prevAdminState = agent.device.AdminState
khenaidoo6e55d9e2019-12-12 18:26:26 -0500557
Maninder2195ccc2021-06-23 20:23:01 +0530558 if !agent.proceedWithRequest(cloned) {
khenaidood948f772021-08-11 17:49:24 -0400559 err = status.Errorf(codes.FailedPrecondition, "cannot complete operation as device deletion is in progress or reconciling is in progress/failed: %s", agent.deviceID)
Maninder2195ccc2021-06-23 20:23:01 +0530560 agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -0400561 return err
Maninder2195ccc2021-06-23 20:23:01 +0530562 }
563
khenaidoo6e55d9e2019-12-12 18:26:26 -0500564 if cloned.AdminState == voltha.AdminState_DISABLED {
Maninder9a1bc0d2020-10-26 11:34:02 +0530565 desc = "device-already-disabled"
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400566 agent.requestQueue.RequestComplete()
npujar1d86a522019-11-14 17:11:16 +0530567 return nil
568 }
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530569 if cloned.AdminState == voltha.AdminState_PREPROVISIONED {
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400570 agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -0400571 err = status.Errorf(codes.FailedPrecondition, "deviceId:%s, invalid-admin-state:%s", agent.deviceID, cloned.AdminState)
572 return err
npujar1d86a522019-11-14 17:11:16 +0530573 }
Maninder0aabf0c2021-03-17 14:55:14 +0530574
npujar1d86a522019-11-14 17:11:16 +0530575 // Update the Admin State and operational state before sending the request out
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400576 cloned.AdminState = voltha.AdminState_DISABLED
577 cloned.OperStatus = voltha.OperStatus_UNKNOWN
Maninder9a1bc0d2020-10-26 11:34:02 +0530578
khenaidood948f772021-08-11 17:49:24 -0400579 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
khenaidoo442e7c72020-03-10 16:13:48 -0400580 if err != nil {
khenaidood948f772021-08-11 17:49:24 -0400581 logger.Errorw(ctx, "grpc-client-nil",
582 log.Fields{
583 "error": err,
584 "device-id": agent.deviceID,
585 "device-type": agent.deviceType,
586 "adapter-endpoint": cloned.AdapterEndpoint,
587 })
588 agent.requestQueue.RequestComplete()
npujar1d86a522019-11-14 17:11:16 +0530589 return err
khenaidoo0a822f92019-05-08 15:15:57 -0400590 }
khenaidood948f772021-08-11 17:49:24 -0400591 subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
592 requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS
593 go func() {
594 defer cancel()
595 _, err := client.DisableDevice(subCtx, cloned)
596 if err == nil {
597 agent.onSuccess(subCtx, nil, nil, true)
598 } else {
599 agent.onFailure(subCtx, err, nil, nil, true)
600 }
601 }()
Maninder9a1bc0d2020-10-26 11:34:02 +0530602
khenaidood948f772021-08-11 17:49:24 -0400603 // Update device
604 if err = agent.updateDeviceAndReleaseLock(ctx, cloned); err != nil {
605 return err
606 }
607 currAdminState = cloned.AdminState
khenaidoo0a822f92019-05-08 15:15:57 -0400608
khenaidoo92e62c52018-10-03 14:02:54 -0400609 return nil
610}
611
Kent Hagerman2b216042020-04-03 18:28:56 -0400612func (agent *Agent) rebootDevice(ctx context.Context) error {
Maninder9a1bc0d2020-10-26 11:34:02 +0530613 var desc string
khenaidood948f772021-08-11 17:49:24 -0400614 var err error
615 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
616 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
Maninder9a1bc0d2020-10-26 11:34:02 +0530617
khenaidood948f772021-08-11 17:49:24 -0400618 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
Maninder9a1bc0d2020-10-26 11:34:02 +0530619 desc = err.Error()
npujar1d86a522019-11-14 17:11:16 +0530620 return err
khenaidoo4d4802d2018-10-04 21:59:49 -0400621 }
khenaidoo442e7c72020-03-10 16:13:48 -0400622 defer agent.requestQueue.RequestComplete()
Himani Chawlab4c25912020-11-12 17:16:38 +0530623 logger.Debugw(ctx, "reboot-device", log.Fields{"device-id": agent.deviceID})
khenaidoo442e7c72020-03-10 16:13:48 -0400624
Kent Hagermancba2f302020-07-28 13:37:36 -0400625 device := agent.getDeviceReadOnlyWithoutLock()
Maninder2195ccc2021-06-23 20:23:01 +0530626
627 if !agent.proceedWithRequest(device) {
khenaidood948f772021-08-11 17:49:24 -0400628 err = status.Errorf(codes.FailedPrecondition, "cannot complete operation as device deletion is in progress or reconciling is in progress/failed:%s", agent.deviceID)
khenaidoo442e7c72020-03-10 16:13:48 -0400629 return err
630 }
Maninder9a1bc0d2020-10-26 11:34:02 +0530631
khenaidood948f772021-08-11 17:49:24 -0400632 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
633 if err != nil {
634 logger.Errorw(ctx, "grpc-client-nil",
635 log.Fields{
636 "error": err,
637 "device-id": agent.deviceID,
638 "device-type": agent.deviceType,
639 "adapter-endpoint": device.AdapterEndpoint,
640 })
641 return err
642 }
643 subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
644 requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS
645 go func() {
646 defer cancel()
647 _, err := client.RebootDevice(subCtx, device)
648 if err == nil {
649 agent.onSuccess(subCtx, nil, nil, true)
650 } else {
651 agent.onFailure(subCtx, err, nil, nil, true)
652 }
653 }()
khenaidoo4d4802d2018-10-04 21:59:49 -0400654 return nil
655}
656
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530657func (agent *Agent) deleteDeviceForce(ctx context.Context) error {
Himani Chawlab4c25912020-11-12 17:16:38 +0530658 logger.Debugw(ctx, "delete-device-force", log.Fields{"device-id": agent.deviceID})
Maninder9a1bc0d2020-10-26 11:34:02 +0530659
660 var desc string
khenaidood948f772021-08-11 17:49:24 -0400661 var err error
662 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
663 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
Maninder9a1bc0d2020-10-26 11:34:02 +0530664
khenaidood948f772021-08-11 17:49:24 -0400665 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530666 return err
667 }
668 // Get the device Transient state, return err if it is DELETING
669 previousDeviceTransientState := agent.getTransientState()
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530670 device := agent.cloneDeviceWithoutLock()
khenaidood948f772021-08-11 17:49:24 -0400671 if !agent.isForceDeletingAllowed(previousDeviceTransientState, device) {
672 agent.requestQueue.RequestComplete()
673 err = status.Error(codes.FailedPrecondition, fmt.Sprintf("deviceId:%s, force deletion is in progress", agent.deviceID))
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530674 return err
675 }
Himani Chawlab4c25912020-11-12 17:16:38 +0530676
khenaidood948f772021-08-11 17:49:24 -0400677 previousAdminState := device.AdminState
678 if previousAdminState != common.AdminState_PREPROVISIONED {
khenaidoo9beaaf12021-10-19 17:32:01 -0400679 var client adapter_service.AdapterServiceClient
khenaidood948f772021-08-11 17:49:24 -0400680 client, err = agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530681 if err != nil {
khenaidood948f772021-08-11 17:49:24 -0400682 logger.Errorw(ctx, "grpc-client-nil",
683 log.Fields{
684 "error": err,
685 "device-id": agent.deviceID,
686 "device-type": agent.deviceType,
687 "adapter-endpoint": device.AdapterEndpoint,
688 })
689 agent.requestQueue.RequestComplete()
khenaidoo68a5e0c2021-11-06 13:08:03 -0400690 return fmt.Errorf("remote-not-reachable %w", errNoConnection)
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530691 }
khenaidood948f772021-08-11 17:49:24 -0400692 subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
693 requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS
694 go func() {
695 defer cancel()
696 _, err := client.DeleteDevice(subCtx, device)
697 if err == nil {
698 agent.onSuccess(subCtx, nil, nil, true)
699 } else {
700 agent.onFailure(subCtx, err, nil, nil, true)
701 }
702 }()
703 }
704
705 // Update device
706 if err = agent.updateDeviceWithTransientStateAndReleaseLock(ctx, device,
707 core.DeviceTransientState_FORCE_DELETING, previousDeviceTransientState); err != nil {
708 return err
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530709 }
710 return nil
711}
712
Kent Hagerman2b216042020-04-03 18:28:56 -0400713func (agent *Agent) deleteDevice(ctx context.Context) error {
Himani Chawlab4c25912020-11-12 17:16:38 +0530714 logger.Debugw(ctx, "delete-device", log.Fields{"device-id": agent.deviceID})
Maninder9a1bc0d2020-10-26 11:34:02 +0530715
716 var desc string
khenaidood948f772021-08-11 17:49:24 -0400717 var err error
718 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
719 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
Maninder9a1bc0d2020-10-26 11:34:02 +0530720
khenaidood948f772021-08-11 17:49:24 -0400721 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
Maninder9a1bc0d2020-10-26 11:34:02 +0530722 desc = err.Error()
khenaidoo442e7c72020-03-10 16:13:48 -0400723 return err
724 }
Maninder0aabf0c2021-03-17 14:55:14 +0530725
Maninder2195ccc2021-06-23 20:23:01 +0530726 device := agent.cloneDeviceWithoutLock()
727
728 if !agent.proceedWithRequest(device) {
Maninder0aabf0c2021-03-17 14:55:14 +0530729 agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -0400730 err = status.Errorf(codes.FailedPrecondition, "cannot complete operation as device deletion is in progress or reconciling is in progress/failed: %s", agent.deviceID)
731 return err
Maninder0aabf0c2021-03-17 14:55:14 +0530732 }
733
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530734 // Get the device Transient state, return err if it is DELETING
735 previousDeviceTransientState := agent.getTransientState()
khenaidoo6e55d9e2019-12-12 18:26:26 -0500736
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530737 previousAdminState := device.AdminState
738 // Change the device transient state to DELETING_FROM_ADAPTER state till the device is removed from adapters.
khenaidood948f772021-08-11 17:49:24 -0400739 currentDeviceTransientState := core.DeviceTransientState_DELETING_FROM_ADAPTER
khenaidoo442e7c72020-03-10 16:13:48 -0400740
khenaidood948f772021-08-11 17:49:24 -0400741 if previousAdminState == common.AdminState_PREPROVISIONED {
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530742 // Change the state to DELETING POST ADAPTER RESPONSE directly as adapters have no info of the device.
khenaidood948f772021-08-11 17:49:24 -0400743 currentDeviceTransientState = core.DeviceTransientState_DELETING_POST_ADAPTER_RESPONSE
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530744 }
Gustavo Silva9a0ed002022-10-11 11:06:58 -0300745 // Update device and release lock
746 if err = agent.updateDeviceWithTransientStateAndReleaseLock(ctx, device,
747 currentDeviceTransientState, previousDeviceTransientState); err != nil {
748 desc = err.Error()
749 return err
750 }
khenaidood948f772021-08-11 17:49:24 -0400751 // If the device was in pre-prov state (only parent device are in that state) then do not send the request to the
752 // adapter
753 if previousAdminState != common.AdminState_PREPROVISIONED {
khenaidoo9beaaf12021-10-19 17:32:01 -0400754 var client adapter_service.AdapterServiceClient
khenaidood948f772021-08-11 17:49:24 -0400755 client, err = agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
756 if err != nil {
757 logger.Errorw(ctx, "grpc-client-nil",
758 log.Fields{
759 "error": err,
760 "device-id": agent.deviceID,
761 "device-type": agent.deviceType,
762 "adapter-endpoint": device.AdapterEndpoint,
763 })
764 agent.requestQueue.RequestComplete()
765 return err
766 }
767 subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
768 requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS
Gustavo Silva9a0ed002022-10-11 11:06:58 -0300769 if _, err = client.DeleteDevice(subCtx, device); err != nil {
770 agent.onDeleteFailure(subCtx, err, &previousAdminState, &agent.device.AdminState)
771 } else {
772 agent.onDeleteSuccess(subCtx, &previousAdminState, &agent.device.AdminState)
773 }
774 cancel()
khenaidood948f772021-08-11 17:49:24 -0400775 }
Gustavo Silva9a0ed002022-10-11 11:06:58 -0300776 return err
khenaidoo4d4802d2018-10-04 21:59:49 -0400777}
778
Kent Hagerman2b216042020-04-03 18:28:56 -0400779func (agent *Agent) setParentID(ctx context.Context, device *voltha.Device, parentID string) error {
khenaidoo442e7c72020-03-10 16:13:48 -0400780 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
781 return err
782 }
Himani Chawlab4c25912020-11-12 17:16:38 +0530783 logger.Debugw(ctx, "set-parent-id", log.Fields{"device-id": device.Id, "parent-id": parentID})
khenaidoo6e55d9e2019-12-12 18:26:26 -0500784
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400785 cloned := agent.cloneDeviceWithoutLock()
npujar1d86a522019-11-14 17:11:16 +0530786 cloned.ParentId = parentID
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400787 return agent.updateDeviceAndReleaseLock(ctx, cloned)
khenaidooad06fd72019-10-28 12:26:05 -0400788}
789
khenaidoo442e7c72020-03-10 16:13:48 -0400790// getSwitchCapability retrieves the switch capability of a parent device
khenaidoo9beaaf12021-10-19 17:32:01 -0400791func (agent *Agent) getSwitchCapability(ctx context.Context) (*ca.SwitchCapability, error) {
Himani Chawlab4c25912020-11-12 17:16:38 +0530792 logger.Debugw(ctx, "get-switch-capability", log.Fields{"device-id": agent.deviceID})
khenaidoo442e7c72020-03-10 16:13:48 -0400793
Kent Hagermancba2f302020-07-28 13:37:36 -0400794 device, err := agent.getDeviceReadOnly(ctx)
khenaidoo442e7c72020-03-10 16:13:48 -0400795 if err != nil {
khenaidoob9203542018-09-17 22:56:37 -0400796 return nil, err
khenaidoob9203542018-09-17 22:56:37 -0400797 }
khenaidood948f772021-08-11 17:49:24 -0400798
799 // Get the gRPC client
800 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
khenaidoo442e7c72020-03-10 16:13:48 -0400801 if err != nil {
802 return nil, err
803 }
804
khenaidood948f772021-08-11 17:49:24 -0400805 return client.GetOfpDeviceInfo(ctx, device)
khenaidoob9203542018-09-17 22:56:37 -0400806}
807
khenaidood948f772021-08-11 17:49:24 -0400808func (agent *Agent) onPacketFailure(ctx context.Context, err error, packet *ofp.OfpPacketOut) {
809 logger.Errorw(ctx, "packet-out-error", log.Fields{
khenaidoo442e7c72020-03-10 16:13:48 -0400810 "device-id": agent.deviceID,
khenaidood948f772021-08-11 17:49:24 -0400811 "error": err.Error(),
812 "packet": hex.EncodeToString(packet.Data),
khenaidoo442e7c72020-03-10 16:13:48 -0400813 })
khenaidood948f772021-08-11 17:49:24 -0400814 rpce := agent.deviceMgr.NewRPCEvent(ctx, agent.deviceID, err.Error(), nil)
815 go agent.deviceMgr.SendRPCEvent(ctx, "RPC_ERROR_RAISE_EVENT", rpce,
816 voltha.EventCategory_COMMUNICATION, nil, time.Now().Unix())
khenaidoo442e7c72020-03-10 16:13:48 -0400817}
818
Kent Hagerman2b216042020-04-03 18:28:56 -0400819func (agent *Agent) packetOut(ctx context.Context, outPort uint32, packet *ofp.OfpPacketOut) error {
Scott Baker80678602019-11-14 16:57:36 -0800820 if agent.deviceType == "" {
npujar467fe752020-01-16 20:17:45 +0530821 agent.reconcileWithKVStore(ctx)
Scott Baker80678602019-11-14 16:57:36 -0800822 }
khenaidoofdbad6e2018-11-06 22:26:38 -0500823 // Send packet to adapter
khenaidood948f772021-08-11 17:49:24 -0400824 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
khenaidoo442e7c72020-03-10 16:13:48 -0400825 if err != nil {
khenaidood948f772021-08-11 17:49:24 -0400826 logger.Errorw(ctx, "grpc-client-nil",
827 log.Fields{
828 "error": err,
829 "device-id": agent.deviceID,
830 "device-type": agent.deviceType,
831 })
832 return err
khenaidoofdbad6e2018-11-06 22:26:38 -0500833 }
khenaidood948f772021-08-11 17:49:24 -0400834 subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
835 go func() {
836 defer cancel()
khenaidoo9beaaf12021-10-19 17:32:01 -0400837 _, err := client.SendPacketOut(subCtx, &ca.PacketOut{
khenaidood948f772021-08-11 17:49:24 -0400838 DeviceId: agent.deviceID,
839 EgressPortNo: outPort,
840 Packet: packet,
841 })
842 if err == nil {
843 agent.onSuccess(subCtx, nil, nil, false)
844 } else {
845 agent.onPacketFailure(subCtx, err, packet)
846 }
847 }()
khenaidoofdbad6e2018-11-06 22:26:38 -0500848 return nil
849}
850
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400851func (agent *Agent) updateDeviceUsingAdapterData(ctx context.Context, device *voltha.Device) error {
khenaidood948f772021-08-11 17:49:24 -0400852 var err error
853 var desc string
854 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
855 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
856
857 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400858 return err
859 }
Himani Chawlab4c25912020-11-12 17:16:38 +0530860 logger.Debugw(ctx, "update-device-using-adapter-data", log.Fields{"device-id": device.Id})
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400861
862 cloned := agent.cloneDeviceWithoutLock()
Mahir Gunyel8e2707d2019-07-25 00:36:21 -0700863 cloned.Root = device.Root
864 cloned.Vendor = device.Vendor
865 cloned.Model = device.Model
866 cloned.SerialNumber = device.SerialNumber
867 cloned.MacAddress = device.MacAddress
868 cloned.Vlan = device.Vlan
869 cloned.Reason = device.Reason
Andrea Campanella025667e2021-01-14 11:50:07 +0100870 cloned.ImageDownloads = device.ImageDownloads
khenaidood948f772021-08-11 17:49:24 -0400871 cloned.OperStatus = device.OperStatus
872 cloned.ConnectStatus = device.ConnectStatus
873 if err = agent.updateDeviceAndReleaseLock(ctx, cloned); err == nil {
874 requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
875 }
876 return err
khenaidoo43c82122018-11-22 18:38:28 -0500877}
878
Kent Hagerman2b216042020-04-03 18:28:56 -0400879func (agent *Agent) updateDeviceStatus(ctx context.Context, operStatus voltha.OperStatus_Types, connStatus voltha.ConnectStatus_Types) error {
khenaidood948f772021-08-11 17:49:24 -0400880 var err error
881 var desc string
882 opStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
883 defer func() { agent.logDeviceUpdate(ctx, nil, nil, opStatus, err, desc) }()
884
885 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
khenaidoo442e7c72020-03-10 16:13:48 -0400886 return err
887 }
khenaidoo6e55d9e2019-12-12 18:26:26 -0500888
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400889 cloned := agent.cloneDeviceWithoutLock()
npujar1d86a522019-11-14 17:11:16 +0530890 // Ensure the enums passed in are valid - they will be invalid if they are not set when this function is invoked
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400891 if s, ok := voltha.ConnectStatus_Types_name[int32(connStatus)]; ok {
khenaidood948f772021-08-11 17:49:24 -0400892 logger.Debugw(ctx, "update-device-conn-status", log.Fields{"ok": ok, "val": s})
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400893 cloned.ConnectStatus = connStatus
npujar1d86a522019-11-14 17:11:16 +0530894 }
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400895 if s, ok := voltha.OperStatus_Types_name[int32(operStatus)]; ok {
khenaidood948f772021-08-11 17:49:24 -0400896 logger.Debugw(ctx, "update-device-oper-status", log.Fields{"ok": ok, "val": s})
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400897 cloned.OperStatus = operStatus
npujar1d86a522019-11-14 17:11:16 +0530898 }
Himani Chawlab4c25912020-11-12 17:16:38 +0530899 logger.Debugw(ctx, "update-device-status", log.Fields{"device-id": cloned.Id, "oper-status": cloned.OperStatus, "connect-status": cloned.ConnectStatus})
npujar1d86a522019-11-14 17:11:16 +0530900 // Store the device
khenaidood948f772021-08-11 17:49:24 -0400901 if err = agent.updateDeviceAndReleaseLock(ctx, cloned); err == nil {
902 opStatus.Code = common.OperationResp_OPERATION_SUCCESS
903 }
904 return err
khenaidoo92e62c52018-10-03 14:02:54 -0400905}
906
khenaidoob9203542018-09-17 22:56:37 -0400907// TODO: A generic device update by attribute
Kent Hagerman2b216042020-04-03 18:28:56 -0400908func (agent *Agent) updateDeviceAttribute(ctx context.Context, name string, value interface{}) {
khenaidoob9203542018-09-17 22:56:37 -0400909 if value == nil {
910 return
911 }
khenaidoo6e55d9e2019-12-12 18:26:26 -0500912
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400913 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
914 logger.Warnw(ctx, "request-aborted", log.Fields{"device-id": agent.deviceID, "name": name, "error": err})
915 return
916 }
917
918 cloned := agent.cloneDeviceWithoutLock()
khenaidoob9203542018-09-17 22:56:37 -0400919 updated := false
khenaidoo6e55d9e2019-12-12 18:26:26 -0500920 s := reflect.ValueOf(cloned).Elem()
khenaidoob9203542018-09-17 22:56:37 -0400921 if s.Kind() == reflect.Struct {
922 // exported field
923 f := s.FieldByName(name)
924 if f.IsValid() && f.CanSet() {
925 switch f.Kind() {
926 case reflect.String:
927 f.SetString(value.(string))
928 updated = true
929 case reflect.Uint32:
930 f.SetUint(uint64(value.(uint32)))
931 updated = true
932 case reflect.Bool:
933 f.SetBool(value.(bool))
934 updated = true
935 }
936 }
937 }
divyadesaicb8b59d2020-08-18 09:55:47 +0000938 logger.Debugw(ctx, "update-field-status", log.Fields{"device-id": cloned.Id, "name": name, "updated": updated})
khenaidoob9203542018-09-17 22:56:37 -0400939 // Save the data
khenaidoo6e55d9e2019-12-12 18:26:26 -0500940
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400941 if err := agent.updateDeviceAndReleaseLock(ctx, cloned); err != nil {
Rohan Agrawal31f21802020-06-12 05:38:46 +0000942 logger.Warnw(ctx, "attribute-update-failed", log.Fields{"attribute": name, "value": value})
khenaidoob9203542018-09-17 22:56:37 -0400943 }
khenaidoob9203542018-09-17 22:56:37 -0400944}
serkant.uluderya334479d2019-04-10 08:26:15 -0700945
Kent Hagerman45a13e42020-04-13 12:23:50 -0400946func (agent *Agent) simulateAlarm(ctx context.Context, simulateReq *voltha.SimulateAlarmRequest) error {
khenaidood948f772021-08-11 17:49:24 -0400947 var err error
948 var desc string
949 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
950 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
951
952 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
khenaidoo442e7c72020-03-10 16:13:48 -0400953 return err
954 }
955 defer agent.requestQueue.RequestComplete()
Himani Chawlab4c25912020-11-12 17:16:38 +0530956 logger.Debugw(ctx, "simulate-alarm", log.Fields{"device-id": agent.deviceID})
khenaidoo6e55d9e2019-12-12 18:26:26 -0500957
Kent Hagermancba2f302020-07-28 13:37:36 -0400958 device := agent.getDeviceReadOnlyWithoutLock()
khenaidoo6e55d9e2019-12-12 18:26:26 -0500959
khenaidood948f772021-08-11 17:49:24 -0400960 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
khenaidoo442e7c72020-03-10 16:13:48 -0400961 if err != nil {
khenaidood948f772021-08-11 17:49:24 -0400962 logger.Errorw(ctx, "grpc-client-nil",
963 log.Fields{
964 "error": err,
965 "device-id": agent.deviceID,
966 "device-type": agent.deviceType,
967 "adapter-endpoint": device.AdapterEndpoint,
968 })
npujar1d86a522019-11-14 17:11:16 +0530969 return err
serkant.uluderya334479d2019-04-10 08:26:15 -0700970 }
khenaidood948f772021-08-11 17:49:24 -0400971 subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
972 requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS
973 go func() {
974 defer cancel()
khenaidoo9beaaf12021-10-19 17:32:01 -0400975 _, err := client.SimulateAlarm(subCtx, &ca.SimulateAlarmMessage{Device: device, Request: simulateReq})
khenaidood948f772021-08-11 17:49:24 -0400976 if err == nil {
977 agent.onSuccess(subCtx, nil, nil, false)
978 } else {
979 agent.onFailure(subCtx, err, nil, nil, false)
980 }
981 }()
serkant.uluderya334479d2019-04-10 08:26:15 -0700982 return nil
983}
Mahir Gunyelb5851672019-07-24 10:46:26 +0300984
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400985// This function updates the device in the DB, releases the device lock, and runs any state transitions.
986// The calling function MUST hold the device lock. The caller MUST NOT modify the device after this is called.
987func (agent *Agent) updateDeviceAndReleaseLock(ctx context.Context, device *voltha.Device) error {
988 // fail early if this agent is no longer valid
Kent Hagerman4f355f52020-03-30 16:01:33 -0400989 if agent.stopped {
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400990 agent.requestQueue.RequestComplete()
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530991 return errors.New("device-agent-stopped")
Thomas Lee Se5a44012019-11-07 20:32:24 +0530992 }
Kent Hagerman4f355f52020-03-30 16:01:33 -0400993
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400994 // update in db
Kent Hagermanf5a67352020-04-30 15:15:26 -0400995 if err := agent.dbProxy.Set(ctx, agent.deviceID, device); err != nil {
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400996 agent.requestQueue.RequestComplete()
Kent Hagerman4f355f52020-03-30 16:01:33 -0400997 return status.Errorf(codes.Internal, "failed-update-device:%s: %s", agent.deviceID, err)
Mahir Gunyelb5851672019-07-24 10:46:26 +0300998 }
divyadesaicb8b59d2020-08-18 09:55:47 +0000999 logger.Debugw(ctx, "updated-device-in-store", log.Fields{"device-id: ": agent.deviceID})
Mahir Gunyelb5851672019-07-24 10:46:26 +03001000
Kent Hagerman6031aad2020-07-29 16:36:33 -04001001 prevDevice := agent.device
Kent Hagermanf6db9f12020-07-22 17:16:19 -04001002 // update the device
khenaidoo0db4c812020-05-27 15:27:30 -04001003 agent.device = device
Mahir Gunyelb0343bf2021-05-11 14:14:26 -07001004 //If any of the states has chenged, send the change event.
1005 if prevDevice.OperStatus != device.OperStatus || prevDevice.ConnectStatus != device.ConnectStatus || prevDevice.AdminState != device.AdminState {
1006 _ = agent.deviceMgr.Agent.SendDeviceStateChangeEvent(ctx, prevDevice.OperStatus, prevDevice.ConnectStatus, prevDevice.AdminState, device, time.Now().Unix())
1007 }
Maninder0aabf0c2021-03-17 14:55:14 +05301008 deviceTransientState := agent.getTransientState()
1009
Kent Hagermanf6db9f12020-07-22 17:16:19 -04001010 // release lock before processing transition
1011 agent.requestQueue.RequestComplete()
Himani Chawlab4c25912020-11-12 17:16:38 +05301012 subCtx := coreutils.WithSpanAndRPCMetadataFromContext(ctx)
Kent Hagermanf6db9f12020-07-22 17:16:19 -04001013
Himani Chawlab4c25912020-11-12 17:16:38 +05301014 if err := agent.deviceMgr.stateTransitions.ProcessTransition(subCtx,
Maninder0aabf0c2021-03-17 14:55:14 +05301015 device, prevDevice, deviceTransientState, deviceTransientState); err != nil {
Himani Chawlab4c25912020-11-12 17:16:38 +05301016 logger.Errorw(ctx, "failed-process-transition", log.Fields{"device-id": device.Id, "previous-admin-state": prevDevice.AdminState, "current-admin-state": device.AdminState})
1017 // Sending RPC EVENT here
1018 rpce := agent.deviceMgr.NewRPCEvent(ctx, agent.deviceID, err.Error(), nil)
Himani Chawla606a4f02021-03-23 19:45:58 +05301019 agent.deviceMgr.SendRPCEvent(ctx, "RPC_ERROR_RAISE_EVENT", rpce, voltha.EventCategory_COMMUNICATION,
1020 nil, time.Now().Unix())
Himani Chawlab4c25912020-11-12 17:16:38 +05301021
Kent Hagermanf6db9f12020-07-22 17:16:19 -04001022 }
Mahir Gunyelb5851672019-07-24 10:46:26 +03001023 return nil
1024}
Mahir Gunyelfdee9212019-10-16 16:52:21 -07001025
Himani Chawla2ba1c9c2020-10-07 13:19:03 +05301026// This function updates the device transient in the DB through loader, releases the device lock, and runs any state transitions.
1027// The calling function MUST hold the device lock. The caller MUST NOT modify the device after this is called.
1028func (agent *Agent) updateDeviceWithTransientStateAndReleaseLock(ctx context.Context, device *voltha.Device,
khenaidood948f772021-08-11 17:49:24 -04001029 transientState, prevTransientState core.DeviceTransientState_Types) error {
Himani Chawla2ba1c9c2020-10-07 13:19:03 +05301030 // fail early if this agent is no longer valid
1031 if agent.stopped {
1032 agent.requestQueue.RequestComplete()
1033 return errors.New("device-agent-stopped")
1034 }
1035 //update device TransientState
1036 if err := agent.updateTransientState(ctx, transientState); err != nil {
1037 agent.requestQueue.RequestComplete()
1038 return err
1039 }
1040 // update in db
1041 if err := agent.dbProxy.Set(ctx, agent.deviceID, device); err != nil {
1042 //Reverting TransientState update
khenaidood948f772021-08-11 17:49:24 -04001043 if errTransient := agent.updateTransientState(ctx, prevTransientState); errTransient != nil {
1044 logger.Errorw(ctx, "failed-to-revert-transient-state-update-on-error", log.Fields{"device-id": device.Id,
1045 "previous-transient-state": prevTransientState, "current-transient-state": transientState, "error": errTransient})
1046 }
Himani Chawla2ba1c9c2020-10-07 13:19:03 +05301047 agent.requestQueue.RequestComplete()
1048 return status.Errorf(codes.Internal, "failed-update-device:%s: %s", agent.deviceID, err)
1049 }
1050
1051 logger.Debugw(ctx, "updated-device-in-store", log.Fields{"device-id: ": agent.deviceID})
1052
1053 prevDevice := agent.device
1054 // update the device
1055 agent.device = device
Mahir Gunyelb0343bf2021-05-11 14:14:26 -07001056 //If any of the states has chenged, send the change event.
1057 if prevDevice.OperStatus != device.OperStatus || prevDevice.ConnectStatus != device.ConnectStatus || prevDevice.AdminState != device.AdminState {
1058 _ = agent.deviceMgr.Agent.SendDeviceStateChangeEvent(ctx, prevDevice.OperStatus, prevDevice.ConnectStatus, prevDevice.AdminState, device, time.Now().Unix())
1059 }
Himani Chawla2ba1c9c2020-10-07 13:19:03 +05301060
1061 // release lock before processing transition
1062 agent.requestQueue.RequestComplete()
nikesh.krishnan95142d52023-02-24 15:32:11 +05301063
1064 if err := agent.deviceMgr.stateTransitions.ProcessTransition(ctx,
1065 device, prevDevice, transientState, prevTransientState); err != nil {
1066 logger.Errorw(ctx, "failed-process-transition", log.Fields{"device-id": device.Id, "previous-admin-state": prevDevice.AdminState, "current-admin-state": device.AdminState})
1067 // Sending RPC EVENT here
1068 rpce := agent.deviceMgr.NewRPCEvent(ctx, agent.deviceID, err.Error(), nil)
1069 go agent.deviceMgr.SendRPCEvent(ctx, "RPC_ERROR_RAISE_EVENT", rpce, voltha.EventCategory_COMMUNICATION,
1070 nil, time.Now().Unix())
1071 }
1072
Himani Chawla2ba1c9c2020-10-07 13:19:03 +05301073 return nil
1074}
Kent Hagerman2b216042020-04-03 18:28:56 -04001075func (agent *Agent) updateDeviceReason(ctx context.Context, reason string) error {
khenaidood948f772021-08-11 17:49:24 -04001076 logger.Debugw(ctx, "update-device-reason", log.Fields{"device-id": agent.deviceID, "reason": reason})
1077
1078 var err error
1079 var desc string
1080 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
1081 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
1082
1083 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
khenaidoo442e7c72020-03-10 16:13:48 -04001084 return err
1085 }
Maninder9a1bc0d2020-10-26 11:34:02 +05301086
Kent Hagermanf6db9f12020-07-22 17:16:19 -04001087 cloned := agent.cloneDeviceWithoutLock()
npujar1d86a522019-11-14 17:11:16 +05301088 cloned.Reason = reason
khenaidood948f772021-08-11 17:49:24 -04001089 if err = agent.updateDeviceAndReleaseLock(ctx, cloned); err == nil {
1090 requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
Maninder9a1bc0d2020-10-26 11:34:02 +05301091 }
khenaidood948f772021-08-11 17:49:24 -04001092 return err
Mahir Gunyelfdee9212019-10-16 16:52:21 -07001093}
kesavandbc2d1622020-01-21 00:42:01 -05001094
Kent Hagerman2b216042020-04-03 18:28:56 -04001095func (agent *Agent) ChildDeviceLost(ctx context.Context, device *voltha.Device) error {
Himani Chawlab4c25912020-11-12 17:16:38 +05301096 logger.Debugw(ctx, "child-device-lost", log.Fields{"child-device-id": device.Id, "parent-device-id": agent.deviceID})
Chaitrashree G S543df3e2020-02-24 22:36:54 -05001097
khenaidood948f772021-08-11 17:49:24 -04001098 var err error
1099 var desc string
1100 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
1101 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
1102
Kent Hagerman2a07b862020-06-19 15:23:07 -04001103 // Remove the associated peer ports on the parent device
1104 for portID := range agent.portLoader.ListIDs() {
1105 if portHandle, have := agent.portLoader.Lock(portID); have {
1106 oldPort := portHandle.GetReadOnly()
1107 updatedPeers := make([]*voltha.Port_PeerPort, 0)
1108 for _, peerPort := range oldPort.Peers {
1109 if peerPort.DeviceId != device.Id {
1110 updatedPeers = append(updatedPeers, peerPort)
1111 }
khenaidoo442e7c72020-03-10 16:13:48 -04001112 }
Kent Hagerman2a07b862020-06-19 15:23:07 -04001113 newPort := *oldPort
1114 newPort.Peers = updatedPeers
1115 if err := portHandle.Update(ctx, &newPort); err != nil {
1116 portHandle.Unlock()
1117 return nil
1118 }
1119 portHandle.Unlock()
khenaidoo442e7c72020-03-10 16:13:48 -04001120 }
Chaitrashree G S543df3e2020-02-24 22:36:54 -05001121 }
1122
khenaidoo442e7c72020-03-10 16:13:48 -04001123 //send request to adapter
khenaidood948f772021-08-11 17:49:24 -04001124 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
khenaidoo442e7c72020-03-10 16:13:48 -04001125 if err != nil {
khenaidood948f772021-08-11 17:49:24 -04001126 logger.Errorw(ctx, "grpc-client-nil",
1127 log.Fields{
1128 "error": err,
1129 "device-id": agent.deviceID,
1130 "device-type": agent.deviceType,
1131 "adapter-endpoint": device.AdapterEndpoint,
1132 })
khenaidoo442e7c72020-03-10 16:13:48 -04001133 return err
Chaitrashree G S543df3e2020-02-24 22:36:54 -05001134 }
khenaidood948f772021-08-11 17:49:24 -04001135 subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
1136 requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS
nikesh.krishnan95142d52023-02-24 15:32:11 +05301137
1138 defer cancel()
1139 _, err = client.ChildDeviceLost(subCtx, device)
1140 if err == nil {
1141 agent.onSuccess(subCtx, nil, nil, true)
1142 } else {
1143 agent.onFailure(subCtx, err, nil, nil, true)
1144 }
1145
Chaitrashree G S543df3e2020-02-24 22:36:54 -05001146 return nil
Chaitrashree G S543df3e2020-02-24 22:36:54 -05001147}
onkarkundargi87285252020-01-27 11:34:52 +05301148
khenaidoo9beaaf12021-10-19 17:32:01 -04001149func (agent *Agent) startOmciTest(ctx context.Context, omcitestrequest *omci.OmciTestRequest) (*omci.TestResponse, error) {
khenaidood948f772021-08-11 17:49:24 -04001150 var err error
1151 var desc string
1152 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
1153 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
onkarkundargi87285252020-01-27 11:34:52 +05301154
khenaidood948f772021-08-11 17:49:24 -04001155 // OMCI test may be performed on a pre-provisioned device. If a device is in that state both its device type and endpoint
1156 // may not have been set yet.
1157 // First check if we need to update the type or endpoint
1158 cloned, err := agent.getDeviceReadOnly(ctx)
onkarkundargi87285252020-01-27 11:34:52 +05301159 if err != nil {
1160 return nil, err
1161 }
khenaidood948f772021-08-11 17:49:24 -04001162 if cloned.Type == "" || cloned.AdapterEndpoint == "" {
1163 if err = agent.updateDeviceTypeAndEndpoint(ctx); err != nil {
1164 return nil, err
1165 }
1166 cloned, err = agent.getDeviceReadOnly(ctx)
1167 if err != nil {
1168 return nil, err
1169 }
onkarkundargi87285252020-01-27 11:34:52 +05301170 }
1171
khenaidood948f772021-08-11 17:49:24 -04001172 // Send request to the adapter
1173 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
1174 if err != nil {
1175 logger.Errorw(ctx, "grpc-client-nil",
1176 log.Fields{
1177 "error": err,
1178 "device-id": agent.deviceID,
1179 "device-type": agent.deviceType,
1180 "adapter-endpoint": cloned.AdapterEndpoint,
1181 })
1182 return nil, err
onkarkundargi87285252020-01-27 11:34:52 +05301183 }
khenaidood948f772021-08-11 17:49:24 -04001184
khenaidoo9beaaf12021-10-19 17:32:01 -04001185 res, err := client.StartOmciTest(ctx, &ca.OMCITest{
khenaidood948f772021-08-11 17:49:24 -04001186 Device: cloned,
1187 Request: omcitestrequest,
1188 })
1189 if err == nil {
1190 requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
1191 }
1192 return res, err
onkarkundargi87285252020-01-27 11:34:52 +05301193}
Dinesh Belwalkarc1129f12020-02-27 10:41:33 -08001194
khenaidoo9beaaf12021-10-19 17:32:01 -04001195func (agent *Agent) getExtValue(ctx context.Context, pdevice *voltha.Device, cdevice *voltha.Device, valueparam *extension.ValueSpecifier) (*extension.ReturnValues, error) {
Himani Chawlab4c25912020-11-12 17:16:38 +05301196 logger.Debugw(ctx, "get-ext-value", log.Fields{"device-id": agent.deviceID, "onu-id": valueparam.Id, "value-type": valueparam.Value})
khenaidood948f772021-08-11 17:49:24 -04001197 var err error
1198 var desc string
1199 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
1200 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
1201
1202 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
Dinesh Belwalkarc1129f12020-02-27 10:41:33 -08001203 return nil, err
1204 }
1205
khenaidood948f772021-08-11 17:49:24 -04001206 //send request to adapter synchronously
1207 client, err := agent.adapterMgr.GetAdapterClient(ctx, pdevice.AdapterEndpoint)
Dinesh Belwalkarc1129f12020-02-27 10:41:33 -08001208 if err != nil {
khenaidood948f772021-08-11 17:49:24 -04001209 logger.Errorw(ctx, "grpc-client-nil",
1210 log.Fields{
1211 "error": err,
1212 "device-id": agent.deviceID,
1213 "device-type": agent.deviceType,
1214 "adapter-endpoint": pdevice.AdapterEndpoint,
1215 })
1216 agent.requestQueue.RequestComplete()
Dinesh Belwalkarc1129f12020-02-27 10:41:33 -08001217 return nil, err
1218 }
1219
khenaidood948f772021-08-11 17:49:24 -04001220 // Release lock before sending to adapter
1221 agent.requestQueue.RequestComplete()
Dinesh Belwalkarc1129f12020-02-27 10:41:33 -08001222
khenaidoo9beaaf12021-10-19 17:32:01 -04001223 retVal, err := client.GetExtValue(ctx, &ca.GetExtValueMessage{
khenaidood948f772021-08-11 17:49:24 -04001224 ParentDevice: pdevice,
1225 ChildDevice: cdevice,
1226 ValueType: valueparam.Value,
1227 })
1228 if err == nil {
1229 requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
Dinesh Belwalkarc1129f12020-02-27 10:41:33 -08001230 }
khenaidood948f772021-08-11 17:49:24 -04001231 return retVal, err
Dinesh Belwalkarc1129f12020-02-27 10:41:33 -08001232}
dpaul62686312020-06-23 14:17:36 +05301233
khenaidoo9beaaf12021-10-19 17:32:01 -04001234func (agent *Agent) setExtValue(ctx context.Context, device *voltha.Device, value *extension.ValueSet) (*empty.Empty, error) {
Himani Chawlab4c25912020-11-12 17:16:38 +05301235 logger.Debugw(ctx, "set-ext-value", log.Fields{"device-id": value.Id})
khenaidood948f772021-08-11 17:49:24 -04001236
1237 var err error
1238 var desc string
1239 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
1240 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
1241
1242 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
dpaul62686312020-06-23 14:17:36 +05301243 return nil, err
1244 }
1245
1246 //send request to adapter
khenaidood948f772021-08-11 17:49:24 -04001247 //send request to adapter synchronously
1248 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
dpaul62686312020-06-23 14:17:36 +05301249 if err != nil {
khenaidood948f772021-08-11 17:49:24 -04001250 logger.Errorw(ctx, "grpc-client-nil",
1251 log.Fields{
1252 "error": err,
1253 "device-id": agent.deviceID,
1254 "device-type": agent.deviceType,
1255 "adapter-endpoint": device.AdapterEndpoint,
1256 })
1257 agent.requestQueue.RequestComplete()
dpaul62686312020-06-23 14:17:36 +05301258 return nil, err
1259 }
khenaidood948f772021-08-11 17:49:24 -04001260 // Release lock before sending request to adapter
1261 agent.requestQueue.RequestComplete()
dpaul62686312020-06-23 14:17:36 +05301262
khenaidoo9beaaf12021-10-19 17:32:01 -04001263 retVal, err := client.SetExtValue(ctx, &ca.SetExtValueMessage{
khenaidood948f772021-08-11 17:49:24 -04001264 Device: device,
1265 Value: value,
1266 })
1267 if err == nil {
1268 requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
dpaul62686312020-06-23 14:17:36 +05301269 }
khenaidood948f772021-08-11 17:49:24 -04001270 return retVal, err
dpaul62686312020-06-23 14:17:36 +05301271}
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301272
1273func (agent *Agent) getSingleValue(ctx context.Context, request *extension.SingleGetValueRequest) (*extension.SingleGetValueResponse, error) {
Himani Chawlab4c25912020-11-12 17:16:38 +05301274 logger.Debugw(ctx, "get-single-value", log.Fields{"device-id": request.TargetId})
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301275
khenaidood948f772021-08-11 17:49:24 -04001276 var err error
1277 var desc string
1278 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
1279 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
1280
1281 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301282 return nil, err
1283 }
1284
1285 cloned := agent.cloneDeviceWithoutLock()
1286
1287 //send request to adapter
khenaidood948f772021-08-11 17:49:24 -04001288 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301289 if err != nil {
khenaidood948f772021-08-11 17:49:24 -04001290 logger.Errorw(ctx, "grpc-client-nil",
1291 log.Fields{
1292 "error": err,
1293 "device-id": cloned.Id,
1294 "adapter-endpoint": cloned.AdapterEndpoint,
1295 })
1296 agent.requestQueue.RequestComplete()
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301297 return nil, err
1298 }
khenaidood948f772021-08-11 17:49:24 -04001299 // Release lock before sending request to adapter
1300 agent.requestQueue.RequestComplete()
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301301
khenaidood948f772021-08-11 17:49:24 -04001302 resp, err := client.GetSingleValue(ctx, request)
1303 if err == nil {
1304 requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301305 }
khenaidood948f772021-08-11 17:49:24 -04001306 return resp, err
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301307}
1308
1309func (agent *Agent) setSingleValue(ctx context.Context, request *extension.SingleSetValueRequest) (*extension.SingleSetValueResponse, error) {
Himani Chawlab4c25912020-11-12 17:16:38 +05301310 logger.Debugw(ctx, "set-single-value", log.Fields{"device-id": request.TargetId})
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301311
khenaidood948f772021-08-11 17:49:24 -04001312 var err error
1313 var desc string
1314 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
1315 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
1316
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301317 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
1318 return nil, err
1319 }
1320
1321 cloned := agent.cloneDeviceWithoutLock()
1322
1323 //send request to adapter
khenaidood948f772021-08-11 17:49:24 -04001324 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301325 if err != nil {
khenaidood948f772021-08-11 17:49:24 -04001326 logger.Errorw(ctx, "grpc-client-nil",
1327 log.Fields{
1328 "error": err,
1329 "device-id": agent.deviceID,
1330 "device-type": agent.deviceType,
1331 "adapter-endpoint": cloned.AdapterEndpoint,
1332 })
1333 agent.requestQueue.RequestComplete()
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301334 return nil, err
1335 }
khenaidood948f772021-08-11 17:49:24 -04001336 // Release lock before sending request to adapter
1337 agent.requestQueue.RequestComplete()
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301338
khenaidood948f772021-08-11 17:49:24 -04001339 resp, err := client.SetSingleValue(ctx, request)
1340 if err == nil {
1341 requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301342 }
khenaidood948f772021-08-11 17:49:24 -04001343 return resp, err
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301344}
Maninder0aabf0c2021-03-17 14:55:14 +05301345
Maninder2195ccc2021-06-23 20:23:01 +05301346func (agent *Agent) proceedWithRequest(device *voltha.Device) bool {
1347 return !agent.isDeletionInProgress() && !agent.isInReconcileState(device)
Maninder0aabf0c2021-03-17 14:55:14 +05301348}
1349
1350func (agent *Agent) stopReconcile() {
1351 agent.stopReconcilingMutex.Lock()
1352 if agent.stopReconciling != nil {
1353 agent.stopReconciling <- 0
1354 }
1355 agent.stopReconcilingMutex.Unlock()
1356}
1357
khenaidood948f772021-08-11 17:49:24 -04001358// abortAllProcessing is invoked when an adapter managing this device is restarted
1359func (agent *Agent) abortAllProcessing(ctx context.Context) error {
1360 logger.Infow(ctx, "aborting-current-running-requests", log.Fields{"device-id": agent.deviceID})
1361 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
1362 return err
1363 }
khenaidoo25057da2021-12-08 14:40:45 -05001364 logger.Infow(ctx, "aborting-current-running-requests-after-wait", log.Fields{"device-id": agent.deviceID})
1365
khenaidood948f772021-08-11 17:49:24 -04001366 defer agent.requestQueue.RequestComplete()
1367
1368 // If any reconciling is in progress just abort it. The adapter is gone.
1369 agent.stopReconcile()
1370
khenaidoo25057da2021-12-08 14:40:45 -05001371 logger.Infow(ctx, "aborting-current-running-requests-after-sendstop", log.Fields{"device-id": agent.deviceID})
1372
khenaidood948f772021-08-11 17:49:24 -04001373 // Update the Core device transient state accordingly
1374 var updatedState core.DeviceTransientState_Types
1375 switch agent.getTransientState() {
1376 case core.DeviceTransientState_RECONCILE_IN_PROGRESS:
1377 updatedState = core.DeviceTransientState_NONE
1378 case core.DeviceTransientState_FORCE_DELETING:
1379 updatedState = core.DeviceTransientState_DELETE_FAILED
1380 case core.DeviceTransientState_DELETING_FROM_ADAPTER:
1381 updatedState = core.DeviceTransientState_DELETE_FAILED
khenaidoo68a5e0c2021-11-06 13:08:03 -04001382 case core.DeviceTransientState_DELETE_FAILED:
1383 // do not change state
1384 return nil
khenaidood948f772021-08-11 17:49:24 -04001385 default:
1386 updatedState = core.DeviceTransientState_NONE
1387 }
1388 if err := agent.updateTransientState(ctx, updatedState); err != nil {
1389 logger.Errorf(ctx, "transient-state-update-failed", log.Fields{"error": err})
1390 return err
1391 }
1392 return nil
1393}
1394
khenaidoo68a5e0c2021-11-06 13:08:03 -04001395func (agent *Agent) DeleteDevicePostAdapterRestart(ctx context.Context) error {
1396 logger.Debugw(ctx, "delete-post-restart", log.Fields{"device-id": agent.deviceID})
1397 ctx = utils.WithNewSpanAndRPCMetadataContext(ctx, "DelteDevicePostAdapterRestart")
1398
1399 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
1400 return err
1401 }
1402
1403 device := agent.getDeviceReadOnlyWithoutLock()
1404 if device.AdminState == voltha.AdminState_PREPROVISIONED {
1405 logger.Debugw(ctx, "device-in-preprovisioning-state-reconcile-not-needed", log.Fields{"device-id": device.Id})
1406 agent.requestQueue.RequestComplete()
1407 return nil
1408 }
1409 // Change device transient state to FORCE_DELETING
1410 if err := agent.updateTransientState(ctx, core.DeviceTransientState_FORCE_DELETING); err != nil {
1411 logger.Errorw(ctx, "failure-updating-transient-state", log.Fields{"error": err, "device-id": agent.deviceID})
1412 agent.requestQueue.RequestComplete()
1413 return err
1414 }
1415
1416 // Ensure we have a valid grpc client available as we have just restarted
1417 deleteBackoff := backoff.NewExponentialBackOff()
1418 deleteBackoff.InitialInterval = agent.config.BackoffRetryInitialInterval
1419 deleteBackoff.MaxElapsedTime = agent.config.BackoffRetryMaxElapsedTime
1420 deleteBackoff.MaxInterval = agent.config.BackoffRetryMaxInterval
1421 var backoffTimer *time.Timer
1422 var err error
1423 var client adapter_service.AdapterServiceClient
1424retry:
1425 for {
1426 client, err = agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
1427 if err == nil {
1428 break retry
1429 }
1430 duration := deleteBackoff.NextBackOff()
1431 if duration == backoff.Stop {
1432 deleteBackoff.Reset()
1433 duration = deleteBackoff.NextBackOff()
1434 }
1435 backoffTimer = time.NewTimer(duration)
1436 select {
1437 case <-backoffTimer.C:
1438 logger.Debugw(ctx, "backoff-timer-expires", log.Fields{"device-id": agent.deviceID})
1439 case <-ctx.Done():
1440 err = ctx.Err()
1441 break retry
1442 }
1443 }
1444 if backoffTimer != nil && !backoffTimer.Stop() {
1445 select {
1446 case <-backoffTimer.C:
1447 default:
1448 }
1449 }
1450 if err != nil || client == nil {
1451 agent.requestQueue.RequestComplete()
1452 return err
1453 }
1454
1455 // Release the device lock to allow for device state update, if any
1456 agent.requestQueue.RequestComplete()
1457
1458 // Send the delete request to the adapter
1459 subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
1460 defer cancel()
khenaidoo6cd8e862021-11-12 16:33:43 -05001461 _, err = client.DeleteDevice(subCtx, device)
1462 agent.onForceDeleteResponse(subCtx, nil, nil, err)
1463 return err
khenaidoo68a5e0c2021-11-06 13:08:03 -04001464}
1465
khenaidood948f772021-08-11 17:49:24 -04001466func (agent *Agent) ReconcileDevice(ctx context.Context) {
khenaidoo68a5e0c2021-11-06 13:08:03 -04001467 // Do not reconcile if the device was in DELETE_FAILED transient state. Just invoke the force delete on that device.
1468 state := agent.getTransientState()
1469 logger.Debugw(ctx, "starting-reconcile", log.Fields{"device-id": agent.deviceID, "state": state})
1470 if agent.getTransientState() == core.DeviceTransientState_DELETE_FAILED {
1471 if err := agent.DeleteDevicePostAdapterRestart(ctx); err != nil {
1472 logger.Errorw(ctx, "delete-post-restart-failed", log.Fields{"error": err, "device-id": agent.deviceID})
1473 }
1474 return
1475 }
1476
khenaidood948f772021-08-11 17:49:24 -04001477 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
Maninder0aabf0c2021-03-17 14:55:14 +05301478 var desc string
Maninder0aabf0c2021-03-17 14:55:14 +05301479
1480 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
khenaidood948f772021-08-11 17:49:24 -04001481 agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc)
1482 return
1483 }
1484
1485 device := agent.getDeviceReadOnlyWithoutLock()
1486 if device.AdminState == voltha.AdminState_PREPROVISIONED {
1487 agent.requestQueue.RequestComplete()
1488 logger.Debugw(ctx, "device-in-preprovisioning-state-reconcile-not-needed", log.Fields{"device-id": device.Id})
Maninder0aabf0c2021-03-17 14:55:14 +05301489 return
1490 }
1491
Maninder2195ccc2021-06-23 20:23:01 +05301492 if !agent.proceedWithRequest(device) {
Maninder0aabf0c2021-03-17 14:55:14 +05301493 agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -04001494 err := fmt.Errorf("cannot complete operation as device deletion/reconciling is in progress or reconcile failed for device : %s", device.Id)
1495 logger.Errorw(ctx, "reconcile-failed", log.Fields{"error": err})
1496 agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc)
Maninder0aabf0c2021-03-17 14:55:14 +05301497 return
1498 }
1499
1500 //set transient state to RECONCILE IN PROGRESS
khenaidood948f772021-08-11 17:49:24 -04001501 err := agent.updateTransientState(ctx, core.DeviceTransientState_RECONCILE_IN_PROGRESS)
Maninder0aabf0c2021-03-17 14:55:14 +05301502 if err != nil {
1503 agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -04001504 logger.Errorw(ctx, "setting-transient-state-failed", log.Fields{"error": err})
1505 agent.logDeviceUpdate(ctx, nil, nil, requestStatus, nil, desc)
Maninder0aabf0c2021-03-17 14:55:14 +05301506 return
1507 }
1508
Maninder0aabf0c2021-03-17 14:55:14 +05301509 reconcilingBackoff := backoff.NewExponentialBackOff()
1510 reconcilingBackoff.InitialInterval = agent.config.BackoffRetryInitialInterval
1511 reconcilingBackoff.MaxElapsedTime = agent.config.BackoffRetryMaxElapsedTime
1512 reconcilingBackoff.MaxInterval = agent.config.BackoffRetryMaxInterval
1513
1514 //making here to keep lifecycle of this channel within the scope of retryReconcile
1515 agent.stopReconcilingMutex.Lock()
khenaidoo25057da2021-12-08 14:40:45 -05001516 agent.stopReconciling = make(chan int, 1)
Maninder0aabf0c2021-03-17 14:55:14 +05301517 agent.stopReconcilingMutex.Unlock()
1518
David K. Bainbridge482e4422021-06-30 12:23:42 -07001519 // defined outside the retry loop so it can be cleaned
1520 // up when the loop breaks
1521 var backoffTimer *time.Timer
1522
1523retry:
Maninder0aabf0c2021-03-17 14:55:14 +05301524 for {
David K. Bainbridge482e4422021-06-30 12:23:42 -07001525 // If the operations state of the device is RECONCILING_FAILED then we do not
1526 // want to continue to attempt reconciliation.
1527 deviceRef := agent.getDeviceReadOnlyWithoutLock()
1528 if deviceRef.OperStatus == common.OperStatus_RECONCILING_FAILED {
1529 logger.Warnw(ctx, "reconciling-failed-halting-retries",
1530 log.Fields{"device-id": device.Id})
1531 agent.requestQueue.RequestComplete()
1532 break retry
1533 }
1534
Maninder0aabf0c2021-03-17 14:55:14 +05301535 // Use an exponential back off to prevent getting into a tight loop
1536 duration := reconcilingBackoff.NextBackOff()
1537 //This case should never occur in default case as max elapsed time for backoff is 0(by default) , so it will never return stop
1538 if duration == backoff.Stop {
1539 // If we reach a maximum then warn and reset the backoff
1540 // timer and keep attempting.
1541 logger.Warnw(ctx, "maximum-reconciling-backoff-reached--resetting-backoff-timer",
1542 log.Fields{"max-reconciling-backoff": reconcilingBackoff.MaxElapsedTime,
1543 "device-id": device.Id})
1544 reconcilingBackoff.Reset()
1545 duration = reconcilingBackoff.NextBackOff()
1546 }
1547
David K. Bainbridge482e4422021-06-30 12:23:42 -07001548 backoffTimer = time.NewTimer(duration)
Maninder0aabf0c2021-03-17 14:55:14 +05301549
khenaidood948f772021-08-11 17:49:24 -04001550 logger.Debugw(ctx, "retrying-reconciling", log.Fields{"deviceID": device.Id, "endpoint": device.AdapterEndpoint})
1551 // Release lock before sending request to adapter
Maninder0aabf0c2021-03-17 14:55:14 +05301552 agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -04001553
1554 // Send a reconcile request to the adapter.
1555 err := agent.sendReconcileRequestToAdapter(ctx, device)
khenaidoo6cd8e862021-11-12 16:33:43 -05001556
1557 // Check the transient state after a response from the adapter. If a device delete
1558 // request was issued due to a callback during that time and failed then just delete
1559 // the device and stop the reconcile loop and invoke the device deletion
1560 if agent.getTransientState() == core.DeviceTransientState_DELETE_FAILED {
1561 if dErr := agent.DeleteDevicePostAdapterRestart(ctx); dErr != nil {
1562 logger.Errorw(ctx, "delete-post-restart-failed", log.Fields{"error": dErr, "device-id": agent.deviceID})
1563 }
1564 break retry
1565 }
1566
khenaidood948f772021-08-11 17:49:24 -04001567 if errors.Is(err, errContextExpired) || errors.Is(err, errReconcileAborted) {
1568 logger.Errorw(ctx, "reconcile-aborted", log.Fields{"error": err})
1569 requestStatus = &common.OperationResp{Code: common.OperationResp_OperationReturnCode(common.OperStatus_FAILED)}
1570 desc = "aborted"
1571 agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc)
1572 break retry
1573 }
Maninder0aabf0c2021-03-17 14:55:14 +05301574 if err != nil {
khenaidood948f772021-08-11 17:49:24 -04001575 agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc)
Maninder0aabf0c2021-03-17 14:55:14 +05301576 <-backoffTimer.C
1577 // backoffTimer expired continue
1578 // Take lock back before retrying
1579 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
khenaidood948f772021-08-11 17:49:24 -04001580 agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc)
David K. Bainbridge482e4422021-06-30 12:23:42 -07001581 break retry
Maninder0aabf0c2021-03-17 14:55:14 +05301582 }
1583 continue
1584 }
khenaidood948f772021-08-11 17:49:24 -04001585 // Success
1586 requestStatus = &common.OperationResp{Code: common.OperationResp_OPERATION_SUCCESS}
1587 desc = "adapter-response"
1588 agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc)
1589 break retry
David K. Bainbridge482e4422021-06-30 12:23:42 -07001590 }
1591
khenaidoo25057da2021-12-08 14:40:45 -05001592 logger.Debugw(ctx, "reconcile-retry-ends", log.Fields{"adapter-endpoint": agent.adapterEndpoint})
1593
David K. Bainbridge482e4422021-06-30 12:23:42 -07001594 // Retry loop is broken, so stop any timers and drain the channel
1595 if backoffTimer != nil && !backoffTimer.Stop() {
1596
1597 // As per documentation and stack overflow when a timer is stopped its
1598 // channel should be drained. The issue is that Stop returns false
1599 // either if the timer has already been fired "OR" if the timer can be
1600 // stopped before being fired. This means that in some cases the
1601 // channel has already be emptied so attempting to read from it means
1602 // a blocked thread. To get around this use a select so if the
1603 // channel is already empty the default case hits and we are not
1604 // blocked.
1605 select {
1606 case <-backoffTimer.C:
1607 default:
Maninder0aabf0c2021-03-17 14:55:14 +05301608 }
1609 }
1610}
1611
khenaidood948f772021-08-11 17:49:24 -04001612func (agent *Agent) sendReconcileRequestToAdapter(ctx context.Context, device *voltha.Device) error {
1613 logger.Debugw(ctx, "sending-reconcile-to-adapter", log.Fields{"device-id": device.Id, "endpoint": agent.adapterEndpoint})
1614 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
1615 if err != nil {
1616 return err
1617 }
1618 adapterResponse := make(chan error)
1619 go func() {
1620 _, err := client.ReconcileDevice(ctx, device)
1621 adapterResponse <- err
1622 }()
Maninder0aabf0c2021-03-17 14:55:14 +05301623 select {
1624 // wait for response
khenaidood948f772021-08-11 17:49:24 -04001625 case err := <-adapterResponse:
1626 if err != nil {
1627 return err
Maninder0aabf0c2021-03-17 14:55:14 +05301628 }
Maninder0aabf0c2021-03-17 14:55:14 +05301629 //In case of success quit retrying and wait for adapter to reset operation state of device
1630 agent.stopReconcilingMutex.Lock()
1631 agent.stopReconciling = nil
1632 agent.stopReconcilingMutex.Unlock()
1633 return nil
1634
1635 //if reconciling need to be stopped
1636 case _, ok := <-agent.stopReconciling:
1637 agent.stopReconcilingMutex.Lock()
1638 agent.stopReconciling = nil
1639 agent.stopReconcilingMutex.Unlock()
1640 if !ok {
1641 //channel-closed
khenaidood948f772021-08-11 17:49:24 -04001642 return fmt.Errorf("reconcile channel closed:%w", errReconcileAborted)
Maninder0aabf0c2021-03-17 14:55:14 +05301643 }
khenaidood948f772021-08-11 17:49:24 -04001644 return fmt.Errorf("reconciling aborted:%w", errReconcileAborted)
1645 // Context expired
1646 case <-ctx.Done():
1647 return fmt.Errorf("context expired:%s :%w", ctx.Err(), errContextExpired)
Maninder0aabf0c2021-03-17 14:55:14 +05301648 }
Maninder0aabf0c2021-03-17 14:55:14 +05301649}
1650
1651func (agent *Agent) reconcilingCleanup(ctx context.Context) error {
1652 var desc string
khenaidood948f772021-08-11 17:49:24 -04001653 var err error
Maninder0aabf0c2021-03-17 14:55:14 +05301654 operStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
khenaidood948f772021-08-11 17:49:24 -04001655 defer func() { agent.logDeviceUpdate(ctx, nil, nil, operStatus, err, desc) }()
1656
1657 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
1658 desc = "reconcile-cleanup-failed"
Maninder0aabf0c2021-03-17 14:55:14 +05301659 return err
1660 }
1661 defer agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -04001662 err = agent.updateTransientState(ctx, core.DeviceTransientState_NONE)
Maninder0aabf0c2021-03-17 14:55:14 +05301663 if err != nil {
khenaidood948f772021-08-11 17:49:24 -04001664 logger.Errorf(ctx, "transient-state-update-failed", log.Fields{"error": err})
Maninder0aabf0c2021-03-17 14:55:14 +05301665 return err
1666 }
khenaidood948f772021-08-11 17:49:24 -04001667 operStatus.Code = common.OperationResp_OPERATION_SUCCESS
Maninder0aabf0c2021-03-17 14:55:14 +05301668 return nil
1669}
khenaidood948f772021-08-11 17:49:24 -04001670
1671func (agent *Agent) isAdapterConnectionUp(ctx context.Context) bool {
1672 c, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
1673 return c != nil && err == nil
1674}
1675
1676func (agent *Agent) canDeviceRequestProceed(ctx context.Context) error {
1677 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
1678 return err
1679 }
1680 defer agent.requestQueue.RequestComplete()
1681 if agent.proceedWithRequest(agent.device) {
1682 return nil
1683 }
1684 return fmt.Errorf("device-cannot-process-request-%s", agent.deviceID)
1685}