blob: 89228e12d9003dc7437b5e9e4381420040963ee1 [file] [log] [blame]
khenaidoob9203542018-09-17 22:56:37 -04001/*
Joey Armstrong5f51f2e2023-01-17 17:06:26 -05002 * Copyright 2018-2023 Open Networking Foundation (ONF) and the ONF Contributors
khenaidoob9203542018-09-17 22:56:37 -04003
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7
8 * http://www.apache.org/licenses/LICENSE-2.0
9
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
npujar1d86a522019-11-14 17:11:16 +053016
Kent Hagerman2b216042020-04-03 18:28:56 -040017package device
khenaidoob9203542018-09-17 22:56:37 -040018
19import (
20 "context"
Matteo Scandolo360605d2019-11-05 18:29:17 -080021 "encoding/hex"
Kent Hagerman4f355f52020-03-30 16:01:33 -040022 "errors"
khenaidoo3ab34882019-05-02 21:33:30 -040023 "fmt"
Mahir Gunyelb0343bf2021-05-11 14:14:26 -070024 "reflect"
25 "sync"
26 "time"
27
khenaidoo9beaaf12021-10-19 17:32:01 -040028 "github.com/opencord/voltha-protos/v5/go/adapter_service"
khenaidood948f772021-08-11 17:49:24 -040029 "github.com/opencord/voltha-protos/v5/go/core"
khenaidoo9beaaf12021-10-19 17:32:01 -040030 "github.com/opencord/voltha-protos/v5/go/omci"
khenaidood948f772021-08-11 17:49:24 -040031
Maninder0aabf0c2021-03-17 14:55:14 +053032 "github.com/cenkalti/backoff/v3"
Maninder9a1bc0d2020-10-26 11:34:02 +053033 "github.com/gogo/protobuf/proto"
Maninder9a1bc0d2020-10-26 11:34:02 +053034 "github.com/golang/protobuf/ptypes/empty"
Maninder0aabf0c2021-03-17 14:55:14 +053035 "github.com/opencord/voltha-go/rw_core/config"
khenaidoo68a5e0c2021-11-06 13:08:03 -040036 "github.com/opencord/voltha-go/rw_core/utils"
Maninder9a1bc0d2020-10-26 11:34:02 +053037 "google.golang.org/grpc/codes"
38 "google.golang.org/grpc/status"
Mahir Gunyeladdb66a2020-04-29 18:08:50 -070039
Himani Chawla2ba1c9c2020-10-07 13:19:03 +053040 "github.com/opencord/voltha-go/db/model"
Kent Hagerman2b216042020-04-03 18:28:56 -040041 "github.com/opencord/voltha-go/rw_core/core/adapter"
Mahir Gunyel03de0d32020-06-03 01:36:59 -070042 "github.com/opencord/voltha-go/rw_core/core/device/flow"
43 "github.com/opencord/voltha-go/rw_core/core/device/group"
Kent Hagerman2a07b862020-06-19 15:23:07 -040044 "github.com/opencord/voltha-go/rw_core/core/device/port"
Himani Chawla2ba1c9c2020-10-07 13:19:03 +053045 "github.com/opencord/voltha-go/rw_core/core/device/transientstate"
Scott Bakerb671a862019-10-24 10:53:40 -070046 coreutils "github.com/opencord/voltha-go/rw_core/utils"
khenaidood948f772021-08-11 17:49:24 -040047 "github.com/opencord/voltha-lib-go/v7/pkg/log"
48 "github.com/opencord/voltha-protos/v5/go/common"
khenaidoo9beaaf12021-10-19 17:32:01 -040049 ca "github.com/opencord/voltha-protos/v5/go/core_adapter"
khenaidood948f772021-08-11 17:49:24 -040050 "github.com/opencord/voltha-protos/v5/go/extension"
khenaidood948f772021-08-11 17:49:24 -040051 ofp "github.com/opencord/voltha-protos/v5/go/openflow_13"
52 "github.com/opencord/voltha-protos/v5/go/voltha"
khenaidoob9203542018-09-17 22:56:37 -040053)
54
khenaidood948f772021-08-11 17:49:24 -040055var errReconcileAborted = errors.New("reconcile aborted")
56var errContextExpired = errors.New("context expired")
khenaidoo68a5e0c2021-11-06 13:08:03 -040057var errNoConnection = errors.New("no connection")
khenaidood948f772021-08-11 17:49:24 -040058
Kent Hagerman2b216042020-04-03 18:28:56 -040059// Agent represents device agent attributes
60type Agent struct {
Maninder0aabf0c2021-03-17 14:55:14 +053061 deviceID string
62 parentID string
63 deviceType string
khenaidood948f772021-08-11 17:49:24 -040064 adapterEndpoint string
Maninder0aabf0c2021-03-17 14:55:14 +053065 isRootDevice bool
Maninder0aabf0c2021-03-17 14:55:14 +053066 adapterMgr *adapter.Manager
67 deviceMgr *Manager
68 dbProxy *model.Proxy
69 exitChannel chan int
70 device *voltha.Device
71 requestQueue *coreutils.RequestQueue
khenaidood948f772021-08-11 17:49:24 -040072 internalTimeout time.Duration
73 rpcTimeout time.Duration
Himani Chawla4b4bd252021-11-08 15:59:40 +053074 flowTimeout time.Duration
Maninder0aabf0c2021-03-17 14:55:14 +053075 startOnce sync.Once
76 stopOnce sync.Once
77 stopped bool
78 stopReconciling chan int
79 stopReconcilingMutex sync.RWMutex
80 config *config.RWCoreFlags
Mahir Gunyel03de0d32020-06-03 01:36:59 -070081
khenaidoo7585a962021-06-10 16:15:38 -040082 flowCache *flow.Cache
83 groupCache *group.Cache
Himani Chawla2ba1c9c2020-10-07 13:19:03 +053084 portLoader *port.Loader
85 transientStateLoader *transientstate.Loader
khenaidoob9203542018-09-17 22:56:37 -040086}
87
Joey Armstrong393daca2023-07-06 08:47:54 -040088// newAgent creates a new device agent. The device will be initialized when start() is called.
Himani Chawla4b4bd252021-11-08 15:59:40 +053089func newAgent(device *voltha.Device, deviceMgr *Manager, dbPath *model.Path, deviceProxy *model.Proxy, internalTimeout, rpcTimeout, flowTimeout time.Duration) *Agent {
Kent Hagerman2a07b862020-06-19 15:23:07 -040090 deviceID := device.Id
91 if deviceID == "" {
92 deviceID = coreutils.CreateDeviceID()
Stephane Barbarie1ab43272018-12-08 21:42:13 -050093 }
Scott Baker80678602019-11-14 16:57:36 -080094
Kent Hagerman2a07b862020-06-19 15:23:07 -040095 return &Agent{
Himani Chawla2ba1c9c2020-10-07 13:19:03 +053096 deviceID: deviceID,
Himani Chawla2ba1c9c2020-10-07 13:19:03 +053097 isRootDevice: device.Root,
98 parentID: device.ParentId,
99 deviceType: device.Type,
khenaidood948f772021-08-11 17:49:24 -0400100 adapterEndpoint: device.AdapterEndpoint,
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530101 deviceMgr: deviceMgr,
102 adapterMgr: deviceMgr.adapterMgr,
103 exitChannel: make(chan int, 1),
104 dbProxy: deviceProxy,
khenaidood948f772021-08-11 17:49:24 -0400105 internalTimeout: internalTimeout,
106 rpcTimeout: rpcTimeout,
Himani Chawla4b4bd252021-11-08 15:59:40 +0530107 flowTimeout: flowTimeout,
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530108 device: proto.Clone(device).(*voltha.Device),
109 requestQueue: coreutils.NewRequestQueue(),
Maninder0aabf0c2021-03-17 14:55:14 +0530110 config: deviceMgr.config,
khenaidoo7585a962021-06-10 16:15:38 -0400111 flowCache: flow.NewCache(),
112 groupCache: group.NewCache(),
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530113 portLoader: port.NewLoader(dbPath.SubPath("ports").Proxy(deviceID)),
114 transientStateLoader: transientstate.NewLoader(dbPath.SubPath("core").Proxy("transientstate"), deviceID),
Kent Hagerman2a07b862020-06-19 15:23:07 -0400115 }
khenaidoob9203542018-09-17 22:56:37 -0400116}
117
khenaidoo442e7c72020-03-10 16:13:48 -0400118// start() saves the device to the data model and registers for callbacks on that device if deviceToCreate!=nil.
119// Otherwise, it will load the data from the dB and setup the necessary callbacks and proxies. Returns the device that
Scott Baker80678602019-11-14 16:57:36 -0800120// was started.
khenaidoo7585a962021-06-10 16:15:38 -0400121func (agent *Agent) start(ctx context.Context, deviceExist bool, deviceToCreate *voltha.Device) (*voltha.Device, error) {
khenaidoo442e7c72020-03-10 16:13:48 -0400122 needToStart := false
123 if agent.startOnce.Do(func() { needToStart = true }); !needToStart {
Kent Hagermancba2f302020-07-28 13:37:36 -0400124 return agent.getDeviceReadOnly(ctx)
khenaidoo442e7c72020-03-10 16:13:48 -0400125 }
126 var startSucceeded bool
127 defer func() {
128 if !startSucceeded {
129 if err := agent.stop(ctx); err != nil {
Rohan Agrawal31f21802020-06-12 05:38:46 +0000130 logger.Errorw(ctx, "failed-to-cleanup-after-unsuccessful-start", log.Fields{"device-id": agent.deviceID, "error": err})
khenaidoo442e7c72020-03-10 16:13:48 -0400131 }
132 }
133 }()
khenaidoo7585a962021-06-10 16:15:38 -0400134 if deviceExist {
135 device := deviceToCreate
136 if device == nil {
137 // Load from dB
138 device = &voltha.Device{}
139 have, err := agent.dbProxy.Get(ctx, agent.deviceID, device)
140 if err != nil {
141 return nil, err
142 } else if !have {
143 return nil, status.Errorf(codes.NotFound, "device-%s", agent.deviceID)
144 }
khenaidood948f772021-08-11 17:49:24 -0400145 logger.Infow(ctx, "device-loaded-from-db", log.Fields{"device-id": agent.deviceID, "adapter-endpoint": device.AdapterEndpoint, "type": device.Type})
Thomas Lee Se5a44012019-11-07 20:32:24 +0530146 }
khenaidood948f772021-08-11 17:49:24 -0400147 agent.deviceType = device.Type
148 agent.adapterEndpoint = device.AdapterEndpoint
Kent Hagerman4f355f52020-03-30 16:01:33 -0400149 agent.device = proto.Clone(device).(*voltha.Device)
khenaidoo7585a962021-06-10 16:15:38 -0400150 // load the ports from KV to cache
Kent Hagerman2a07b862020-06-19 15:23:07 -0400151 agent.portLoader.Load(ctx)
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530152 agent.transientStateLoader.Load(ctx)
khenaidoo297cd252019-02-07 22:10:23 -0500153 } else {
Scott Baker80678602019-11-14 16:57:36 -0800154 // Create a new device
Maninder9a1bc0d2020-10-26 11:34:02 +0530155 var desc string
khenaidood948f772021-08-11 17:49:24 -0400156 var err error
Maninder9a1bc0d2020-10-26 11:34:02 +0530157 prevState := common.AdminState_UNKNOWN
158 currState := common.AdminState_UNKNOWN
khenaidood948f772021-08-11 17:49:24 -0400159 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
Maninder9a1bc0d2020-10-26 11:34:02 +0530160
khenaidood948f772021-08-11 17:49:24 -0400161 defer func() { agent.logDeviceUpdate(ctx, &prevState, &currState, requestStatus, err, desc) }()
Maninder9a1bc0d2020-10-26 11:34:02 +0530162
Kent Hagermanf5a67352020-04-30 15:15:26 -0400163 // Assumption is that AdminState, FlowGroups, and Flows are uninitialized since this
164 // is a new device, so populate them here before passing the device to ldProxy.Set.
Kent Hagerman2b216042020-04-03 18:28:56 -0400165 // agent.deviceId will also have been set during newAgent().
khenaidoo7585a962021-06-10 16:15:38 -0400166 device := (proto.Clone(deviceToCreate)).(*voltha.Device)
npujar1d86a522019-11-14 17:11:16 +0530167 device.Id = agent.deviceID
Scott Baker80678602019-11-14 16:57:36 -0800168 device.AdminState = voltha.AdminState_PREPROVISIONED
Maninder9a1bc0d2020-10-26 11:34:02 +0530169 currState = device.AdminState
Scott Baker80678602019-11-14 16:57:36 -0800170 if !deviceToCreate.GetRoot() && deviceToCreate.ProxyAddress != nil {
171 // Set the default vlan ID to the one specified by the parent adapter. It can be
172 // overwritten by the child adapter during a device update request
173 device.Vlan = deviceToCreate.ProxyAddress.ChannelId
174 }
175
khenaidood948f772021-08-11 17:49:24 -0400176 // Save the device to the model
177 if err = agent.dbProxy.Set(ctx, agent.deviceID, device); err != nil {
178 err = status.Errorf(codes.Aborted, "failed-adding-device-%s: %s", agent.deviceID, err)
179 return nil, err
khenaidoo297cd252019-02-07 22:10:23 -0500180 }
Mahir Gunyelb0343bf2021-05-11 14:14:26 -0700181 _ = agent.deviceMgr.Agent.SendDeviceStateChangeEvent(ctx, device.OperStatus, device.ConnectStatus, prevState, device, time.Now().Unix())
khenaidood948f772021-08-11 17:49:24 -0400182 requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
khenaidoo442e7c72020-03-10 16:13:48 -0400183 agent.device = device
khenaidoob9203542018-09-17 22:56:37 -0400184 }
khenaidoo442e7c72020-03-10 16:13:48 -0400185 startSucceeded = true
Rohan Agrawalcf12f202020-08-03 04:42:01 +0000186 log.EnrichSpan(ctx, log.Fields{"device-id": agent.deviceID})
Rohan Agrawal31f21802020-06-12 05:38:46 +0000187 logger.Debugw(ctx, "device-agent-started", log.Fields{"device-id": agent.deviceID})
khenaidoo442e7c72020-03-10 16:13:48 -0400188
Kent Hagermancba2f302020-07-28 13:37:36 -0400189 return agent.getDeviceReadOnly(ctx)
khenaidoob9203542018-09-17 22:56:37 -0400190}
191
khenaidoo4d4802d2018-10-04 21:59:49 -0400192// stop stops the device agent. Not much to do for now
Kent Hagerman2b216042020-04-03 18:28:56 -0400193func (agent *Agent) stop(ctx context.Context) error {
khenaidoo442e7c72020-03-10 16:13:48 -0400194 needToStop := false
195 if agent.stopOnce.Do(func() { needToStop = true }); !needToStop {
196 return nil
197 }
198 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
199 return err
200 }
201 defer agent.requestQueue.RequestComplete()
khenaidoo49085352020-01-13 19:15:43 -0500202
Himani Chawlab4c25912020-11-12 17:16:38 +0530203 logger.Infow(ctx, "stopping-device-agent", log.Fields{"device-id": agent.deviceID, "parent-id": agent.parentID})
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530204 // Remove the device transient loader
205 if err := agent.deleteTransientState(ctx); err != nil {
206 return err
207 }
khenaidoo0a822f92019-05-08 15:15:57 -0400208 // Remove the device from the KV store
Kent Hagermanf5a67352020-04-30 15:15:26 -0400209 if err := agent.dbProxy.Remove(ctx, agent.deviceID); err != nil {
khenaidoo442e7c72020-03-10 16:13:48 -0400210 return err
Thomas Lee Se5a44012019-11-07 20:32:24 +0530211 }
Abhilash Laxmeshwar75517ea2021-11-24 18:38:10 +0530212 //send the device event to the message bus
213 _ = agent.deviceMgr.Agent.SendDeviceDeletedEvent(ctx, agent.device, time.Now().Unix())
khenaidoo442e7c72020-03-10 16:13:48 -0400214
khenaidoo442e7c72020-03-10 16:13:48 -0400215 close(agent.exitChannel)
216
217 agent.stopped = true
218
Rohan Agrawal31f21802020-06-12 05:38:46 +0000219 logger.Infow(ctx, "device-agent-stopped", log.Fields{"device-id": agent.deviceID, "parent-id": agent.parentID})
khenaidoo442e7c72020-03-10 16:13:48 -0400220
221 return nil
khenaidoob9203542018-09-17 22:56:37 -0400222}
223
Scott Baker80678602019-11-14 16:57:36 -0800224// Load the most recent state from the KVStore for the device.
Kent Hagerman2b216042020-04-03 18:28:56 -0400225func (agent *Agent) reconcileWithKVStore(ctx context.Context) {
khenaidoo442e7c72020-03-10 16:13:48 -0400226 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
Rohan Agrawal31f21802020-06-12 05:38:46 +0000227 logger.Warnw(ctx, "request-aborted", log.Fields{"device-id": agent.deviceID, "error": err})
khenaidoo442e7c72020-03-10 16:13:48 -0400228 return
229 }
230 defer agent.requestQueue.RequestComplete()
Rohan Agrawal31f21802020-06-12 05:38:46 +0000231 logger.Debug(ctx, "reconciling-device-agent-devicetype")
Scott Baker80678602019-11-14 16:57:36 -0800232 // TODO: context timeout
Kent Hagerman4f355f52020-03-30 16:01:33 -0400233 device := &voltha.Device{}
Kent Hagermanf5a67352020-04-30 15:15:26 -0400234 if have, err := agent.dbProxy.Get(ctx, agent.deviceID, device); err != nil {
Rohan Agrawal31f21802020-06-12 05:38:46 +0000235 logger.Errorw(ctx, "kv-get-failed", log.Fields{"device-id": agent.deviceID, "error": err})
Thomas Lee Se5a44012019-11-07 20:32:24 +0530236 return
Kent Hagerman4f355f52020-03-30 16:01:33 -0400237 } else if !have {
238 return // not found in kv
Thomas Lee Se5a44012019-11-07 20:32:24 +0530239 }
Kent Hagerman4f355f52020-03-30 16:01:33 -0400240
khenaidood948f772021-08-11 17:49:24 -0400241 agent.deviceType = device.Type
Kent Hagerman4f355f52020-03-30 16:01:33 -0400242 agent.device = device
khenaidood948f772021-08-11 17:49:24 -0400243 agent.adapterEndpoint = device.AdapterEndpoint
Kent Hagerman2a07b862020-06-19 15:23:07 -0400244 agent.portLoader.Load(ctx)
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530245 agent.transientStateLoader.Load(ctx)
246
Rohan Agrawal31f21802020-06-12 05:38:46 +0000247 logger.Debugw(ctx, "reconciled-device-agent-devicetype", log.Fields{"device-id": agent.deviceID, "type": agent.deviceType})
Scott Baker80678602019-11-14 16:57:36 -0800248}
249
khenaidoo442e7c72020-03-10 16:13:48 -0400250// onSuccess is a common callback for scenarios where we receive a nil response following a request to an adapter
khenaidood948f772021-08-11 17:49:24 -0400251func (agent *Agent) onSuccess(ctx context.Context, prevState, currState *common.AdminState_Types, deviceUpdateLog bool) {
252 if deviceUpdateLog {
253 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_SUCCESS}
254 desc := "adapter-response"
255 agent.logDeviceUpdate(ctx, prevState, currState, requestStatus, nil, desc)
256 return
257 }
258 logger.Debugw(ctx, "successful-operation", log.Fields{"device-id": agent.deviceID, "rpc": coreutils.GetRPCMetadataFromContext(ctx)})
khenaidoo442e7c72020-03-10 16:13:48 -0400259}
260
261// onFailure is a common callback for scenarios where we receive an error response following a request to an adapter
262// and the only action required is to publish the failed result on kafka
khenaidood948f772021-08-11 17:49:24 -0400263func (agent *Agent) onFailure(ctx context.Context, err error, prevState, currState *common.AdminState_Types, deviceUpdateLog bool) {
264 // Send an event on kafka
265 rpce := agent.deviceMgr.NewRPCEvent(ctx, agent.deviceID, err.Error(), nil)
266 go agent.deviceMgr.SendRPCEvent(ctx, "RPC_ERROR_RAISE_EVENT", rpce,
267 voltha.EventCategory_COMMUNICATION, nil, time.Now().Unix())
khenaidoo442e7c72020-03-10 16:13:48 -0400268
khenaidood948f772021-08-11 17:49:24 -0400269 // Log the device update event
270 if deviceUpdateLog {
271 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
272 desc := "adapter-response"
273 agent.logDeviceUpdate(ctx, prevState, currState, requestStatus, err, desc)
274 return
khenaidoo442e7c72020-03-10 16:13:48 -0400275 }
khenaidood948f772021-08-11 17:49:24 -0400276 logger.Errorw(ctx, "failed-operation", log.Fields{"error": err, "device-id": agent.deviceID, "rpc": coreutils.GetRPCMetadataFromContext(ctx)})
khenaidoo442e7c72020-03-10 16:13:48 -0400277}
278
khenaidoo6cd8e862021-11-12 16:33:43 -0500279// onForceDeleteResponse is invoked following a force delete request to an adapter.
280func (agent *Agent) onForceDeleteResponse(ctx context.Context, prevState, currState *common.AdminState_Types, dErr error) {
281 // Log the status
282 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_SUCCESS}
283 if dErr != nil {
284 requestStatus = &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
285 }
286 agent.logDeviceUpdate(ctx, prevState, currState, requestStatus, dErr, "adapter-force-delete-response")
287
288 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
289 logger.Errorw(ctx, "failed-getting-device-request-lock", log.Fields{"device-id": agent.deviceID, "error": err})
290 }
291 previousDeviceTransientState := agent.getTransientState()
292 newDevice := agent.cloneDeviceWithoutLock()
293
294 // Even on a delete error response, cleaup the device in the core
295 requestStatus = &common.OperationResp{Code: common.OperationResp_OPERATION_SUCCESS}
296 err := agent.updateDeviceWithTransientStateAndReleaseLock(ctx, newDevice,
297 core.DeviceTransientState_DELETING_POST_ADAPTER_RESPONSE, previousDeviceTransientState)
298 if err != nil {
299 requestStatus = &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
300 }
301 agent.logDeviceUpdate(ctx, prevState, currState, requestStatus, err, "transient-state-update")
302}
303
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530304// onDeleteSuccess is a common callback for scenarios where we receive a nil response following a delete request
305// to an adapter.
khenaidood948f772021-08-11 17:49:24 -0400306func (agent *Agent) onDeleteSuccess(ctx context.Context, prevState, currState *common.AdminState_Types) {
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530307 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
khenaidood948f772021-08-11 17:49:24 -0400308 logger.Errorw(ctx, "delete-device-failure", log.Fields{"device-id": agent.deviceID, "error": err})
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530309 }
310 previousDeviceTransientState := agent.getTransientState()
311 newDevice := agent.cloneDeviceWithoutLock()
312 if err := agent.updateDeviceWithTransientStateAndReleaseLock(ctx, newDevice,
khenaidood948f772021-08-11 17:49:24 -0400313 core.DeviceTransientState_DELETING_POST_ADAPTER_RESPONSE, previousDeviceTransientState); err != nil {
314 logger.Errorw(ctx, "delete-device-failure", log.Fields{"device-id": agent.deviceID, "error": err})
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530315 }
khenaidood948f772021-08-11 17:49:24 -0400316 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_SUCCESS}
317 desc := "adapter-response"
318 agent.logDeviceUpdate(ctx, prevState, currState, requestStatus, nil, desc)
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530319}
320
321// onDeleteFailure is a common callback for scenarios where we receive an error response following a delete request
Joey Armstrong393daca2023-07-06 08:47:54 -0400322//
323// to an adapter and the only action required is to return the error response.
khenaidood948f772021-08-11 17:49:24 -0400324func (agent *Agent) onDeleteFailure(ctx context.Context, err error, prevState, currState *common.AdminState_Types) {
325 logger.Errorw(ctx, "rpc-failed", log.Fields{"rpc": coreutils.GetRPCMetadataFromContext(ctx), "device-id": agent.deviceID, "error": err})
326
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530327 //Only updating of transient state is required, no transition.
khenaidood948f772021-08-11 17:49:24 -0400328 if er := agent.updateTransientState(ctx, core.DeviceTransientState_DELETE_FAILED); er != nil {
329 logger.Errorw(ctx, "failed-to-update-transient-state-as-delete-failed", log.Fields{"device-id": agent.deviceID, "error": er})
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530330 }
khenaidood948f772021-08-11 17:49:24 -0400331 rpce := agent.deviceMgr.NewRPCEvent(ctx, agent.deviceID, err.Error(), nil)
332 go agent.deviceMgr.SendRPCEvent(ctx, "RPC_ERROR_RAISE_EVENT", rpce,
333 voltha.EventCategory_COMMUNICATION, nil, time.Now().Unix())
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530334
khenaidood948f772021-08-11 17:49:24 -0400335 // Log the device update event
336 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
337 desc := "adapter-response"
338 agent.logDeviceUpdate(ctx, prevState, currState, requestStatus, err, desc)
Maninder9a1bc0d2020-10-26 11:34:02 +0530339}
340
Kent Hagermancba2f302020-07-28 13:37:36 -0400341// getDeviceReadOnly returns a device which MUST NOT be modified, but is safe to keep forever.
342func (agent *Agent) getDeviceReadOnly(ctx context.Context) (*voltha.Device, error) {
khenaidoo442e7c72020-03-10 16:13:48 -0400343 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
344 return nil, err
345 }
346 defer agent.requestQueue.RequestComplete()
Kent Hagermancba2f302020-07-28 13:37:36 -0400347 return agent.device, nil
khenaidoo92e62c52018-10-03 14:02:54 -0400348}
349
Kent Hagermancba2f302020-07-28 13:37:36 -0400350// getDeviceReadOnlyWithoutLock returns a device which MUST NOT be modified, but is safe to keep forever. This is very efficient.
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400351// The device lock MUST be held by the caller.
Kent Hagermancba2f302020-07-28 13:37:36 -0400352func (agent *Agent) getDeviceReadOnlyWithoutLock() *voltha.Device {
khenaidoo0db4c812020-05-27 15:27:30 -0400353 return agent.device
khenaidoo92e62c52018-10-03 14:02:54 -0400354}
355
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400356// cloneDeviceWithoutLock returns a copy of the device which is safe to modify.
357// The device lock MUST be held by the caller.
358func (agent *Agent) cloneDeviceWithoutLock() *voltha.Device {
359 return proto.Clone(agent.device).(*voltha.Device)
360}
361
khenaidood948f772021-08-11 17:49:24 -0400362func (agent *Agent) updateDeviceTypeAndEndpoint(ctx context.Context) error {
363 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
364 return err
365 }
366 changed := false
367 cloned := agent.cloneDeviceWithoutLock()
368 if cloned.Type == "" {
369 adapterType, err := agent.adapterMgr.GetAdapterType(cloned.Type)
370 if err != nil {
371 agent.requestQueue.RequestComplete()
372 return err
373 }
374 cloned.Type = adapterType
375 changed = true
376 }
377
378 if cloned.AdapterEndpoint == "" {
379 var err error
380 if cloned.AdapterEndpoint, err = agent.adapterMgr.GetAdapterEndpoint(ctx, cloned.Id, cloned.Type); err != nil {
381 agent.requestQueue.RequestComplete()
382 return err
383 }
384 agent.adapterEndpoint = cloned.AdapterEndpoint
385 changed = true
386 }
387
388 if changed {
389 return agent.updateDeviceAndReleaseLock(ctx, cloned)
390 }
391 agent.requestQueue.RequestComplete()
392 return nil
393}
394
khenaidoo3ab34882019-05-02 21:33:30 -0400395// enableDevice activates a preprovisioned or a disable device
Kent Hagerman2b216042020-04-03 18:28:56 -0400396func (agent *Agent) enableDevice(ctx context.Context) error {
Maninder9a1bc0d2020-10-26 11:34:02 +0530397 //To preserve and use oldDevice state as prev state in new device
khenaidood948f772021-08-11 17:49:24 -0400398 var err error
Maninder9a1bc0d2020-10-26 11:34:02 +0530399 var desc string
khenaidood948f772021-08-11 17:49:24 -0400400 var prevAdminState, currAdminState common.AdminState_Types
401 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
Maninder9a1bc0d2020-10-26 11:34:02 +0530402
khenaidood948f772021-08-11 17:49:24 -0400403 defer func() { agent.logDeviceUpdate(ctx, &prevAdminState, &currAdminState, requestStatus, err, desc) }()
Maninder9a1bc0d2020-10-26 11:34:02 +0530404
khenaidood948f772021-08-11 17:49:24 -0400405 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
khenaidoo442e7c72020-03-10 16:13:48 -0400406 return err
407 }
Himani Chawlab4c25912020-11-12 17:16:38 +0530408 logger.Debugw(ctx, "enable-device", log.Fields{"device-id": agent.deviceID})
khenaidoo21d51152019-02-01 13:48:37 -0500409
Kent Hagermancba2f302020-07-28 13:37:36 -0400410 oldDevice := agent.getDeviceReadOnlyWithoutLock()
khenaidood948f772021-08-11 17:49:24 -0400411 prevAdminState = oldDevice.AdminState
Maninder9a1bc0d2020-10-26 11:34:02 +0530412
Maninder2195ccc2021-06-23 20:23:01 +0530413 if !agent.proceedWithRequest(oldDevice) {
414 agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -0400415 err = status.Errorf(codes.FailedPrecondition, "cannot complete operation as device deletion is in progress or reconciling is in progress/failed: %s", agent.deviceID)
416 return err
Maninder2195ccc2021-06-23 20:23:01 +0530417 }
Mahir Gunyel92dd1212021-10-22 11:42:56 -0700418 //vol-4275 TST meeting 08/04/2021: Let EnableDevice to be called again if device is in FAILED operational state,
419 //even the admin state is ENABLED.
420 if oldDevice.AdminState == voltha.AdminState_ENABLED && oldDevice.OperStatus != voltha.OperStatus_FAILED {
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400421 logger.Warnw(ctx, "device-already-enabled", log.Fields{"device-id": agent.deviceID})
422 agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -0400423 err = status.Errorf(codes.FailedPrecondition, fmt.Sprintf("cannot-enable-an-already-enabled-device: %s", oldDevice.Id))
Matteo Scandolod525ae32020-04-02 17:27:29 -0700424 return err
npujar1d86a522019-11-14 17:11:16 +0530425 }
426
khenaidood948f772021-08-11 17:49:24 -0400427 // Verify whether there is a device type that supports this device type
428 _, err = agent.adapterMgr.GetAdapterType(oldDevice.Type)
429 if err != nil {
430 agent.requestQueue.RequestComplete()
431 return err
432 }
433
434 // Update device adapter endpoint if not set. This is set once by the Core and use as is by the adapters. E.g if this is a
435 // child device then the parent adapter will use this device's adapter endpoint (set here) to communicate with it.
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400436 newDevice := agent.cloneDeviceWithoutLock()
khenaidood948f772021-08-11 17:49:24 -0400437 if newDevice.AdapterEndpoint == "" {
438 if newDevice.AdapterEndpoint, err = agent.adapterMgr.GetAdapterEndpoint(ctx, newDevice.Id, newDevice.Type); err != nil {
439 agent.requestQueue.RequestComplete()
440 return err
441 }
442 agent.adapterEndpoint = newDevice.AdapterEndpoint
443 }
npujar1d86a522019-11-14 17:11:16 +0530444
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400445 // Update the Admin State and set the operational state to activating before sending the request to the Adapters
446 newDevice.AdminState = voltha.AdminState_ENABLED
447 newDevice.OperStatus = voltha.OperStatus_ACTIVATING
Maninder9a1bc0d2020-10-26 11:34:02 +0530448
khenaidoo442e7c72020-03-10 16:13:48 -0400449 // Adopt the device if it was in pre-provision state. In all other cases, try to re-enable it.
khenaidood948f772021-08-11 17:49:24 -0400450 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
khenaidoo442e7c72020-03-10 16:13:48 -0400451 if err != nil {
khenaidood948f772021-08-11 17:49:24 -0400452 logger.Errorw(ctx, "grpc-client-nil",
453 log.Fields{
454 "error": err,
455 "device-id": agent.deviceID,
456 "device-type": agent.deviceType,
457 "adapter-endpoint": newDevice.AdapterEndpoint,
458 })
459 agent.requestQueue.RequestComplete()
khenaidoo442e7c72020-03-10 16:13:48 -0400460 return err
461 }
khenaidood948f772021-08-11 17:49:24 -0400462 subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
463 requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS
464 go func() {
465 defer cancel()
466 var err error
467 if oldDevice.AdminState == voltha.AdminState_PREPROVISIONED {
468 _, err = client.AdoptDevice(subCtx, newDevice)
469 } else {
470 _, err = client.ReEnableDevice(subCtx, newDevice)
471 }
472 if err == nil {
473 agent.onSuccess(subCtx, nil, nil, true)
474 } else {
475 agent.onFailure(subCtx, err, nil, nil, true)
Himani Chawlab4c25912020-11-12 17:16:38 +0530476 }
477 }()
khenaidood948f772021-08-11 17:49:24 -0400478
479 // Update device
480 if err = agent.updateDeviceAndReleaseLock(ctx, newDevice); err != nil {
481 return err
khenaidoo2c6a0992019-04-29 13:46:56 -0400482 }
khenaidood948f772021-08-11 17:49:24 -0400483 currAdminState = newDevice.AdminState
484 return nil
khenaidoo2c6a0992019-04-29 13:46:56 -0400485}
486
Joey Armstrong393daca2023-07-06 08:47:54 -0400487// addFlowsAndGroups adds the "newFlows" and "newGroups" from the existing flows/groups and sends the update to the
488// adapters
khenaidoo9beaaf12021-10-19 17:32:01 -0400489func (agent *Agent) addFlowsAndGroups(ctx context.Context, newFlows []*ofp.OfpFlowStats, newGroups []*ofp.OfpGroupEntry, flowMetadata *ofp.FlowMetadata) error {
Mahir Gunyel03de0d32020-06-03 01:36:59 -0700490 var flwResponse, grpResponse coreutils.Response
491 var err error
492 //if new flow list is empty then the called function returns quickly
493 if flwResponse, err = agent.addFlowsToAdapter(ctx, newFlows, flowMetadata); err != nil {
A R Karthick5c28f552019-12-11 22:47:44 -0800494 return err
495 }
Mahir Gunyel03de0d32020-06-03 01:36:59 -0700496 //if new group list is empty then the called function returns quickly
497 if grpResponse, err = agent.addGroupsToAdapter(ctx, newGroups, flowMetadata); err != nil {
498 return err
499 }
Himani Chawla4b4bd252021-11-08 15:59:40 +0530500 if errs := coreutils.WaitForNilOrErrorResponses(agent.flowTimeout, flwResponse, grpResponse); errs != nil {
khenaidood948f772021-08-11 17:49:24 -0400501 logger.Warnw(ctx, "adapter-response", log.Fields{"device-id": agent.deviceID, "result": errs})
khenaidoo442e7c72020-03-10 16:13:48 -0400502 return status.Errorf(codes.Aborted, "flow-failure-device-%s", agent.deviceID)
khenaidoo0458db62019-06-20 08:50:36 -0400503 }
khenaidoo0458db62019-06-20 08:50:36 -0400504 return nil
505}
506
Joey Armstrong393daca2023-07-06 08:47:54 -0400507// deleteFlowsAndGroups removes the "flowsToDel" and "groupsToDel" from the existing flows/groups and sends the update to the
508// adapters
khenaidoo9beaaf12021-10-19 17:32:01 -0400509func (agent *Agent) deleteFlowsAndGroups(ctx context.Context, flowsToDel []*ofp.OfpFlowStats, groupsToDel []*ofp.OfpGroupEntry, flowMetadata *ofp.FlowMetadata) error {
Mahir Gunyel03de0d32020-06-03 01:36:59 -0700510 var flwResponse, grpResponse coreutils.Response
511 var err error
512 if flwResponse, err = agent.deleteFlowsFromAdapter(ctx, flowsToDel, flowMetadata); err != nil {
A R Karthick5c28f552019-12-11 22:47:44 -0800513 return err
514 }
Mahir Gunyel03de0d32020-06-03 01:36:59 -0700515 if grpResponse, err = agent.deleteGroupsFromAdapter(ctx, groupsToDel, flowMetadata); err != nil {
516 return err
517 }
518
Himani Chawla4b4bd252021-11-08 15:59:40 +0530519 if res := coreutils.WaitForNilOrErrorResponses(agent.flowTimeout, flwResponse, grpResponse); res != nil {
khenaidoo0458db62019-06-20 08:50:36 -0400520 return status.Errorf(codes.Aborted, "errors-%s", res)
521 }
522 return nil
khenaidoo0458db62019-06-20 08:50:36 -0400523}
524
Joey Armstrong393daca2023-07-06 08:47:54 -0400525// updateFlowsAndGroups replaces the existing flows and groups with "updatedFlows" and "updatedGroups" respectively. It
526// also sends the updates to the adapters
khenaidoo9beaaf12021-10-19 17:32:01 -0400527func (agent *Agent) updateFlowsAndGroups(ctx context.Context, updatedFlows []*ofp.OfpFlowStats, updatedGroups []*ofp.OfpGroupEntry, flowMetadata *ofp.FlowMetadata) error {
Mahir Gunyel03de0d32020-06-03 01:36:59 -0700528 var flwResponse, grpResponse coreutils.Response
529 var err error
530 if flwResponse, err = agent.updateFlowsToAdapter(ctx, updatedFlows, flowMetadata); err != nil {
A R Karthick5c28f552019-12-11 22:47:44 -0800531 return err
532 }
Mahir Gunyel03de0d32020-06-03 01:36:59 -0700533 if grpResponse, err = agent.updateGroupsToAdapter(ctx, updatedGroups, flowMetadata); err != nil {
534 return err
535 }
536
Himani Chawla4b4bd252021-11-08 15:59:40 +0530537 if res := coreutils.WaitForNilOrErrorResponses(agent.flowTimeout, flwResponse, grpResponse); res != nil {
khenaidoo0458db62019-06-20 08:50:36 -0400538 return status.Errorf(codes.Aborted, "errors-%s", res)
539 }
540 return nil
khenaidoo19d7b632018-10-30 10:49:50 -0400541}
542
Joey Armstrong393daca2023-07-06 08:47:54 -0400543// disableDevice disable a device
Kent Hagerman2b216042020-04-03 18:28:56 -0400544func (agent *Agent) disableDevice(ctx context.Context) error {
khenaidood948f772021-08-11 17:49:24 -0400545 var err error
Maninder9a1bc0d2020-10-26 11:34:02 +0530546 var desc string
khenaidood948f772021-08-11 17:49:24 -0400547 var prevAdminState, currAdminState common.AdminState_Types
548 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
549 defer func() { agent.logDeviceUpdate(ctx, &prevAdminState, &currAdminState, requestStatus, err, desc) }()
Maninder9a1bc0d2020-10-26 11:34:02 +0530550
khenaidood948f772021-08-11 17:49:24 -0400551 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
khenaidoo442e7c72020-03-10 16:13:48 -0400552 return err
553 }
Himani Chawlab4c25912020-11-12 17:16:38 +0530554 logger.Debugw(ctx, "disable-device", log.Fields{"device-id": agent.deviceID})
khenaidoo6e55d9e2019-12-12 18:26:26 -0500555
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400556 cloned := agent.cloneDeviceWithoutLock()
khenaidood948f772021-08-11 17:49:24 -0400557 prevAdminState = agent.device.AdminState
khenaidoo6e55d9e2019-12-12 18:26:26 -0500558
Maninder2195ccc2021-06-23 20:23:01 +0530559 if !agent.proceedWithRequest(cloned) {
nikesh.krishnanc8beca52023-12-07 13:45:03 +0530560 err = status.Errorf(codes.FailedPrecondition, "cannot complete operation as device deletion is in progress/failed: %s", agent.deviceID)
Maninder2195ccc2021-06-23 20:23:01 +0530561 agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -0400562 return err
Maninder2195ccc2021-06-23 20:23:01 +0530563 }
564
khenaidoo6e55d9e2019-12-12 18:26:26 -0500565 if cloned.AdminState == voltha.AdminState_DISABLED {
Maninder9a1bc0d2020-10-26 11:34:02 +0530566 desc = "device-already-disabled"
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400567 agent.requestQueue.RequestComplete()
npujar1d86a522019-11-14 17:11:16 +0530568 return nil
569 }
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530570 if cloned.AdminState == voltha.AdminState_PREPROVISIONED {
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400571 agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -0400572 err = status.Errorf(codes.FailedPrecondition, "deviceId:%s, invalid-admin-state:%s", agent.deviceID, cloned.AdminState)
573 return err
npujar1d86a522019-11-14 17:11:16 +0530574 }
Maninder0aabf0c2021-03-17 14:55:14 +0530575
npujar1d86a522019-11-14 17:11:16 +0530576 // Update the Admin State and operational state before sending the request out
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400577 cloned.AdminState = voltha.AdminState_DISABLED
578 cloned.OperStatus = voltha.OperStatus_UNKNOWN
Maninder9a1bc0d2020-10-26 11:34:02 +0530579
khenaidood948f772021-08-11 17:49:24 -0400580 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
khenaidoo442e7c72020-03-10 16:13:48 -0400581 if err != nil {
khenaidood948f772021-08-11 17:49:24 -0400582 logger.Errorw(ctx, "grpc-client-nil",
583 log.Fields{
584 "error": err,
585 "device-id": agent.deviceID,
586 "device-type": agent.deviceType,
587 "adapter-endpoint": cloned.AdapterEndpoint,
588 })
589 agent.requestQueue.RequestComplete()
npujar1d86a522019-11-14 17:11:16 +0530590 return err
khenaidoo0a822f92019-05-08 15:15:57 -0400591 }
khenaidood948f772021-08-11 17:49:24 -0400592 subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
593 requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS
594 go func() {
595 defer cancel()
596 _, err := client.DisableDevice(subCtx, cloned)
597 if err == nil {
598 agent.onSuccess(subCtx, nil, nil, true)
599 } else {
600 agent.onFailure(subCtx, err, nil, nil, true)
601 }
602 }()
Maninder9a1bc0d2020-10-26 11:34:02 +0530603
khenaidood948f772021-08-11 17:49:24 -0400604 // Update device
605 if err = agent.updateDeviceAndReleaseLock(ctx, cloned); err != nil {
606 return err
607 }
608 currAdminState = cloned.AdminState
khenaidoo0a822f92019-05-08 15:15:57 -0400609
khenaidoo92e62c52018-10-03 14:02:54 -0400610 return nil
611}
612
Kent Hagerman2b216042020-04-03 18:28:56 -0400613func (agent *Agent) rebootDevice(ctx context.Context) error {
Maninder9a1bc0d2020-10-26 11:34:02 +0530614 var desc string
khenaidood948f772021-08-11 17:49:24 -0400615 var err error
616 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
617 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
Maninder9a1bc0d2020-10-26 11:34:02 +0530618
khenaidood948f772021-08-11 17:49:24 -0400619 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
Maninder9a1bc0d2020-10-26 11:34:02 +0530620 desc = err.Error()
npujar1d86a522019-11-14 17:11:16 +0530621 return err
khenaidoo4d4802d2018-10-04 21:59:49 -0400622 }
khenaidoo442e7c72020-03-10 16:13:48 -0400623 defer agent.requestQueue.RequestComplete()
Himani Chawlab4c25912020-11-12 17:16:38 +0530624 logger.Debugw(ctx, "reboot-device", log.Fields{"device-id": agent.deviceID})
khenaidoo442e7c72020-03-10 16:13:48 -0400625
Kent Hagermancba2f302020-07-28 13:37:36 -0400626 device := agent.getDeviceReadOnlyWithoutLock()
Maninder2195ccc2021-06-23 20:23:01 +0530627
628 if !agent.proceedWithRequest(device) {
khenaidood948f772021-08-11 17:49:24 -0400629 err = status.Errorf(codes.FailedPrecondition, "cannot complete operation as device deletion is in progress or reconciling is in progress/failed:%s", agent.deviceID)
khenaidoo442e7c72020-03-10 16:13:48 -0400630 return err
631 }
Maninder9a1bc0d2020-10-26 11:34:02 +0530632
khenaidood948f772021-08-11 17:49:24 -0400633 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
634 if err != nil {
635 logger.Errorw(ctx, "grpc-client-nil",
636 log.Fields{
637 "error": err,
638 "device-id": agent.deviceID,
639 "device-type": agent.deviceType,
640 "adapter-endpoint": device.AdapterEndpoint,
641 })
642 return err
643 }
644 subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
645 requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS
646 go func() {
647 defer cancel()
648 _, err := client.RebootDevice(subCtx, device)
649 if err == nil {
650 agent.onSuccess(subCtx, nil, nil, true)
651 } else {
652 agent.onFailure(subCtx, err, nil, nil, true)
653 }
654 }()
khenaidoo4d4802d2018-10-04 21:59:49 -0400655 return nil
656}
657
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530658func (agent *Agent) deleteDeviceForce(ctx context.Context) error {
Himani Chawlab4c25912020-11-12 17:16:38 +0530659 logger.Debugw(ctx, "delete-device-force", log.Fields{"device-id": agent.deviceID})
Maninder9a1bc0d2020-10-26 11:34:02 +0530660
661 var desc string
khenaidood948f772021-08-11 17:49:24 -0400662 var err error
663 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
664 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
Maninder9a1bc0d2020-10-26 11:34:02 +0530665
khenaidood948f772021-08-11 17:49:24 -0400666 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530667 return err
668 }
669 // Get the device Transient state, return err if it is DELETING
670 previousDeviceTransientState := agent.getTransientState()
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530671 device := agent.cloneDeviceWithoutLock()
khenaidood948f772021-08-11 17:49:24 -0400672 if !agent.isForceDeletingAllowed(previousDeviceTransientState, device) {
673 agent.requestQueue.RequestComplete()
674 err = status.Error(codes.FailedPrecondition, fmt.Sprintf("deviceId:%s, force deletion is in progress", agent.deviceID))
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530675 return err
676 }
Himani Chawlab4c25912020-11-12 17:16:38 +0530677
khenaidood948f772021-08-11 17:49:24 -0400678 previousAdminState := device.AdminState
679 if previousAdminState != common.AdminState_PREPROVISIONED {
khenaidoo9beaaf12021-10-19 17:32:01 -0400680 var client adapter_service.AdapterServiceClient
khenaidood948f772021-08-11 17:49:24 -0400681 client, err = agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530682 if err != nil {
khenaidood948f772021-08-11 17:49:24 -0400683 logger.Errorw(ctx, "grpc-client-nil",
684 log.Fields{
685 "error": err,
686 "device-id": agent.deviceID,
687 "device-type": agent.deviceType,
688 "adapter-endpoint": device.AdapterEndpoint,
689 })
690 agent.requestQueue.RequestComplete()
khenaidoo68a5e0c2021-11-06 13:08:03 -0400691 return fmt.Errorf("remote-not-reachable %w", errNoConnection)
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530692 }
khenaidood948f772021-08-11 17:49:24 -0400693 subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
694 requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS
695 go func() {
696 defer cancel()
697 _, err := client.DeleteDevice(subCtx, device)
698 if err == nil {
699 agent.onSuccess(subCtx, nil, nil, true)
700 } else {
701 agent.onFailure(subCtx, err, nil, nil, true)
702 }
703 }()
704 }
705
706 // Update device
707 if err = agent.updateDeviceWithTransientStateAndReleaseLock(ctx, device,
708 core.DeviceTransientState_FORCE_DELETING, previousDeviceTransientState); err != nil {
709 return err
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530710 }
711 return nil
712}
713
Kent Hagerman2b216042020-04-03 18:28:56 -0400714func (agent *Agent) deleteDevice(ctx context.Context) error {
Himani Chawlab4c25912020-11-12 17:16:38 +0530715 logger.Debugw(ctx, "delete-device", log.Fields{"device-id": agent.deviceID})
Maninder9a1bc0d2020-10-26 11:34:02 +0530716
717 var desc string
khenaidood948f772021-08-11 17:49:24 -0400718 var err error
719 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
720 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
Maninder9a1bc0d2020-10-26 11:34:02 +0530721
khenaidood948f772021-08-11 17:49:24 -0400722 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
Maninder9a1bc0d2020-10-26 11:34:02 +0530723 desc = err.Error()
khenaidoo442e7c72020-03-10 16:13:48 -0400724 return err
725 }
Maninder0aabf0c2021-03-17 14:55:14 +0530726
Maninder2195ccc2021-06-23 20:23:01 +0530727 device := agent.cloneDeviceWithoutLock()
728
729 if !agent.proceedWithRequest(device) {
Maninder0aabf0c2021-03-17 14:55:14 +0530730 agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -0400731 err = status.Errorf(codes.FailedPrecondition, "cannot complete operation as device deletion is in progress or reconciling is in progress/failed: %s", agent.deviceID)
732 return err
Maninder0aabf0c2021-03-17 14:55:14 +0530733 }
734
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530735 // Get the device Transient state, return err if it is DELETING
736 previousDeviceTransientState := agent.getTransientState()
khenaidoo6e55d9e2019-12-12 18:26:26 -0500737
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530738 previousAdminState := device.AdminState
739 // Change the device transient state to DELETING_FROM_ADAPTER state till the device is removed from adapters.
khenaidood948f772021-08-11 17:49:24 -0400740 currentDeviceTransientState := core.DeviceTransientState_DELETING_FROM_ADAPTER
khenaidoo442e7c72020-03-10 16:13:48 -0400741
khenaidood948f772021-08-11 17:49:24 -0400742 if previousAdminState == common.AdminState_PREPROVISIONED {
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530743 // Change the state to DELETING POST ADAPTER RESPONSE directly as adapters have no info of the device.
khenaidood948f772021-08-11 17:49:24 -0400744 currentDeviceTransientState = core.DeviceTransientState_DELETING_POST_ADAPTER_RESPONSE
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530745 }
Gustavo Silva9a0ed002022-10-11 11:06:58 -0300746 // Update device and release lock
747 if err = agent.updateDeviceWithTransientStateAndReleaseLock(ctx, device,
748 currentDeviceTransientState, previousDeviceTransientState); err != nil {
749 desc = err.Error()
750 return err
751 }
khenaidood948f772021-08-11 17:49:24 -0400752 // If the device was in pre-prov state (only parent device are in that state) then do not send the request to the
753 // adapter
754 if previousAdminState != common.AdminState_PREPROVISIONED {
khenaidoo9beaaf12021-10-19 17:32:01 -0400755 var client adapter_service.AdapterServiceClient
khenaidood948f772021-08-11 17:49:24 -0400756 client, err = agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
757 if err != nil {
758 logger.Errorw(ctx, "grpc-client-nil",
759 log.Fields{
760 "error": err,
761 "device-id": agent.deviceID,
762 "device-type": agent.deviceType,
763 "adapter-endpoint": device.AdapterEndpoint,
764 })
765 agent.requestQueue.RequestComplete()
766 return err
767 }
768 subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
769 requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS
Gustavo Silva9a0ed002022-10-11 11:06:58 -0300770 if _, err = client.DeleteDevice(subCtx, device); err != nil {
771 agent.onDeleteFailure(subCtx, err, &previousAdminState, &agent.device.AdminState)
772 } else {
773 agent.onDeleteSuccess(subCtx, &previousAdminState, &agent.device.AdminState)
774 }
775 cancel()
khenaidood948f772021-08-11 17:49:24 -0400776 }
Gustavo Silva9a0ed002022-10-11 11:06:58 -0300777 return err
khenaidoo4d4802d2018-10-04 21:59:49 -0400778}
779
Kent Hagerman2b216042020-04-03 18:28:56 -0400780func (agent *Agent) setParentID(ctx context.Context, device *voltha.Device, parentID string) error {
khenaidoo442e7c72020-03-10 16:13:48 -0400781 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
782 return err
783 }
Himani Chawlab4c25912020-11-12 17:16:38 +0530784 logger.Debugw(ctx, "set-parent-id", log.Fields{"device-id": device.Id, "parent-id": parentID})
khenaidoo6e55d9e2019-12-12 18:26:26 -0500785
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400786 cloned := agent.cloneDeviceWithoutLock()
npujar1d86a522019-11-14 17:11:16 +0530787 cloned.ParentId = parentID
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400788 return agent.updateDeviceAndReleaseLock(ctx, cloned)
khenaidooad06fd72019-10-28 12:26:05 -0400789}
790
khenaidoo442e7c72020-03-10 16:13:48 -0400791// getSwitchCapability retrieves the switch capability of a parent device
khenaidoo9beaaf12021-10-19 17:32:01 -0400792func (agent *Agent) getSwitchCapability(ctx context.Context) (*ca.SwitchCapability, error) {
Himani Chawlab4c25912020-11-12 17:16:38 +0530793 logger.Debugw(ctx, "get-switch-capability", log.Fields{"device-id": agent.deviceID})
khenaidoo442e7c72020-03-10 16:13:48 -0400794
Kent Hagermancba2f302020-07-28 13:37:36 -0400795 device, err := agent.getDeviceReadOnly(ctx)
khenaidoo442e7c72020-03-10 16:13:48 -0400796 if err != nil {
khenaidoob9203542018-09-17 22:56:37 -0400797 return nil, err
khenaidoob9203542018-09-17 22:56:37 -0400798 }
khenaidood948f772021-08-11 17:49:24 -0400799
800 // Get the gRPC client
801 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
khenaidoo442e7c72020-03-10 16:13:48 -0400802 if err != nil {
803 return nil, err
804 }
805
khenaidood948f772021-08-11 17:49:24 -0400806 return client.GetOfpDeviceInfo(ctx, device)
khenaidoob9203542018-09-17 22:56:37 -0400807}
808
khenaidood948f772021-08-11 17:49:24 -0400809func (agent *Agent) onPacketFailure(ctx context.Context, err error, packet *ofp.OfpPacketOut) {
810 logger.Errorw(ctx, "packet-out-error", log.Fields{
khenaidoo442e7c72020-03-10 16:13:48 -0400811 "device-id": agent.deviceID,
khenaidood948f772021-08-11 17:49:24 -0400812 "error": err.Error(),
813 "packet": hex.EncodeToString(packet.Data),
khenaidoo442e7c72020-03-10 16:13:48 -0400814 })
khenaidood948f772021-08-11 17:49:24 -0400815 rpce := agent.deviceMgr.NewRPCEvent(ctx, agent.deviceID, err.Error(), nil)
816 go agent.deviceMgr.SendRPCEvent(ctx, "RPC_ERROR_RAISE_EVENT", rpce,
817 voltha.EventCategory_COMMUNICATION, nil, time.Now().Unix())
khenaidoo442e7c72020-03-10 16:13:48 -0400818}
819
Kent Hagerman2b216042020-04-03 18:28:56 -0400820func (agent *Agent) packetOut(ctx context.Context, outPort uint32, packet *ofp.OfpPacketOut) error {
Scott Baker80678602019-11-14 16:57:36 -0800821 if agent.deviceType == "" {
npujar467fe752020-01-16 20:17:45 +0530822 agent.reconcileWithKVStore(ctx)
Scott Baker80678602019-11-14 16:57:36 -0800823 }
khenaidoofdbad6e2018-11-06 22:26:38 -0500824 // Send packet to adapter
khenaidood948f772021-08-11 17:49:24 -0400825 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
khenaidoo442e7c72020-03-10 16:13:48 -0400826 if err != nil {
khenaidood948f772021-08-11 17:49:24 -0400827 logger.Errorw(ctx, "grpc-client-nil",
828 log.Fields{
829 "error": err,
830 "device-id": agent.deviceID,
831 "device-type": agent.deviceType,
832 })
833 return err
khenaidoofdbad6e2018-11-06 22:26:38 -0500834 }
khenaidood948f772021-08-11 17:49:24 -0400835 subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
836 go func() {
837 defer cancel()
khenaidoo9beaaf12021-10-19 17:32:01 -0400838 _, err := client.SendPacketOut(subCtx, &ca.PacketOut{
khenaidood948f772021-08-11 17:49:24 -0400839 DeviceId: agent.deviceID,
840 EgressPortNo: outPort,
841 Packet: packet,
842 })
843 if err == nil {
844 agent.onSuccess(subCtx, nil, nil, false)
845 } else {
846 agent.onPacketFailure(subCtx, err, packet)
847 }
848 }()
khenaidoofdbad6e2018-11-06 22:26:38 -0500849 return nil
850}
851
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400852func (agent *Agent) updateDeviceUsingAdapterData(ctx context.Context, device *voltha.Device) error {
khenaidood948f772021-08-11 17:49:24 -0400853 var err error
854 var desc string
855 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
856 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
857
858 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400859 return err
860 }
Himani Chawlab4c25912020-11-12 17:16:38 +0530861 logger.Debugw(ctx, "update-device-using-adapter-data", log.Fields{"device-id": device.Id})
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400862
863 cloned := agent.cloneDeviceWithoutLock()
Mahir Gunyel8e2707d2019-07-25 00:36:21 -0700864 cloned.Root = device.Root
865 cloned.Vendor = device.Vendor
866 cloned.Model = device.Model
867 cloned.SerialNumber = device.SerialNumber
868 cloned.MacAddress = device.MacAddress
869 cloned.Vlan = device.Vlan
870 cloned.Reason = device.Reason
Andrea Campanella025667e2021-01-14 11:50:07 +0100871 cloned.ImageDownloads = device.ImageDownloads
khenaidood948f772021-08-11 17:49:24 -0400872 cloned.OperStatus = device.OperStatus
873 cloned.ConnectStatus = device.ConnectStatus
874 if err = agent.updateDeviceAndReleaseLock(ctx, cloned); err == nil {
875 requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
876 }
877 return err
khenaidoo43c82122018-11-22 18:38:28 -0500878}
879
Kent Hagerman2b216042020-04-03 18:28:56 -0400880func (agent *Agent) updateDeviceStatus(ctx context.Context, operStatus voltha.OperStatus_Types, connStatus voltha.ConnectStatus_Types) error {
khenaidood948f772021-08-11 17:49:24 -0400881 var err error
882 var desc string
883 opStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
884 defer func() { agent.logDeviceUpdate(ctx, nil, nil, opStatus, err, desc) }()
885
886 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
khenaidoo442e7c72020-03-10 16:13:48 -0400887 return err
888 }
khenaidoo6e55d9e2019-12-12 18:26:26 -0500889
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400890 cloned := agent.cloneDeviceWithoutLock()
npujar1d86a522019-11-14 17:11:16 +0530891 // Ensure the enums passed in are valid - they will be invalid if they are not set when this function is invoked
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400892 if s, ok := voltha.ConnectStatus_Types_name[int32(connStatus)]; ok {
khenaidood948f772021-08-11 17:49:24 -0400893 logger.Debugw(ctx, "update-device-conn-status", log.Fields{"ok": ok, "val": s})
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400894 cloned.ConnectStatus = connStatus
npujar1d86a522019-11-14 17:11:16 +0530895 }
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400896 if s, ok := voltha.OperStatus_Types_name[int32(operStatus)]; ok {
khenaidood948f772021-08-11 17:49:24 -0400897 logger.Debugw(ctx, "update-device-oper-status", log.Fields{"ok": ok, "val": s})
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400898 cloned.OperStatus = operStatus
npujar1d86a522019-11-14 17:11:16 +0530899 }
Himani Chawlab4c25912020-11-12 17:16:38 +0530900 logger.Debugw(ctx, "update-device-status", log.Fields{"device-id": cloned.Id, "oper-status": cloned.OperStatus, "connect-status": cloned.ConnectStatus})
npujar1d86a522019-11-14 17:11:16 +0530901 // Store the device
khenaidood948f772021-08-11 17:49:24 -0400902 if err = agent.updateDeviceAndReleaseLock(ctx, cloned); err == nil {
903 opStatus.Code = common.OperationResp_OPERATION_SUCCESS
904 }
905 return err
khenaidoo92e62c52018-10-03 14:02:54 -0400906}
907
khenaidoob9203542018-09-17 22:56:37 -0400908// TODO: A generic device update by attribute
Kent Hagerman2b216042020-04-03 18:28:56 -0400909func (agent *Agent) updateDeviceAttribute(ctx context.Context, name string, value interface{}) {
khenaidoob9203542018-09-17 22:56:37 -0400910 if value == nil {
911 return
912 }
khenaidoo6e55d9e2019-12-12 18:26:26 -0500913
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400914 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
915 logger.Warnw(ctx, "request-aborted", log.Fields{"device-id": agent.deviceID, "name": name, "error": err})
916 return
917 }
918
919 cloned := agent.cloneDeviceWithoutLock()
khenaidoob9203542018-09-17 22:56:37 -0400920 updated := false
khenaidoo6e55d9e2019-12-12 18:26:26 -0500921 s := reflect.ValueOf(cloned).Elem()
khenaidoob9203542018-09-17 22:56:37 -0400922 if s.Kind() == reflect.Struct {
923 // exported field
924 f := s.FieldByName(name)
925 if f.IsValid() && f.CanSet() {
926 switch f.Kind() {
927 case reflect.String:
928 f.SetString(value.(string))
929 updated = true
930 case reflect.Uint32:
931 f.SetUint(uint64(value.(uint32)))
932 updated = true
933 case reflect.Bool:
934 f.SetBool(value.(bool))
935 updated = true
936 }
937 }
938 }
divyadesaicb8b59d2020-08-18 09:55:47 +0000939 logger.Debugw(ctx, "update-field-status", log.Fields{"device-id": cloned.Id, "name": name, "updated": updated})
khenaidoob9203542018-09-17 22:56:37 -0400940 // Save the data
khenaidoo6e55d9e2019-12-12 18:26:26 -0500941
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400942 if err := agent.updateDeviceAndReleaseLock(ctx, cloned); err != nil {
Rohan Agrawal31f21802020-06-12 05:38:46 +0000943 logger.Warnw(ctx, "attribute-update-failed", log.Fields{"attribute": name, "value": value})
khenaidoob9203542018-09-17 22:56:37 -0400944 }
khenaidoob9203542018-09-17 22:56:37 -0400945}
serkant.uluderya334479d2019-04-10 08:26:15 -0700946
Kent Hagerman45a13e42020-04-13 12:23:50 -0400947func (agent *Agent) simulateAlarm(ctx context.Context, simulateReq *voltha.SimulateAlarmRequest) error {
khenaidood948f772021-08-11 17:49:24 -0400948 var err error
949 var desc string
950 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
951 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
952
953 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
khenaidoo442e7c72020-03-10 16:13:48 -0400954 return err
955 }
956 defer agent.requestQueue.RequestComplete()
Himani Chawlab4c25912020-11-12 17:16:38 +0530957 logger.Debugw(ctx, "simulate-alarm", log.Fields{"device-id": agent.deviceID})
khenaidoo6e55d9e2019-12-12 18:26:26 -0500958
Kent Hagermancba2f302020-07-28 13:37:36 -0400959 device := agent.getDeviceReadOnlyWithoutLock()
khenaidoo6e55d9e2019-12-12 18:26:26 -0500960
khenaidood948f772021-08-11 17:49:24 -0400961 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
khenaidoo442e7c72020-03-10 16:13:48 -0400962 if err != nil {
khenaidood948f772021-08-11 17:49:24 -0400963 logger.Errorw(ctx, "grpc-client-nil",
964 log.Fields{
965 "error": err,
966 "device-id": agent.deviceID,
967 "device-type": agent.deviceType,
968 "adapter-endpoint": device.AdapterEndpoint,
969 })
npujar1d86a522019-11-14 17:11:16 +0530970 return err
serkant.uluderya334479d2019-04-10 08:26:15 -0700971 }
khenaidood948f772021-08-11 17:49:24 -0400972 subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
973 requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS
974 go func() {
975 defer cancel()
khenaidoo9beaaf12021-10-19 17:32:01 -0400976 _, err := client.SimulateAlarm(subCtx, &ca.SimulateAlarmMessage{Device: device, Request: simulateReq})
khenaidood948f772021-08-11 17:49:24 -0400977 if err == nil {
978 agent.onSuccess(subCtx, nil, nil, false)
979 } else {
980 agent.onFailure(subCtx, err, nil, nil, false)
981 }
982 }()
serkant.uluderya334479d2019-04-10 08:26:15 -0700983 return nil
984}
Mahir Gunyelb5851672019-07-24 10:46:26 +0300985
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400986// This function updates the device in the DB, releases the device lock, and runs any state transitions.
987// The calling function MUST hold the device lock. The caller MUST NOT modify the device after this is called.
988func (agent *Agent) updateDeviceAndReleaseLock(ctx context.Context, device *voltha.Device) error {
989 // fail early if this agent is no longer valid
Kent Hagerman4f355f52020-03-30 16:01:33 -0400990 if agent.stopped {
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400991 agent.requestQueue.RequestComplete()
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530992 return errors.New("device-agent-stopped")
Thomas Lee Se5a44012019-11-07 20:32:24 +0530993 }
Kent Hagerman4f355f52020-03-30 16:01:33 -0400994
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400995 // update in db
Kent Hagermanf5a67352020-04-30 15:15:26 -0400996 if err := agent.dbProxy.Set(ctx, agent.deviceID, device); err != nil {
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400997 agent.requestQueue.RequestComplete()
Kent Hagerman4f355f52020-03-30 16:01:33 -0400998 return status.Errorf(codes.Internal, "failed-update-device:%s: %s", agent.deviceID, err)
Mahir Gunyelb5851672019-07-24 10:46:26 +0300999 }
divyadesaicb8b59d2020-08-18 09:55:47 +00001000 logger.Debugw(ctx, "updated-device-in-store", log.Fields{"device-id: ": agent.deviceID})
Mahir Gunyelb5851672019-07-24 10:46:26 +03001001
Kent Hagerman6031aad2020-07-29 16:36:33 -04001002 prevDevice := agent.device
Kent Hagermanf6db9f12020-07-22 17:16:19 -04001003 // update the device
khenaidoo0db4c812020-05-27 15:27:30 -04001004 agent.device = device
Mahir Gunyelb0343bf2021-05-11 14:14:26 -07001005 //If any of the states has chenged, send the change event.
1006 if prevDevice.OperStatus != device.OperStatus || prevDevice.ConnectStatus != device.ConnectStatus || prevDevice.AdminState != device.AdminState {
1007 _ = agent.deviceMgr.Agent.SendDeviceStateChangeEvent(ctx, prevDevice.OperStatus, prevDevice.ConnectStatus, prevDevice.AdminState, device, time.Now().Unix())
1008 }
Maninder0aabf0c2021-03-17 14:55:14 +05301009 deviceTransientState := agent.getTransientState()
1010
Kent Hagermanf6db9f12020-07-22 17:16:19 -04001011 // release lock before processing transition
1012 agent.requestQueue.RequestComplete()
Himani Chawlab4c25912020-11-12 17:16:38 +05301013 subCtx := coreutils.WithSpanAndRPCMetadataFromContext(ctx)
Kent Hagermanf6db9f12020-07-22 17:16:19 -04001014
Himani Chawlab4c25912020-11-12 17:16:38 +05301015 if err := agent.deviceMgr.stateTransitions.ProcessTransition(subCtx,
Maninder0aabf0c2021-03-17 14:55:14 +05301016 device, prevDevice, deviceTransientState, deviceTransientState); err != nil {
Himani Chawlab4c25912020-11-12 17:16:38 +05301017 logger.Errorw(ctx, "failed-process-transition", log.Fields{"device-id": device.Id, "previous-admin-state": prevDevice.AdminState, "current-admin-state": device.AdminState})
1018 // Sending RPC EVENT here
1019 rpce := agent.deviceMgr.NewRPCEvent(ctx, agent.deviceID, err.Error(), nil)
Himani Chawla606a4f02021-03-23 19:45:58 +05301020 agent.deviceMgr.SendRPCEvent(ctx, "RPC_ERROR_RAISE_EVENT", rpce, voltha.EventCategory_COMMUNICATION,
1021 nil, time.Now().Unix())
Himani Chawlab4c25912020-11-12 17:16:38 +05301022
Kent Hagermanf6db9f12020-07-22 17:16:19 -04001023 }
Mahir Gunyelb5851672019-07-24 10:46:26 +03001024 return nil
1025}
Mahir Gunyelfdee9212019-10-16 16:52:21 -07001026
Himani Chawla2ba1c9c2020-10-07 13:19:03 +05301027// This function updates the device transient in the DB through loader, releases the device lock, and runs any state transitions.
1028// The calling function MUST hold the device lock. The caller MUST NOT modify the device after this is called.
1029func (agent *Agent) updateDeviceWithTransientStateAndReleaseLock(ctx context.Context, device *voltha.Device,
khenaidood948f772021-08-11 17:49:24 -04001030 transientState, prevTransientState core.DeviceTransientState_Types) error {
Himani Chawla2ba1c9c2020-10-07 13:19:03 +05301031 // fail early if this agent is no longer valid
1032 if agent.stopped {
1033 agent.requestQueue.RequestComplete()
1034 return errors.New("device-agent-stopped")
1035 }
1036 //update device TransientState
1037 if err := agent.updateTransientState(ctx, transientState); err != nil {
1038 agent.requestQueue.RequestComplete()
1039 return err
1040 }
1041 // update in db
1042 if err := agent.dbProxy.Set(ctx, agent.deviceID, device); err != nil {
1043 //Reverting TransientState update
khenaidood948f772021-08-11 17:49:24 -04001044 if errTransient := agent.updateTransientState(ctx, prevTransientState); errTransient != nil {
1045 logger.Errorw(ctx, "failed-to-revert-transient-state-update-on-error", log.Fields{"device-id": device.Id,
1046 "previous-transient-state": prevTransientState, "current-transient-state": transientState, "error": errTransient})
1047 }
Himani Chawla2ba1c9c2020-10-07 13:19:03 +05301048 agent.requestQueue.RequestComplete()
1049 return status.Errorf(codes.Internal, "failed-update-device:%s: %s", agent.deviceID, err)
1050 }
1051
1052 logger.Debugw(ctx, "updated-device-in-store", log.Fields{"device-id: ": agent.deviceID})
1053
1054 prevDevice := agent.device
1055 // update the device
1056 agent.device = device
Mahir Gunyelb0343bf2021-05-11 14:14:26 -07001057 //If any of the states has chenged, send the change event.
1058 if prevDevice.OperStatus != device.OperStatus || prevDevice.ConnectStatus != device.ConnectStatus || prevDevice.AdminState != device.AdminState {
1059 _ = agent.deviceMgr.Agent.SendDeviceStateChangeEvent(ctx, prevDevice.OperStatus, prevDevice.ConnectStatus, prevDevice.AdminState, device, time.Now().Unix())
1060 }
Himani Chawla2ba1c9c2020-10-07 13:19:03 +05301061
1062 // release lock before processing transition
1063 agent.requestQueue.RequestComplete()
nikesh.krishnan95142d52023-02-24 15:32:11 +05301064
1065 if err := agent.deviceMgr.stateTransitions.ProcessTransition(ctx,
1066 device, prevDevice, transientState, prevTransientState); err != nil {
1067 logger.Errorw(ctx, "failed-process-transition", log.Fields{"device-id": device.Id, "previous-admin-state": prevDevice.AdminState, "current-admin-state": device.AdminState})
1068 // Sending RPC EVENT here
1069 rpce := agent.deviceMgr.NewRPCEvent(ctx, agent.deviceID, err.Error(), nil)
1070 go agent.deviceMgr.SendRPCEvent(ctx, "RPC_ERROR_RAISE_EVENT", rpce, voltha.EventCategory_COMMUNICATION,
1071 nil, time.Now().Unix())
1072 }
1073
Himani Chawla2ba1c9c2020-10-07 13:19:03 +05301074 return nil
1075}
Kent Hagerman2b216042020-04-03 18:28:56 -04001076func (agent *Agent) updateDeviceReason(ctx context.Context, reason string) error {
khenaidood948f772021-08-11 17:49:24 -04001077 logger.Debugw(ctx, "update-device-reason", log.Fields{"device-id": agent.deviceID, "reason": reason})
1078
1079 var err error
1080 var desc string
1081 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
1082 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
1083
1084 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
khenaidoo442e7c72020-03-10 16:13:48 -04001085 return err
1086 }
Maninder9a1bc0d2020-10-26 11:34:02 +05301087
Kent Hagermanf6db9f12020-07-22 17:16:19 -04001088 cloned := agent.cloneDeviceWithoutLock()
npujar1d86a522019-11-14 17:11:16 +05301089 cloned.Reason = reason
khenaidood948f772021-08-11 17:49:24 -04001090 if err = agent.updateDeviceAndReleaseLock(ctx, cloned); err == nil {
1091 requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
Maninder9a1bc0d2020-10-26 11:34:02 +05301092 }
khenaidood948f772021-08-11 17:49:24 -04001093 return err
Mahir Gunyelfdee9212019-10-16 16:52:21 -07001094}
kesavandbc2d1622020-01-21 00:42:01 -05001095
Kent Hagerman2b216042020-04-03 18:28:56 -04001096func (agent *Agent) ChildDeviceLost(ctx context.Context, device *voltha.Device) error {
Himani Chawlab4c25912020-11-12 17:16:38 +05301097 logger.Debugw(ctx, "child-device-lost", log.Fields{"child-device-id": device.Id, "parent-device-id": agent.deviceID})
Chaitrashree G S543df3e2020-02-24 22:36:54 -05001098
khenaidood948f772021-08-11 17:49:24 -04001099 var err error
1100 var desc string
1101 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
1102 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
1103
Kent Hagerman2a07b862020-06-19 15:23:07 -04001104 // Remove the associated peer ports on the parent device
1105 for portID := range agent.portLoader.ListIDs() {
1106 if portHandle, have := agent.portLoader.Lock(portID); have {
1107 oldPort := portHandle.GetReadOnly()
1108 updatedPeers := make([]*voltha.Port_PeerPort, 0)
1109 for _, peerPort := range oldPort.Peers {
1110 if peerPort.DeviceId != device.Id {
1111 updatedPeers = append(updatedPeers, peerPort)
1112 }
khenaidoo442e7c72020-03-10 16:13:48 -04001113 }
Kent Hagerman2a07b862020-06-19 15:23:07 -04001114 newPort := *oldPort
1115 newPort.Peers = updatedPeers
1116 if err := portHandle.Update(ctx, &newPort); err != nil {
1117 portHandle.Unlock()
1118 return nil
1119 }
1120 portHandle.Unlock()
khenaidoo442e7c72020-03-10 16:13:48 -04001121 }
Chaitrashree G S543df3e2020-02-24 22:36:54 -05001122 }
1123
khenaidoo442e7c72020-03-10 16:13:48 -04001124 //send request to adapter
khenaidood948f772021-08-11 17:49:24 -04001125 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
khenaidoo442e7c72020-03-10 16:13:48 -04001126 if err != nil {
khenaidood948f772021-08-11 17:49:24 -04001127 logger.Errorw(ctx, "grpc-client-nil",
1128 log.Fields{
1129 "error": err,
1130 "device-id": agent.deviceID,
1131 "device-type": agent.deviceType,
1132 "adapter-endpoint": device.AdapterEndpoint,
1133 })
khenaidoo442e7c72020-03-10 16:13:48 -04001134 return err
Chaitrashree G S543df3e2020-02-24 22:36:54 -05001135 }
khenaidood948f772021-08-11 17:49:24 -04001136 subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
1137 requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS
nikesh.krishnan95142d52023-02-24 15:32:11 +05301138
1139 defer cancel()
1140 _, err = client.ChildDeviceLost(subCtx, device)
1141 if err == nil {
1142 agent.onSuccess(subCtx, nil, nil, true)
1143 } else {
1144 agent.onFailure(subCtx, err, nil, nil, true)
1145 }
1146
Chaitrashree G S543df3e2020-02-24 22:36:54 -05001147 return nil
Chaitrashree G S543df3e2020-02-24 22:36:54 -05001148}
onkarkundargi87285252020-01-27 11:34:52 +05301149
khenaidoo9beaaf12021-10-19 17:32:01 -04001150func (agent *Agent) startOmciTest(ctx context.Context, omcitestrequest *omci.OmciTestRequest) (*omci.TestResponse, error) {
khenaidood948f772021-08-11 17:49:24 -04001151 var err error
1152 var desc string
1153 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
1154 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
onkarkundargi87285252020-01-27 11:34:52 +05301155
khenaidood948f772021-08-11 17:49:24 -04001156 // OMCI test may be performed on a pre-provisioned device. If a device is in that state both its device type and endpoint
1157 // may not have been set yet.
1158 // First check if we need to update the type or endpoint
1159 cloned, err := agent.getDeviceReadOnly(ctx)
onkarkundargi87285252020-01-27 11:34:52 +05301160 if err != nil {
1161 return nil, err
1162 }
khenaidood948f772021-08-11 17:49:24 -04001163 if cloned.Type == "" || cloned.AdapterEndpoint == "" {
1164 if err = agent.updateDeviceTypeAndEndpoint(ctx); err != nil {
1165 return nil, err
1166 }
1167 cloned, err = agent.getDeviceReadOnly(ctx)
1168 if err != nil {
1169 return nil, err
1170 }
onkarkundargi87285252020-01-27 11:34:52 +05301171 }
1172
khenaidood948f772021-08-11 17:49:24 -04001173 // Send request to the adapter
1174 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
1175 if err != nil {
1176 logger.Errorw(ctx, "grpc-client-nil",
1177 log.Fields{
1178 "error": err,
1179 "device-id": agent.deviceID,
1180 "device-type": agent.deviceType,
1181 "adapter-endpoint": cloned.AdapterEndpoint,
1182 })
1183 return nil, err
onkarkundargi87285252020-01-27 11:34:52 +05301184 }
khenaidood948f772021-08-11 17:49:24 -04001185
khenaidoo9beaaf12021-10-19 17:32:01 -04001186 res, err := client.StartOmciTest(ctx, &ca.OMCITest{
khenaidood948f772021-08-11 17:49:24 -04001187 Device: cloned,
1188 Request: omcitestrequest,
1189 })
1190 if err == nil {
1191 requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
1192 }
1193 return res, err
onkarkundargi87285252020-01-27 11:34:52 +05301194}
Dinesh Belwalkarc1129f12020-02-27 10:41:33 -08001195
khenaidoo9beaaf12021-10-19 17:32:01 -04001196func (agent *Agent) getExtValue(ctx context.Context, pdevice *voltha.Device, cdevice *voltha.Device, valueparam *extension.ValueSpecifier) (*extension.ReturnValues, error) {
Himani Chawlab4c25912020-11-12 17:16:38 +05301197 logger.Debugw(ctx, "get-ext-value", log.Fields{"device-id": agent.deviceID, "onu-id": valueparam.Id, "value-type": valueparam.Value})
khenaidood948f772021-08-11 17:49:24 -04001198 var err error
1199 var desc string
1200 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
1201 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
1202
1203 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
Dinesh Belwalkarc1129f12020-02-27 10:41:33 -08001204 return nil, err
1205 }
1206
khenaidood948f772021-08-11 17:49:24 -04001207 //send request to adapter synchronously
1208 client, err := agent.adapterMgr.GetAdapterClient(ctx, pdevice.AdapterEndpoint)
Dinesh Belwalkarc1129f12020-02-27 10:41:33 -08001209 if err != nil {
khenaidood948f772021-08-11 17:49:24 -04001210 logger.Errorw(ctx, "grpc-client-nil",
1211 log.Fields{
1212 "error": err,
1213 "device-id": agent.deviceID,
1214 "device-type": agent.deviceType,
1215 "adapter-endpoint": pdevice.AdapterEndpoint,
1216 })
1217 agent.requestQueue.RequestComplete()
Dinesh Belwalkarc1129f12020-02-27 10:41:33 -08001218 return nil, err
1219 }
1220
khenaidood948f772021-08-11 17:49:24 -04001221 // Release lock before sending to adapter
1222 agent.requestQueue.RequestComplete()
Dinesh Belwalkarc1129f12020-02-27 10:41:33 -08001223
khenaidoo9beaaf12021-10-19 17:32:01 -04001224 retVal, err := client.GetExtValue(ctx, &ca.GetExtValueMessage{
khenaidood948f772021-08-11 17:49:24 -04001225 ParentDevice: pdevice,
1226 ChildDevice: cdevice,
1227 ValueType: valueparam.Value,
1228 })
1229 if err == nil {
1230 requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
Dinesh Belwalkarc1129f12020-02-27 10:41:33 -08001231 }
khenaidood948f772021-08-11 17:49:24 -04001232 return retVal, err
Dinesh Belwalkarc1129f12020-02-27 10:41:33 -08001233}
dpaul62686312020-06-23 14:17:36 +05301234
khenaidoo9beaaf12021-10-19 17:32:01 -04001235func (agent *Agent) setExtValue(ctx context.Context, device *voltha.Device, value *extension.ValueSet) (*empty.Empty, error) {
Himani Chawlab4c25912020-11-12 17:16:38 +05301236 logger.Debugw(ctx, "set-ext-value", log.Fields{"device-id": value.Id})
khenaidood948f772021-08-11 17:49:24 -04001237
1238 var err error
1239 var desc string
1240 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
1241 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
1242
1243 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
dpaul62686312020-06-23 14:17:36 +05301244 return nil, err
1245 }
1246
1247 //send request to adapter
khenaidood948f772021-08-11 17:49:24 -04001248 //send request to adapter synchronously
1249 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
dpaul62686312020-06-23 14:17:36 +05301250 if err != nil {
khenaidood948f772021-08-11 17:49:24 -04001251 logger.Errorw(ctx, "grpc-client-nil",
1252 log.Fields{
1253 "error": err,
1254 "device-id": agent.deviceID,
1255 "device-type": agent.deviceType,
1256 "adapter-endpoint": device.AdapterEndpoint,
1257 })
1258 agent.requestQueue.RequestComplete()
dpaul62686312020-06-23 14:17:36 +05301259 return nil, err
1260 }
khenaidood948f772021-08-11 17:49:24 -04001261 // Release lock before sending request to adapter
1262 agent.requestQueue.RequestComplete()
dpaul62686312020-06-23 14:17:36 +05301263
khenaidoo9beaaf12021-10-19 17:32:01 -04001264 retVal, err := client.SetExtValue(ctx, &ca.SetExtValueMessage{
khenaidood948f772021-08-11 17:49:24 -04001265 Device: device,
1266 Value: value,
1267 })
1268 if err == nil {
1269 requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
dpaul62686312020-06-23 14:17:36 +05301270 }
khenaidood948f772021-08-11 17:49:24 -04001271 return retVal, err
dpaul62686312020-06-23 14:17:36 +05301272}
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301273
1274func (agent *Agent) getSingleValue(ctx context.Context, request *extension.SingleGetValueRequest) (*extension.SingleGetValueResponse, error) {
Himani Chawlab4c25912020-11-12 17:16:38 +05301275 logger.Debugw(ctx, "get-single-value", log.Fields{"device-id": request.TargetId})
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301276
khenaidood948f772021-08-11 17:49:24 -04001277 var err error
1278 var desc string
1279 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
1280 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
1281
1282 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301283 return nil, err
1284 }
1285
1286 cloned := agent.cloneDeviceWithoutLock()
1287
1288 //send request to adapter
khenaidood948f772021-08-11 17:49:24 -04001289 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301290 if err != nil {
khenaidood948f772021-08-11 17:49:24 -04001291 logger.Errorw(ctx, "grpc-client-nil",
1292 log.Fields{
1293 "error": err,
1294 "device-id": cloned.Id,
1295 "adapter-endpoint": cloned.AdapterEndpoint,
1296 })
1297 agent.requestQueue.RequestComplete()
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301298 return nil, err
1299 }
khenaidood948f772021-08-11 17:49:24 -04001300 // Release lock before sending request to adapter
1301 agent.requestQueue.RequestComplete()
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301302
khenaidood948f772021-08-11 17:49:24 -04001303 resp, err := client.GetSingleValue(ctx, request)
1304 if err == nil {
1305 requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301306 }
khenaidood948f772021-08-11 17:49:24 -04001307 return resp, err
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301308}
1309
1310func (agent *Agent) setSingleValue(ctx context.Context, request *extension.SingleSetValueRequest) (*extension.SingleSetValueResponse, error) {
Himani Chawlab4c25912020-11-12 17:16:38 +05301311 logger.Debugw(ctx, "set-single-value", log.Fields{"device-id": request.TargetId})
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301312
khenaidood948f772021-08-11 17:49:24 -04001313 var err error
1314 var desc string
1315 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
1316 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
1317
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301318 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
1319 return nil, err
1320 }
1321
1322 cloned := agent.cloneDeviceWithoutLock()
1323
1324 //send request to adapter
khenaidood948f772021-08-11 17:49:24 -04001325 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301326 if err != nil {
khenaidood948f772021-08-11 17:49:24 -04001327 logger.Errorw(ctx, "grpc-client-nil",
1328 log.Fields{
1329 "error": err,
1330 "device-id": agent.deviceID,
1331 "device-type": agent.deviceType,
1332 "adapter-endpoint": cloned.AdapterEndpoint,
1333 })
1334 agent.requestQueue.RequestComplete()
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301335 return nil, err
1336 }
khenaidood948f772021-08-11 17:49:24 -04001337 // Release lock before sending request to adapter
1338 agent.requestQueue.RequestComplete()
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301339
khenaidood948f772021-08-11 17:49:24 -04001340 resp, err := client.SetSingleValue(ctx, request)
1341 if err == nil {
1342 requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301343 }
khenaidood948f772021-08-11 17:49:24 -04001344 return resp, err
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301345}
Maninder0aabf0c2021-03-17 14:55:14 +05301346
Maninder2195ccc2021-06-23 20:23:01 +05301347func (agent *Agent) proceedWithRequest(device *voltha.Device) bool {
1348 return !agent.isDeletionInProgress() && !agent.isInReconcileState(device)
Maninder0aabf0c2021-03-17 14:55:14 +05301349}
1350
1351func (agent *Agent) stopReconcile() {
1352 agent.stopReconcilingMutex.Lock()
1353 if agent.stopReconciling != nil {
1354 agent.stopReconciling <- 0
1355 }
1356 agent.stopReconcilingMutex.Unlock()
1357}
1358
khenaidood948f772021-08-11 17:49:24 -04001359// abortAllProcessing is invoked when an adapter managing this device is restarted
1360func (agent *Agent) abortAllProcessing(ctx context.Context) error {
1361 logger.Infow(ctx, "aborting-current-running-requests", log.Fields{"device-id": agent.deviceID})
1362 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
1363 return err
1364 }
khenaidoo25057da2021-12-08 14:40:45 -05001365 logger.Infow(ctx, "aborting-current-running-requests-after-wait", log.Fields{"device-id": agent.deviceID})
1366
khenaidood948f772021-08-11 17:49:24 -04001367 defer agent.requestQueue.RequestComplete()
1368
1369 // If any reconciling is in progress just abort it. The adapter is gone.
1370 agent.stopReconcile()
1371
khenaidoo25057da2021-12-08 14:40:45 -05001372 logger.Infow(ctx, "aborting-current-running-requests-after-sendstop", log.Fields{"device-id": agent.deviceID})
1373
khenaidood948f772021-08-11 17:49:24 -04001374 // Update the Core device transient state accordingly
1375 var updatedState core.DeviceTransientState_Types
1376 switch agent.getTransientState() {
1377 case core.DeviceTransientState_RECONCILE_IN_PROGRESS:
1378 updatedState = core.DeviceTransientState_NONE
1379 case core.DeviceTransientState_FORCE_DELETING:
1380 updatedState = core.DeviceTransientState_DELETE_FAILED
1381 case core.DeviceTransientState_DELETING_FROM_ADAPTER:
1382 updatedState = core.DeviceTransientState_DELETE_FAILED
khenaidoo68a5e0c2021-11-06 13:08:03 -04001383 case core.DeviceTransientState_DELETE_FAILED:
1384 // do not change state
1385 return nil
khenaidood948f772021-08-11 17:49:24 -04001386 default:
1387 updatedState = core.DeviceTransientState_NONE
1388 }
1389 if err := agent.updateTransientState(ctx, updatedState); err != nil {
1390 logger.Errorf(ctx, "transient-state-update-failed", log.Fields{"error": err})
1391 return err
1392 }
1393 return nil
1394}
1395
khenaidoo68a5e0c2021-11-06 13:08:03 -04001396func (agent *Agent) DeleteDevicePostAdapterRestart(ctx context.Context) error {
1397 logger.Debugw(ctx, "delete-post-restart", log.Fields{"device-id": agent.deviceID})
1398 ctx = utils.WithNewSpanAndRPCMetadataContext(ctx, "DelteDevicePostAdapterRestart")
1399
1400 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
1401 return err
1402 }
1403
1404 device := agent.getDeviceReadOnlyWithoutLock()
1405 if device.AdminState == voltha.AdminState_PREPROVISIONED {
1406 logger.Debugw(ctx, "device-in-preprovisioning-state-reconcile-not-needed", log.Fields{"device-id": device.Id})
1407 agent.requestQueue.RequestComplete()
1408 return nil
1409 }
1410 // Change device transient state to FORCE_DELETING
1411 if err := agent.updateTransientState(ctx, core.DeviceTransientState_FORCE_DELETING); err != nil {
1412 logger.Errorw(ctx, "failure-updating-transient-state", log.Fields{"error": err, "device-id": agent.deviceID})
1413 agent.requestQueue.RequestComplete()
1414 return err
1415 }
1416
1417 // Ensure we have a valid grpc client available as we have just restarted
1418 deleteBackoff := backoff.NewExponentialBackOff()
1419 deleteBackoff.InitialInterval = agent.config.BackoffRetryInitialInterval
1420 deleteBackoff.MaxElapsedTime = agent.config.BackoffRetryMaxElapsedTime
1421 deleteBackoff.MaxInterval = agent.config.BackoffRetryMaxInterval
1422 var backoffTimer *time.Timer
1423 var err error
1424 var client adapter_service.AdapterServiceClient
1425retry:
1426 for {
1427 client, err = agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
1428 if err == nil {
1429 break retry
1430 }
1431 duration := deleteBackoff.NextBackOff()
1432 if duration == backoff.Stop {
1433 deleteBackoff.Reset()
1434 duration = deleteBackoff.NextBackOff()
1435 }
1436 backoffTimer = time.NewTimer(duration)
1437 select {
1438 case <-backoffTimer.C:
1439 logger.Debugw(ctx, "backoff-timer-expires", log.Fields{"device-id": agent.deviceID})
1440 case <-ctx.Done():
1441 err = ctx.Err()
1442 break retry
1443 }
1444 }
1445 if backoffTimer != nil && !backoffTimer.Stop() {
1446 select {
1447 case <-backoffTimer.C:
1448 default:
1449 }
1450 }
1451 if err != nil || client == nil {
1452 agent.requestQueue.RequestComplete()
1453 return err
1454 }
1455
1456 // Release the device lock to allow for device state update, if any
1457 agent.requestQueue.RequestComplete()
1458
1459 // Send the delete request to the adapter
1460 subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
1461 defer cancel()
khenaidoo6cd8e862021-11-12 16:33:43 -05001462 _, err = client.DeleteDevice(subCtx, device)
1463 agent.onForceDeleteResponse(subCtx, nil, nil, err)
1464 return err
khenaidoo68a5e0c2021-11-06 13:08:03 -04001465}
1466
khenaidood948f772021-08-11 17:49:24 -04001467func (agent *Agent) ReconcileDevice(ctx context.Context) {
khenaidoo68a5e0c2021-11-06 13:08:03 -04001468 // Do not reconcile if the device was in DELETE_FAILED transient state. Just invoke the force delete on that device.
1469 state := agent.getTransientState()
1470 logger.Debugw(ctx, "starting-reconcile", log.Fields{"device-id": agent.deviceID, "state": state})
1471 if agent.getTransientState() == core.DeviceTransientState_DELETE_FAILED {
1472 if err := agent.DeleteDevicePostAdapterRestart(ctx); err != nil {
1473 logger.Errorw(ctx, "delete-post-restart-failed", log.Fields{"error": err, "device-id": agent.deviceID})
1474 }
1475 return
1476 }
1477
khenaidood948f772021-08-11 17:49:24 -04001478 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
Maninder0aabf0c2021-03-17 14:55:14 +05301479 var desc string
Maninder0aabf0c2021-03-17 14:55:14 +05301480
1481 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
khenaidood948f772021-08-11 17:49:24 -04001482 agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc)
1483 return
1484 }
1485
1486 device := agent.getDeviceReadOnlyWithoutLock()
1487 if device.AdminState == voltha.AdminState_PREPROVISIONED {
1488 agent.requestQueue.RequestComplete()
1489 logger.Debugw(ctx, "device-in-preprovisioning-state-reconcile-not-needed", log.Fields{"device-id": device.Id})
Maninder0aabf0c2021-03-17 14:55:14 +05301490 return
1491 }
1492
nikesh.krishnanc8beca52023-12-07 13:45:03 +05301493 if agent.isDeletionInProgress() {
Maninder0aabf0c2021-03-17 14:55:14 +05301494 agent.requestQueue.RequestComplete()
nikesh.krishnanc8beca52023-12-07 13:45:03 +05301495 err := fmt.Errorf("cannot complete operation as device deletion is in progress device : %s", device.Id)
khenaidood948f772021-08-11 17:49:24 -04001496 logger.Errorw(ctx, "reconcile-failed", log.Fields{"error": err})
1497 agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc)
Maninder0aabf0c2021-03-17 14:55:14 +05301498 return
1499 }
1500
1501 //set transient state to RECONCILE IN PROGRESS
khenaidood948f772021-08-11 17:49:24 -04001502 err := agent.updateTransientState(ctx, core.DeviceTransientState_RECONCILE_IN_PROGRESS)
Maninder0aabf0c2021-03-17 14:55:14 +05301503 if err != nil {
1504 agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -04001505 logger.Errorw(ctx, "setting-transient-state-failed", log.Fields{"error": err})
1506 agent.logDeviceUpdate(ctx, nil, nil, requestStatus, nil, desc)
Maninder0aabf0c2021-03-17 14:55:14 +05301507 return
1508 }
1509
Maninder0aabf0c2021-03-17 14:55:14 +05301510 reconcilingBackoff := backoff.NewExponentialBackOff()
1511 reconcilingBackoff.InitialInterval = agent.config.BackoffRetryInitialInterval
1512 reconcilingBackoff.MaxElapsedTime = agent.config.BackoffRetryMaxElapsedTime
1513 reconcilingBackoff.MaxInterval = agent.config.BackoffRetryMaxInterval
1514
1515 //making here to keep lifecycle of this channel within the scope of retryReconcile
1516 agent.stopReconcilingMutex.Lock()
khenaidoo25057da2021-12-08 14:40:45 -05001517 agent.stopReconciling = make(chan int, 1)
Maninder0aabf0c2021-03-17 14:55:14 +05301518 agent.stopReconcilingMutex.Unlock()
1519
David K. Bainbridge482e4422021-06-30 12:23:42 -07001520 // defined outside the retry loop so it can be cleaned
1521 // up when the loop breaks
1522 var backoffTimer *time.Timer
1523
1524retry:
Maninder0aabf0c2021-03-17 14:55:14 +05301525 for {
David K. Bainbridge482e4422021-06-30 12:23:42 -07001526 // If the operations state of the device is RECONCILING_FAILED then we do not
1527 // want to continue to attempt reconciliation.
1528 deviceRef := agent.getDeviceReadOnlyWithoutLock()
1529 if deviceRef.OperStatus == common.OperStatus_RECONCILING_FAILED {
1530 logger.Warnw(ctx, "reconciling-failed-halting-retries",
1531 log.Fields{"device-id": device.Id})
1532 agent.requestQueue.RequestComplete()
1533 break retry
1534 }
1535
Maninder0aabf0c2021-03-17 14:55:14 +05301536 // Use an exponential back off to prevent getting into a tight loop
1537 duration := reconcilingBackoff.NextBackOff()
1538 //This case should never occur in default case as max elapsed time for backoff is 0(by default) , so it will never return stop
1539 if duration == backoff.Stop {
1540 // If we reach a maximum then warn and reset the backoff
1541 // timer and keep attempting.
1542 logger.Warnw(ctx, "maximum-reconciling-backoff-reached--resetting-backoff-timer",
1543 log.Fields{"max-reconciling-backoff": reconcilingBackoff.MaxElapsedTime,
1544 "device-id": device.Id})
1545 reconcilingBackoff.Reset()
1546 duration = reconcilingBackoff.NextBackOff()
1547 }
1548
David K. Bainbridge482e4422021-06-30 12:23:42 -07001549 backoffTimer = time.NewTimer(duration)
Maninder0aabf0c2021-03-17 14:55:14 +05301550
khenaidood948f772021-08-11 17:49:24 -04001551 logger.Debugw(ctx, "retrying-reconciling", log.Fields{"deviceID": device.Id, "endpoint": device.AdapterEndpoint})
1552 // Release lock before sending request to adapter
Maninder0aabf0c2021-03-17 14:55:14 +05301553 agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -04001554
1555 // Send a reconcile request to the adapter.
1556 err := agent.sendReconcileRequestToAdapter(ctx, device)
khenaidoo6cd8e862021-11-12 16:33:43 -05001557
1558 // Check the transient state after a response from the adapter. If a device delete
1559 // request was issued due to a callback during that time and failed then just delete
1560 // the device and stop the reconcile loop and invoke the device deletion
1561 if agent.getTransientState() == core.DeviceTransientState_DELETE_FAILED {
1562 if dErr := agent.DeleteDevicePostAdapterRestart(ctx); dErr != nil {
1563 logger.Errorw(ctx, "delete-post-restart-failed", log.Fields{"error": dErr, "device-id": agent.deviceID})
1564 }
1565 break retry
1566 }
1567
khenaidood948f772021-08-11 17:49:24 -04001568 if errors.Is(err, errContextExpired) || errors.Is(err, errReconcileAborted) {
1569 logger.Errorw(ctx, "reconcile-aborted", log.Fields{"error": err})
1570 requestStatus = &common.OperationResp{Code: common.OperationResp_OperationReturnCode(common.OperStatus_FAILED)}
1571 desc = "aborted"
1572 agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc)
1573 break retry
1574 }
nikesh.krishnan9e422132023-12-16 00:36:47 +05301575 st, ok := status.FromError(err)
1576 if ok {
1577 // Decode the error code and error message
1578 errorCode := st.Code()
1579 if errorCode == codes.AlreadyExists {
1580 logger.Warnw(ctx, "device already reconciled", log.Fields{"error": err})
1581 err := agent.reconcilingCleanup(ctx)
1582 if err != nil {
1583 logger.Errorf(ctx, "error during reconcile cleanup", err.Error())
1584 }
1585 break retry
1586
1587 }
1588
1589 }
Maninder0aabf0c2021-03-17 14:55:14 +05301590 if err != nil {
khenaidood948f772021-08-11 17:49:24 -04001591 agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc)
Maninder0aabf0c2021-03-17 14:55:14 +05301592 <-backoffTimer.C
1593 // backoffTimer expired continue
1594 // Take lock back before retrying
1595 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
khenaidood948f772021-08-11 17:49:24 -04001596 agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc)
David K. Bainbridge482e4422021-06-30 12:23:42 -07001597 break retry
Maninder0aabf0c2021-03-17 14:55:14 +05301598 }
1599 continue
1600 }
khenaidood948f772021-08-11 17:49:24 -04001601 // Success
1602 requestStatus = &common.OperationResp{Code: common.OperationResp_OPERATION_SUCCESS}
1603 desc = "adapter-response"
1604 agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc)
1605 break retry
David K. Bainbridge482e4422021-06-30 12:23:42 -07001606 }
1607
khenaidoo25057da2021-12-08 14:40:45 -05001608 logger.Debugw(ctx, "reconcile-retry-ends", log.Fields{"adapter-endpoint": agent.adapterEndpoint})
1609
David K. Bainbridge482e4422021-06-30 12:23:42 -07001610 // Retry loop is broken, so stop any timers and drain the channel
1611 if backoffTimer != nil && !backoffTimer.Stop() {
1612
1613 // As per documentation and stack overflow when a timer is stopped its
1614 // channel should be drained. The issue is that Stop returns false
1615 // either if the timer has already been fired "OR" if the timer can be
1616 // stopped before being fired. This means that in some cases the
1617 // channel has already be emptied so attempting to read from it means
1618 // a blocked thread. To get around this use a select so if the
1619 // channel is already empty the default case hits and we are not
1620 // blocked.
1621 select {
1622 case <-backoffTimer.C:
1623 default:
Maninder0aabf0c2021-03-17 14:55:14 +05301624 }
1625 }
1626}
1627
khenaidood948f772021-08-11 17:49:24 -04001628func (agent *Agent) sendReconcileRequestToAdapter(ctx context.Context, device *voltha.Device) error {
1629 logger.Debugw(ctx, "sending-reconcile-to-adapter", log.Fields{"device-id": device.Id, "endpoint": agent.adapterEndpoint})
1630 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
1631 if err != nil {
1632 return err
1633 }
1634 adapterResponse := make(chan error)
1635 go func() {
1636 _, err := client.ReconcileDevice(ctx, device)
1637 adapterResponse <- err
1638 }()
Maninder0aabf0c2021-03-17 14:55:14 +05301639 select {
1640 // wait for response
khenaidood948f772021-08-11 17:49:24 -04001641 case err := <-adapterResponse:
1642 if err != nil {
1643 return err
Maninder0aabf0c2021-03-17 14:55:14 +05301644 }
Maninder0aabf0c2021-03-17 14:55:14 +05301645 //In case of success quit retrying and wait for adapter to reset operation state of device
1646 agent.stopReconcilingMutex.Lock()
1647 agent.stopReconciling = nil
1648 agent.stopReconcilingMutex.Unlock()
1649 return nil
1650
1651 //if reconciling need to be stopped
1652 case _, ok := <-agent.stopReconciling:
1653 agent.stopReconcilingMutex.Lock()
1654 agent.stopReconciling = nil
1655 agent.stopReconcilingMutex.Unlock()
1656 if !ok {
1657 //channel-closed
khenaidood948f772021-08-11 17:49:24 -04001658 return fmt.Errorf("reconcile channel closed:%w", errReconcileAborted)
Maninder0aabf0c2021-03-17 14:55:14 +05301659 }
khenaidood948f772021-08-11 17:49:24 -04001660 return fmt.Errorf("reconciling aborted:%w", errReconcileAborted)
1661 // Context expired
1662 case <-ctx.Done():
1663 return fmt.Errorf("context expired:%s :%w", ctx.Err(), errContextExpired)
Maninder0aabf0c2021-03-17 14:55:14 +05301664 }
Maninder0aabf0c2021-03-17 14:55:14 +05301665}
1666
1667func (agent *Agent) reconcilingCleanup(ctx context.Context) error {
1668 var desc string
khenaidood948f772021-08-11 17:49:24 -04001669 var err error
Maninder0aabf0c2021-03-17 14:55:14 +05301670 operStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
khenaidood948f772021-08-11 17:49:24 -04001671 defer func() { agent.logDeviceUpdate(ctx, nil, nil, operStatus, err, desc) }()
1672
1673 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
1674 desc = "reconcile-cleanup-failed"
Maninder0aabf0c2021-03-17 14:55:14 +05301675 return err
1676 }
1677 defer agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -04001678 err = agent.updateTransientState(ctx, core.DeviceTransientState_NONE)
Maninder0aabf0c2021-03-17 14:55:14 +05301679 if err != nil {
khenaidood948f772021-08-11 17:49:24 -04001680 logger.Errorf(ctx, "transient-state-update-failed", log.Fields{"error": err})
Maninder0aabf0c2021-03-17 14:55:14 +05301681 return err
1682 }
khenaidood948f772021-08-11 17:49:24 -04001683 operStatus.Code = common.OperationResp_OPERATION_SUCCESS
Maninder0aabf0c2021-03-17 14:55:14 +05301684 return nil
1685}
khenaidood948f772021-08-11 17:49:24 -04001686
1687func (agent *Agent) isAdapterConnectionUp(ctx context.Context) bool {
1688 c, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
1689 return c != nil && err == nil
1690}
1691
1692func (agent *Agent) canDeviceRequestProceed(ctx context.Context) error {
1693 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
1694 return err
1695 }
1696 defer agent.requestQueue.RequestComplete()
1697 if agent.proceedWithRequest(agent.device) {
1698 return nil
1699 }
1700 return fmt.Errorf("device-cannot-process-request-%s", agent.deviceID)
1701}