blob: d819faeb1e69de1b1a9e6514b50fe42b5e576cc9 [file] [log] [blame]
khenaidoob9203542018-09-17 22:56:37 -04001/*
2 * Copyright 2018-present Open Networking Foundation
3
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7
8 * http://www.apache.org/licenses/LICENSE-2.0
9
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
npujar1d86a522019-11-14 17:11:16 +053016
Kent Hagerman2b216042020-04-03 18:28:56 -040017package device
khenaidoob9203542018-09-17 22:56:37 -040018
19import (
20 "context"
Matteo Scandolo360605d2019-11-05 18:29:17 -080021 "encoding/hex"
Kent Hagerman4f355f52020-03-30 16:01:33 -040022 "errors"
khenaidoo3ab34882019-05-02 21:33:30 -040023 "fmt"
Mahir Gunyelb0343bf2021-05-11 14:14:26 -070024 "reflect"
25 "sync"
26 "time"
27
khenaidoo9beaaf12021-10-19 17:32:01 -040028 "github.com/opencord/voltha-protos/v5/go/adapter_service"
khenaidood948f772021-08-11 17:49:24 -040029 "github.com/opencord/voltha-protos/v5/go/core"
khenaidoo9beaaf12021-10-19 17:32:01 -040030 "github.com/opencord/voltha-protos/v5/go/omci"
khenaidood948f772021-08-11 17:49:24 -040031
Maninder0aabf0c2021-03-17 14:55:14 +053032 "github.com/cenkalti/backoff/v3"
Maninder9a1bc0d2020-10-26 11:34:02 +053033 "github.com/gogo/protobuf/proto"
Maninder9a1bc0d2020-10-26 11:34:02 +053034 "github.com/golang/protobuf/ptypes/empty"
Maninder0aabf0c2021-03-17 14:55:14 +053035 "github.com/opencord/voltha-go/rw_core/config"
khenaidoo68a5e0c2021-11-06 13:08:03 -040036 "github.com/opencord/voltha-go/rw_core/utils"
Maninder9a1bc0d2020-10-26 11:34:02 +053037 "google.golang.org/grpc/codes"
38 "google.golang.org/grpc/status"
Mahir Gunyeladdb66a2020-04-29 18:08:50 -070039
Himani Chawla2ba1c9c2020-10-07 13:19:03 +053040 "github.com/opencord/voltha-go/db/model"
Kent Hagerman2b216042020-04-03 18:28:56 -040041 "github.com/opencord/voltha-go/rw_core/core/adapter"
Mahir Gunyel03de0d32020-06-03 01:36:59 -070042 "github.com/opencord/voltha-go/rw_core/core/device/flow"
43 "github.com/opencord/voltha-go/rw_core/core/device/group"
Kent Hagerman2a07b862020-06-19 15:23:07 -040044 "github.com/opencord/voltha-go/rw_core/core/device/port"
Himani Chawla2ba1c9c2020-10-07 13:19:03 +053045 "github.com/opencord/voltha-go/rw_core/core/device/transientstate"
Scott Bakerb671a862019-10-24 10:53:40 -070046 coreutils "github.com/opencord/voltha-go/rw_core/utils"
khenaidood948f772021-08-11 17:49:24 -040047 "github.com/opencord/voltha-lib-go/v7/pkg/log"
48 "github.com/opencord/voltha-protos/v5/go/common"
khenaidoo9beaaf12021-10-19 17:32:01 -040049 ca "github.com/opencord/voltha-protos/v5/go/core_adapter"
khenaidood948f772021-08-11 17:49:24 -040050 "github.com/opencord/voltha-protos/v5/go/extension"
khenaidood948f772021-08-11 17:49:24 -040051 ofp "github.com/opencord/voltha-protos/v5/go/openflow_13"
52 "github.com/opencord/voltha-protos/v5/go/voltha"
khenaidoob9203542018-09-17 22:56:37 -040053)
54
khenaidood948f772021-08-11 17:49:24 -040055var errReconcileAborted = errors.New("reconcile aborted")
56var errContextExpired = errors.New("context expired")
khenaidoo68a5e0c2021-11-06 13:08:03 -040057var errNoConnection = errors.New("no connection")
khenaidood948f772021-08-11 17:49:24 -040058
Kent Hagerman2b216042020-04-03 18:28:56 -040059// Agent represents device agent attributes
60type Agent struct {
Maninder0aabf0c2021-03-17 14:55:14 +053061 deviceID string
62 parentID string
63 deviceType string
khenaidood948f772021-08-11 17:49:24 -040064 adapterEndpoint string
Maninder0aabf0c2021-03-17 14:55:14 +053065 isRootDevice bool
Maninder0aabf0c2021-03-17 14:55:14 +053066 adapterMgr *adapter.Manager
67 deviceMgr *Manager
68 dbProxy *model.Proxy
69 exitChannel chan int
70 device *voltha.Device
71 requestQueue *coreutils.RequestQueue
khenaidood948f772021-08-11 17:49:24 -040072 internalTimeout time.Duration
73 rpcTimeout time.Duration
Himani Chawla4b4bd252021-11-08 15:59:40 +053074 flowTimeout time.Duration
Maninder0aabf0c2021-03-17 14:55:14 +053075 startOnce sync.Once
76 stopOnce sync.Once
77 stopped bool
78 stopReconciling chan int
79 stopReconcilingMutex sync.RWMutex
80 config *config.RWCoreFlags
Mahir Gunyel03de0d32020-06-03 01:36:59 -070081
khenaidoo7585a962021-06-10 16:15:38 -040082 flowCache *flow.Cache
83 groupCache *group.Cache
Himani Chawla2ba1c9c2020-10-07 13:19:03 +053084 portLoader *port.Loader
85 transientStateLoader *transientstate.Loader
khenaidoob9203542018-09-17 22:56:37 -040086}
87
Kent Hagerman2b216042020-04-03 18:28:56 -040088//newAgent creates a new device agent. The device will be initialized when start() is called.
Himani Chawla4b4bd252021-11-08 15:59:40 +053089func newAgent(device *voltha.Device, deviceMgr *Manager, dbPath *model.Path, deviceProxy *model.Proxy, internalTimeout, rpcTimeout, flowTimeout time.Duration) *Agent {
Kent Hagerman2a07b862020-06-19 15:23:07 -040090 deviceID := device.Id
91 if deviceID == "" {
92 deviceID = coreutils.CreateDeviceID()
Stephane Barbarie1ab43272018-12-08 21:42:13 -050093 }
Scott Baker80678602019-11-14 16:57:36 -080094
Kent Hagerman2a07b862020-06-19 15:23:07 -040095 return &Agent{
Himani Chawla2ba1c9c2020-10-07 13:19:03 +053096 deviceID: deviceID,
Himani Chawla2ba1c9c2020-10-07 13:19:03 +053097 isRootDevice: device.Root,
98 parentID: device.ParentId,
99 deviceType: device.Type,
khenaidood948f772021-08-11 17:49:24 -0400100 adapterEndpoint: device.AdapterEndpoint,
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530101 deviceMgr: deviceMgr,
102 adapterMgr: deviceMgr.adapterMgr,
103 exitChannel: make(chan int, 1),
104 dbProxy: deviceProxy,
khenaidood948f772021-08-11 17:49:24 -0400105 internalTimeout: internalTimeout,
106 rpcTimeout: rpcTimeout,
Himani Chawla4b4bd252021-11-08 15:59:40 +0530107 flowTimeout: flowTimeout,
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530108 device: proto.Clone(device).(*voltha.Device),
109 requestQueue: coreutils.NewRequestQueue(),
Maninder0aabf0c2021-03-17 14:55:14 +0530110 config: deviceMgr.config,
khenaidoo7585a962021-06-10 16:15:38 -0400111 flowCache: flow.NewCache(),
112 groupCache: group.NewCache(),
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530113 portLoader: port.NewLoader(dbPath.SubPath("ports").Proxy(deviceID)),
114 transientStateLoader: transientstate.NewLoader(dbPath.SubPath("core").Proxy("transientstate"), deviceID),
Kent Hagerman2a07b862020-06-19 15:23:07 -0400115 }
khenaidoob9203542018-09-17 22:56:37 -0400116}
117
khenaidoo442e7c72020-03-10 16:13:48 -0400118// start() saves the device to the data model and registers for callbacks on that device if deviceToCreate!=nil.
119// Otherwise, it will load the data from the dB and setup the necessary callbacks and proxies. Returns the device that
Scott Baker80678602019-11-14 16:57:36 -0800120// was started.
khenaidoo7585a962021-06-10 16:15:38 -0400121func (agent *Agent) start(ctx context.Context, deviceExist bool, deviceToCreate *voltha.Device) (*voltha.Device, error) {
khenaidoo442e7c72020-03-10 16:13:48 -0400122 needToStart := false
123 if agent.startOnce.Do(func() { needToStart = true }); !needToStart {
Kent Hagermancba2f302020-07-28 13:37:36 -0400124 return agent.getDeviceReadOnly(ctx)
khenaidoo442e7c72020-03-10 16:13:48 -0400125 }
126 var startSucceeded bool
127 defer func() {
128 if !startSucceeded {
129 if err := agent.stop(ctx); err != nil {
Rohan Agrawal31f21802020-06-12 05:38:46 +0000130 logger.Errorw(ctx, "failed-to-cleanup-after-unsuccessful-start", log.Fields{"device-id": agent.deviceID, "error": err})
khenaidoo442e7c72020-03-10 16:13:48 -0400131 }
132 }
133 }()
khenaidoo7585a962021-06-10 16:15:38 -0400134 if deviceExist {
135 device := deviceToCreate
136 if device == nil {
137 // Load from dB
138 device = &voltha.Device{}
139 have, err := agent.dbProxy.Get(ctx, agent.deviceID, device)
140 if err != nil {
141 return nil, err
142 } else if !have {
143 return nil, status.Errorf(codes.NotFound, "device-%s", agent.deviceID)
144 }
khenaidood948f772021-08-11 17:49:24 -0400145 logger.Infow(ctx, "device-loaded-from-db", log.Fields{"device-id": agent.deviceID, "adapter-endpoint": device.AdapterEndpoint, "type": device.Type})
Thomas Lee Se5a44012019-11-07 20:32:24 +0530146 }
khenaidood948f772021-08-11 17:49:24 -0400147 agent.deviceType = device.Type
148 agent.adapterEndpoint = device.AdapterEndpoint
Kent Hagerman4f355f52020-03-30 16:01:33 -0400149 agent.device = proto.Clone(device).(*voltha.Device)
khenaidoo7585a962021-06-10 16:15:38 -0400150 // load the ports from KV to cache
Kent Hagerman2a07b862020-06-19 15:23:07 -0400151 agent.portLoader.Load(ctx)
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530152 agent.transientStateLoader.Load(ctx)
khenaidoo297cd252019-02-07 22:10:23 -0500153 } else {
Scott Baker80678602019-11-14 16:57:36 -0800154 // Create a new device
Maninder9a1bc0d2020-10-26 11:34:02 +0530155 var desc string
khenaidood948f772021-08-11 17:49:24 -0400156 var err error
Maninder9a1bc0d2020-10-26 11:34:02 +0530157 prevState := common.AdminState_UNKNOWN
158 currState := common.AdminState_UNKNOWN
khenaidood948f772021-08-11 17:49:24 -0400159 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
Maninder9a1bc0d2020-10-26 11:34:02 +0530160
khenaidood948f772021-08-11 17:49:24 -0400161 defer func() { agent.logDeviceUpdate(ctx, &prevState, &currState, requestStatus, err, desc) }()
Maninder9a1bc0d2020-10-26 11:34:02 +0530162
Kent Hagermanf5a67352020-04-30 15:15:26 -0400163 // Assumption is that AdminState, FlowGroups, and Flows are uninitialized since this
164 // is a new device, so populate them here before passing the device to ldProxy.Set.
Kent Hagerman2b216042020-04-03 18:28:56 -0400165 // agent.deviceId will also have been set during newAgent().
khenaidoo7585a962021-06-10 16:15:38 -0400166 device := (proto.Clone(deviceToCreate)).(*voltha.Device)
npujar1d86a522019-11-14 17:11:16 +0530167 device.Id = agent.deviceID
Scott Baker80678602019-11-14 16:57:36 -0800168 device.AdminState = voltha.AdminState_PREPROVISIONED
Maninder9a1bc0d2020-10-26 11:34:02 +0530169 currState = device.AdminState
Scott Baker80678602019-11-14 16:57:36 -0800170 if !deviceToCreate.GetRoot() && deviceToCreate.ProxyAddress != nil {
171 // Set the default vlan ID to the one specified by the parent adapter. It can be
172 // overwritten by the child adapter during a device update request
173 device.Vlan = deviceToCreate.ProxyAddress.ChannelId
174 }
175
khenaidood948f772021-08-11 17:49:24 -0400176 // Save the device to the model
177 if err = agent.dbProxy.Set(ctx, agent.deviceID, device); err != nil {
178 err = status.Errorf(codes.Aborted, "failed-adding-device-%s: %s", agent.deviceID, err)
179 return nil, err
khenaidoo297cd252019-02-07 22:10:23 -0500180 }
Mahir Gunyelb0343bf2021-05-11 14:14:26 -0700181 _ = agent.deviceMgr.Agent.SendDeviceStateChangeEvent(ctx, device.OperStatus, device.ConnectStatus, prevState, device, time.Now().Unix())
khenaidood948f772021-08-11 17:49:24 -0400182 requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
khenaidoo442e7c72020-03-10 16:13:48 -0400183 agent.device = device
khenaidoob9203542018-09-17 22:56:37 -0400184 }
khenaidoo442e7c72020-03-10 16:13:48 -0400185 startSucceeded = true
Rohan Agrawalcf12f202020-08-03 04:42:01 +0000186 log.EnrichSpan(ctx, log.Fields{"device-id": agent.deviceID})
Rohan Agrawal31f21802020-06-12 05:38:46 +0000187 logger.Debugw(ctx, "device-agent-started", log.Fields{"device-id": agent.deviceID})
khenaidoo442e7c72020-03-10 16:13:48 -0400188
Kent Hagermancba2f302020-07-28 13:37:36 -0400189 return agent.getDeviceReadOnly(ctx)
khenaidoob9203542018-09-17 22:56:37 -0400190}
191
khenaidoo4d4802d2018-10-04 21:59:49 -0400192// stop stops the device agent. Not much to do for now
Kent Hagerman2b216042020-04-03 18:28:56 -0400193func (agent *Agent) stop(ctx context.Context) error {
khenaidoo442e7c72020-03-10 16:13:48 -0400194 needToStop := false
195 if agent.stopOnce.Do(func() { needToStop = true }); !needToStop {
196 return nil
197 }
198 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
199 return err
200 }
201 defer agent.requestQueue.RequestComplete()
khenaidoo49085352020-01-13 19:15:43 -0500202
Himani Chawlab4c25912020-11-12 17:16:38 +0530203 logger.Infow(ctx, "stopping-device-agent", log.Fields{"device-id": agent.deviceID, "parent-id": agent.parentID})
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530204 // Remove the device transient loader
205 if err := agent.deleteTransientState(ctx); err != nil {
206 return err
207 }
khenaidoo0a822f92019-05-08 15:15:57 -0400208 // Remove the device from the KV store
Kent Hagermanf5a67352020-04-30 15:15:26 -0400209 if err := agent.dbProxy.Remove(ctx, agent.deviceID); err != nil {
khenaidoo442e7c72020-03-10 16:13:48 -0400210 return err
Thomas Lee Se5a44012019-11-07 20:32:24 +0530211 }
khenaidoo442e7c72020-03-10 16:13:48 -0400212
khenaidoo442e7c72020-03-10 16:13:48 -0400213 close(agent.exitChannel)
214
215 agent.stopped = true
216
Rohan Agrawal31f21802020-06-12 05:38:46 +0000217 logger.Infow(ctx, "device-agent-stopped", log.Fields{"device-id": agent.deviceID, "parent-id": agent.parentID})
khenaidoo442e7c72020-03-10 16:13:48 -0400218
219 return nil
khenaidoob9203542018-09-17 22:56:37 -0400220}
221
Scott Baker80678602019-11-14 16:57:36 -0800222// Load the most recent state from the KVStore for the device.
Kent Hagerman2b216042020-04-03 18:28:56 -0400223func (agent *Agent) reconcileWithKVStore(ctx context.Context) {
khenaidoo442e7c72020-03-10 16:13:48 -0400224 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
Rohan Agrawal31f21802020-06-12 05:38:46 +0000225 logger.Warnw(ctx, "request-aborted", log.Fields{"device-id": agent.deviceID, "error": err})
khenaidoo442e7c72020-03-10 16:13:48 -0400226 return
227 }
228 defer agent.requestQueue.RequestComplete()
Rohan Agrawal31f21802020-06-12 05:38:46 +0000229 logger.Debug(ctx, "reconciling-device-agent-devicetype")
Scott Baker80678602019-11-14 16:57:36 -0800230 // TODO: context timeout
Kent Hagerman4f355f52020-03-30 16:01:33 -0400231 device := &voltha.Device{}
Kent Hagermanf5a67352020-04-30 15:15:26 -0400232 if have, err := agent.dbProxy.Get(ctx, agent.deviceID, device); err != nil {
Rohan Agrawal31f21802020-06-12 05:38:46 +0000233 logger.Errorw(ctx, "kv-get-failed", log.Fields{"device-id": agent.deviceID, "error": err})
Thomas Lee Se5a44012019-11-07 20:32:24 +0530234 return
Kent Hagerman4f355f52020-03-30 16:01:33 -0400235 } else if !have {
236 return // not found in kv
Thomas Lee Se5a44012019-11-07 20:32:24 +0530237 }
Kent Hagerman4f355f52020-03-30 16:01:33 -0400238
khenaidood948f772021-08-11 17:49:24 -0400239 agent.deviceType = device.Type
Kent Hagerman4f355f52020-03-30 16:01:33 -0400240 agent.device = device
khenaidood948f772021-08-11 17:49:24 -0400241 agent.adapterEndpoint = device.AdapterEndpoint
Kent Hagerman2a07b862020-06-19 15:23:07 -0400242 agent.portLoader.Load(ctx)
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530243 agent.transientStateLoader.Load(ctx)
244
Rohan Agrawal31f21802020-06-12 05:38:46 +0000245 logger.Debugw(ctx, "reconciled-device-agent-devicetype", log.Fields{"device-id": agent.deviceID, "type": agent.deviceType})
Scott Baker80678602019-11-14 16:57:36 -0800246}
247
khenaidoo442e7c72020-03-10 16:13:48 -0400248// onSuccess is a common callback for scenarios where we receive a nil response following a request to an adapter
khenaidood948f772021-08-11 17:49:24 -0400249func (agent *Agent) onSuccess(ctx context.Context, prevState, currState *common.AdminState_Types, deviceUpdateLog bool) {
250 if deviceUpdateLog {
251 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_SUCCESS}
252 desc := "adapter-response"
253 agent.logDeviceUpdate(ctx, prevState, currState, requestStatus, nil, desc)
254 return
255 }
256 logger.Debugw(ctx, "successful-operation", log.Fields{"device-id": agent.deviceID, "rpc": coreutils.GetRPCMetadataFromContext(ctx)})
khenaidoo442e7c72020-03-10 16:13:48 -0400257}
258
259// onFailure is a common callback for scenarios where we receive an error response following a request to an adapter
260// and the only action required is to publish the failed result on kafka
khenaidood948f772021-08-11 17:49:24 -0400261func (agent *Agent) onFailure(ctx context.Context, err error, prevState, currState *common.AdminState_Types, deviceUpdateLog bool) {
262 // Send an event on kafka
263 rpce := agent.deviceMgr.NewRPCEvent(ctx, agent.deviceID, err.Error(), nil)
264 go agent.deviceMgr.SendRPCEvent(ctx, "RPC_ERROR_RAISE_EVENT", rpce,
265 voltha.EventCategory_COMMUNICATION, nil, time.Now().Unix())
khenaidoo442e7c72020-03-10 16:13:48 -0400266
khenaidood948f772021-08-11 17:49:24 -0400267 // Log the device update event
268 if deviceUpdateLog {
269 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
270 desc := "adapter-response"
271 agent.logDeviceUpdate(ctx, prevState, currState, requestStatus, err, desc)
272 return
khenaidoo442e7c72020-03-10 16:13:48 -0400273 }
khenaidood948f772021-08-11 17:49:24 -0400274 logger.Errorw(ctx, "failed-operation", log.Fields{"error": err, "device-id": agent.deviceID, "rpc": coreutils.GetRPCMetadataFromContext(ctx)})
khenaidoo442e7c72020-03-10 16:13:48 -0400275}
276
khenaidoo6cd8e862021-11-12 16:33:43 -0500277// onForceDeleteResponse is invoked following a force delete request to an adapter.
278func (agent *Agent) onForceDeleteResponse(ctx context.Context, prevState, currState *common.AdminState_Types, dErr error) {
279 // Log the status
280 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_SUCCESS}
281 if dErr != nil {
282 requestStatus = &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
283 }
284 agent.logDeviceUpdate(ctx, prevState, currState, requestStatus, dErr, "adapter-force-delete-response")
285
286 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
287 logger.Errorw(ctx, "failed-getting-device-request-lock", log.Fields{"device-id": agent.deviceID, "error": err})
288 }
289 previousDeviceTransientState := agent.getTransientState()
290 newDevice := agent.cloneDeviceWithoutLock()
291
292 // Even on a delete error response, cleaup the device in the core
293 requestStatus = &common.OperationResp{Code: common.OperationResp_OPERATION_SUCCESS}
294 err := agent.updateDeviceWithTransientStateAndReleaseLock(ctx, newDevice,
295 core.DeviceTransientState_DELETING_POST_ADAPTER_RESPONSE, previousDeviceTransientState)
296 if err != nil {
297 requestStatus = &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
298 }
299 agent.logDeviceUpdate(ctx, prevState, currState, requestStatus, err, "transient-state-update")
300}
301
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530302// onDeleteSuccess is a common callback for scenarios where we receive a nil response following a delete request
303// to an adapter.
khenaidood948f772021-08-11 17:49:24 -0400304func (agent *Agent) onDeleteSuccess(ctx context.Context, prevState, currState *common.AdminState_Types) {
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530305 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
khenaidood948f772021-08-11 17:49:24 -0400306 logger.Errorw(ctx, "delete-device-failure", log.Fields{"device-id": agent.deviceID, "error": err})
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530307 }
308 previousDeviceTransientState := agent.getTransientState()
309 newDevice := agent.cloneDeviceWithoutLock()
310 if err := agent.updateDeviceWithTransientStateAndReleaseLock(ctx, newDevice,
khenaidood948f772021-08-11 17:49:24 -0400311 core.DeviceTransientState_DELETING_POST_ADAPTER_RESPONSE, previousDeviceTransientState); err != nil {
312 logger.Errorw(ctx, "delete-device-failure", log.Fields{"device-id": agent.deviceID, "error": err})
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530313 }
khenaidood948f772021-08-11 17:49:24 -0400314 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_SUCCESS}
315 desc := "adapter-response"
316 agent.logDeviceUpdate(ctx, prevState, currState, requestStatus, nil, desc)
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530317}
318
319// onDeleteFailure is a common callback for scenarios where we receive an error response following a delete request
320// to an adapter and the only action required is to return the error response.
khenaidood948f772021-08-11 17:49:24 -0400321func (agent *Agent) onDeleteFailure(ctx context.Context, err error, prevState, currState *common.AdminState_Types) {
322 logger.Errorw(ctx, "rpc-failed", log.Fields{"rpc": coreutils.GetRPCMetadataFromContext(ctx), "device-id": agent.deviceID, "error": err})
323
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530324 //Only updating of transient state is required, no transition.
khenaidood948f772021-08-11 17:49:24 -0400325 if er := agent.updateTransientState(ctx, core.DeviceTransientState_DELETE_FAILED); er != nil {
326 logger.Errorw(ctx, "failed-to-update-transient-state-as-delete-failed", log.Fields{"device-id": agent.deviceID, "error": er})
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530327 }
khenaidood948f772021-08-11 17:49:24 -0400328 rpce := agent.deviceMgr.NewRPCEvent(ctx, agent.deviceID, err.Error(), nil)
329 go agent.deviceMgr.SendRPCEvent(ctx, "RPC_ERROR_RAISE_EVENT", rpce,
330 voltha.EventCategory_COMMUNICATION, nil, time.Now().Unix())
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530331
khenaidood948f772021-08-11 17:49:24 -0400332 // Log the device update event
333 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
334 desc := "adapter-response"
335 agent.logDeviceUpdate(ctx, prevState, currState, requestStatus, err, desc)
Maninder9a1bc0d2020-10-26 11:34:02 +0530336}
337
Kent Hagermancba2f302020-07-28 13:37:36 -0400338// getDeviceReadOnly returns a device which MUST NOT be modified, but is safe to keep forever.
339func (agent *Agent) getDeviceReadOnly(ctx context.Context) (*voltha.Device, error) {
khenaidoo442e7c72020-03-10 16:13:48 -0400340 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
341 return nil, err
342 }
343 defer agent.requestQueue.RequestComplete()
Kent Hagermancba2f302020-07-28 13:37:36 -0400344 return agent.device, nil
khenaidoo92e62c52018-10-03 14:02:54 -0400345}
346
Kent Hagermancba2f302020-07-28 13:37:36 -0400347// getDeviceReadOnlyWithoutLock returns a device which MUST NOT be modified, but is safe to keep forever. This is very efficient.
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400348// The device lock MUST be held by the caller.
Kent Hagermancba2f302020-07-28 13:37:36 -0400349func (agent *Agent) getDeviceReadOnlyWithoutLock() *voltha.Device {
khenaidoo0db4c812020-05-27 15:27:30 -0400350 return agent.device
khenaidoo92e62c52018-10-03 14:02:54 -0400351}
352
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400353// cloneDeviceWithoutLock returns a copy of the device which is safe to modify.
354// The device lock MUST be held by the caller.
355func (agent *Agent) cloneDeviceWithoutLock() *voltha.Device {
356 return proto.Clone(agent.device).(*voltha.Device)
357}
358
khenaidood948f772021-08-11 17:49:24 -0400359func (agent *Agent) updateDeviceTypeAndEndpoint(ctx context.Context) error {
360 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
361 return err
362 }
363 changed := false
364 cloned := agent.cloneDeviceWithoutLock()
365 if cloned.Type == "" {
366 adapterType, err := agent.adapterMgr.GetAdapterType(cloned.Type)
367 if err != nil {
368 agent.requestQueue.RequestComplete()
369 return err
370 }
371 cloned.Type = adapterType
372 changed = true
373 }
374
375 if cloned.AdapterEndpoint == "" {
376 var err error
377 if cloned.AdapterEndpoint, err = agent.adapterMgr.GetAdapterEndpoint(ctx, cloned.Id, cloned.Type); err != nil {
378 agent.requestQueue.RequestComplete()
379 return err
380 }
381 agent.adapterEndpoint = cloned.AdapterEndpoint
382 changed = true
383 }
384
385 if changed {
386 return agent.updateDeviceAndReleaseLock(ctx, cloned)
387 }
388 agent.requestQueue.RequestComplete()
389 return nil
390}
391
khenaidoo3ab34882019-05-02 21:33:30 -0400392// enableDevice activates a preprovisioned or a disable device
Kent Hagerman2b216042020-04-03 18:28:56 -0400393func (agent *Agent) enableDevice(ctx context.Context) error {
Maninder9a1bc0d2020-10-26 11:34:02 +0530394 //To preserve and use oldDevice state as prev state in new device
khenaidood948f772021-08-11 17:49:24 -0400395 var err error
Maninder9a1bc0d2020-10-26 11:34:02 +0530396 var desc string
khenaidood948f772021-08-11 17:49:24 -0400397 var prevAdminState, currAdminState common.AdminState_Types
398 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
Maninder9a1bc0d2020-10-26 11:34:02 +0530399
khenaidood948f772021-08-11 17:49:24 -0400400 defer func() { agent.logDeviceUpdate(ctx, &prevAdminState, &currAdminState, requestStatus, err, desc) }()
Maninder9a1bc0d2020-10-26 11:34:02 +0530401
khenaidood948f772021-08-11 17:49:24 -0400402 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
khenaidoo442e7c72020-03-10 16:13:48 -0400403 return err
404 }
Himani Chawlab4c25912020-11-12 17:16:38 +0530405 logger.Debugw(ctx, "enable-device", log.Fields{"device-id": agent.deviceID})
khenaidoo21d51152019-02-01 13:48:37 -0500406
Kent Hagermancba2f302020-07-28 13:37:36 -0400407 oldDevice := agent.getDeviceReadOnlyWithoutLock()
khenaidood948f772021-08-11 17:49:24 -0400408 prevAdminState = oldDevice.AdminState
Maninder9a1bc0d2020-10-26 11:34:02 +0530409
Maninder2195ccc2021-06-23 20:23:01 +0530410 if !agent.proceedWithRequest(oldDevice) {
411 agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -0400412 err = status.Errorf(codes.FailedPrecondition, "cannot complete operation as device deletion is in progress or reconciling is in progress/failed: %s", agent.deviceID)
413 return err
Maninder2195ccc2021-06-23 20:23:01 +0530414 }
Mahir Gunyel92dd1212021-10-22 11:42:56 -0700415 //vol-4275 TST meeting 08/04/2021: Let EnableDevice to be called again if device is in FAILED operational state,
416 //even the admin state is ENABLED.
417 if oldDevice.AdminState == voltha.AdminState_ENABLED && oldDevice.OperStatus != voltha.OperStatus_FAILED {
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400418 logger.Warnw(ctx, "device-already-enabled", log.Fields{"device-id": agent.deviceID})
419 agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -0400420 err = status.Errorf(codes.FailedPrecondition, fmt.Sprintf("cannot-enable-an-already-enabled-device: %s", oldDevice.Id))
Matteo Scandolod525ae32020-04-02 17:27:29 -0700421 return err
npujar1d86a522019-11-14 17:11:16 +0530422 }
423
khenaidood948f772021-08-11 17:49:24 -0400424 // Verify whether there is a device type that supports this device type
425 _, err = agent.adapterMgr.GetAdapterType(oldDevice.Type)
426 if err != nil {
427 agent.requestQueue.RequestComplete()
428 return err
429 }
430
431 // Update device adapter endpoint if not set. This is set once by the Core and use as is by the adapters. E.g if this is a
432 // child device then the parent adapter will use this device's adapter endpoint (set here) to communicate with it.
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400433 newDevice := agent.cloneDeviceWithoutLock()
khenaidood948f772021-08-11 17:49:24 -0400434 if newDevice.AdapterEndpoint == "" {
435 if newDevice.AdapterEndpoint, err = agent.adapterMgr.GetAdapterEndpoint(ctx, newDevice.Id, newDevice.Type); err != nil {
436 agent.requestQueue.RequestComplete()
437 return err
438 }
439 agent.adapterEndpoint = newDevice.AdapterEndpoint
440 }
npujar1d86a522019-11-14 17:11:16 +0530441
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400442 // Update the Admin State and set the operational state to activating before sending the request to the Adapters
443 newDevice.AdminState = voltha.AdminState_ENABLED
444 newDevice.OperStatus = voltha.OperStatus_ACTIVATING
Maninder9a1bc0d2020-10-26 11:34:02 +0530445
khenaidoo442e7c72020-03-10 16:13:48 -0400446 // Adopt the device if it was in pre-provision state. In all other cases, try to re-enable it.
khenaidood948f772021-08-11 17:49:24 -0400447 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
khenaidoo442e7c72020-03-10 16:13:48 -0400448 if err != nil {
khenaidood948f772021-08-11 17:49:24 -0400449 logger.Errorw(ctx, "grpc-client-nil",
450 log.Fields{
451 "error": err,
452 "device-id": agent.deviceID,
453 "device-type": agent.deviceType,
454 "adapter-endpoint": newDevice.AdapterEndpoint,
455 })
456 agent.requestQueue.RequestComplete()
khenaidoo442e7c72020-03-10 16:13:48 -0400457 return err
458 }
khenaidood948f772021-08-11 17:49:24 -0400459 subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
460 requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS
461 go func() {
462 defer cancel()
463 var err error
464 if oldDevice.AdminState == voltha.AdminState_PREPROVISIONED {
465 _, err = client.AdoptDevice(subCtx, newDevice)
466 } else {
467 _, err = client.ReEnableDevice(subCtx, newDevice)
468 }
469 if err == nil {
470 agent.onSuccess(subCtx, nil, nil, true)
471 } else {
472 agent.onFailure(subCtx, err, nil, nil, true)
Himani Chawlab4c25912020-11-12 17:16:38 +0530473 }
474 }()
khenaidood948f772021-08-11 17:49:24 -0400475
476 // Update device
477 if err = agent.updateDeviceAndReleaseLock(ctx, newDevice); err != nil {
478 return err
khenaidoo2c6a0992019-04-29 13:46:56 -0400479 }
khenaidood948f772021-08-11 17:49:24 -0400480 currAdminState = newDevice.AdminState
481 return nil
khenaidoo2c6a0992019-04-29 13:46:56 -0400482}
483
A R Karthick5c28f552019-12-11 22:47:44 -0800484//addFlowsAndGroups adds the "newFlows" and "newGroups" from the existing flows/groups and sends the update to the
485//adapters
khenaidoo9beaaf12021-10-19 17:32:01 -0400486func (agent *Agent) addFlowsAndGroups(ctx context.Context, newFlows []*ofp.OfpFlowStats, newGroups []*ofp.OfpGroupEntry, flowMetadata *ofp.FlowMetadata) error {
Mahir Gunyel03de0d32020-06-03 01:36:59 -0700487 var flwResponse, grpResponse coreutils.Response
488 var err error
489 //if new flow list is empty then the called function returns quickly
490 if flwResponse, err = agent.addFlowsToAdapter(ctx, newFlows, flowMetadata); err != nil {
A R Karthick5c28f552019-12-11 22:47:44 -0800491 return err
492 }
Mahir Gunyel03de0d32020-06-03 01:36:59 -0700493 //if new group list is empty then the called function returns quickly
494 if grpResponse, err = agent.addGroupsToAdapter(ctx, newGroups, flowMetadata); err != nil {
495 return err
496 }
Himani Chawla4b4bd252021-11-08 15:59:40 +0530497 if errs := coreutils.WaitForNilOrErrorResponses(agent.flowTimeout, flwResponse, grpResponse); errs != nil {
khenaidood948f772021-08-11 17:49:24 -0400498 logger.Warnw(ctx, "adapter-response", log.Fields{"device-id": agent.deviceID, "result": errs})
khenaidoo442e7c72020-03-10 16:13:48 -0400499 return status.Errorf(codes.Aborted, "flow-failure-device-%s", agent.deviceID)
khenaidoo0458db62019-06-20 08:50:36 -0400500 }
khenaidoo0458db62019-06-20 08:50:36 -0400501 return nil
502}
503
A R Karthick5c28f552019-12-11 22:47:44 -0800504//deleteFlowsAndGroups removes the "flowsToDel" and "groupsToDel" from the existing flows/groups and sends the update to the
505//adapters
khenaidoo9beaaf12021-10-19 17:32:01 -0400506func (agent *Agent) deleteFlowsAndGroups(ctx context.Context, flowsToDel []*ofp.OfpFlowStats, groupsToDel []*ofp.OfpGroupEntry, flowMetadata *ofp.FlowMetadata) error {
Mahir Gunyel03de0d32020-06-03 01:36:59 -0700507 var flwResponse, grpResponse coreutils.Response
508 var err error
509 if flwResponse, err = agent.deleteFlowsFromAdapter(ctx, flowsToDel, flowMetadata); err != nil {
A R Karthick5c28f552019-12-11 22:47:44 -0800510 return err
511 }
Mahir Gunyel03de0d32020-06-03 01:36:59 -0700512 if grpResponse, err = agent.deleteGroupsFromAdapter(ctx, groupsToDel, flowMetadata); err != nil {
513 return err
514 }
515
Himani Chawla4b4bd252021-11-08 15:59:40 +0530516 if res := coreutils.WaitForNilOrErrorResponses(agent.flowTimeout, flwResponse, grpResponse); res != nil {
khenaidoo0458db62019-06-20 08:50:36 -0400517 return status.Errorf(codes.Aborted, "errors-%s", res)
518 }
519 return nil
khenaidoo0458db62019-06-20 08:50:36 -0400520}
521
A R Karthick5c28f552019-12-11 22:47:44 -0800522//updateFlowsAndGroups replaces the existing flows and groups with "updatedFlows" and "updatedGroups" respectively. It
523//also sends the updates to the adapters
khenaidoo9beaaf12021-10-19 17:32:01 -0400524func (agent *Agent) updateFlowsAndGroups(ctx context.Context, updatedFlows []*ofp.OfpFlowStats, updatedGroups []*ofp.OfpGroupEntry, flowMetadata *ofp.FlowMetadata) error {
Mahir Gunyel03de0d32020-06-03 01:36:59 -0700525 var flwResponse, grpResponse coreutils.Response
526 var err error
527 if flwResponse, err = agent.updateFlowsToAdapter(ctx, updatedFlows, flowMetadata); err != nil {
A R Karthick5c28f552019-12-11 22:47:44 -0800528 return err
529 }
Mahir Gunyel03de0d32020-06-03 01:36:59 -0700530 if grpResponse, err = agent.updateGroupsToAdapter(ctx, updatedGroups, flowMetadata); err != nil {
531 return err
532 }
533
Himani Chawla4b4bd252021-11-08 15:59:40 +0530534 if res := coreutils.WaitForNilOrErrorResponses(agent.flowTimeout, flwResponse, grpResponse); res != nil {
khenaidoo0458db62019-06-20 08:50:36 -0400535 return status.Errorf(codes.Aborted, "errors-%s", res)
536 }
537 return nil
khenaidoo19d7b632018-10-30 10:49:50 -0400538}
539
khenaidoo4d4802d2018-10-04 21:59:49 -0400540//disableDevice disable a device
Kent Hagerman2b216042020-04-03 18:28:56 -0400541func (agent *Agent) disableDevice(ctx context.Context) error {
khenaidood948f772021-08-11 17:49:24 -0400542 var err error
Maninder9a1bc0d2020-10-26 11:34:02 +0530543 var desc string
khenaidood948f772021-08-11 17:49:24 -0400544 var prevAdminState, currAdminState common.AdminState_Types
545 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
546 defer func() { agent.logDeviceUpdate(ctx, &prevAdminState, &currAdminState, requestStatus, err, desc) }()
Maninder9a1bc0d2020-10-26 11:34:02 +0530547
khenaidood948f772021-08-11 17:49:24 -0400548 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
khenaidoo442e7c72020-03-10 16:13:48 -0400549 return err
550 }
Himani Chawlab4c25912020-11-12 17:16:38 +0530551 logger.Debugw(ctx, "disable-device", log.Fields{"device-id": agent.deviceID})
khenaidoo6e55d9e2019-12-12 18:26:26 -0500552
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400553 cloned := agent.cloneDeviceWithoutLock()
khenaidood948f772021-08-11 17:49:24 -0400554 prevAdminState = agent.device.AdminState
khenaidoo6e55d9e2019-12-12 18:26:26 -0500555
Maninder2195ccc2021-06-23 20:23:01 +0530556 if !agent.proceedWithRequest(cloned) {
khenaidood948f772021-08-11 17:49:24 -0400557 err = status.Errorf(codes.FailedPrecondition, "cannot complete operation as device deletion is in progress or reconciling is in progress/failed: %s", agent.deviceID)
Maninder2195ccc2021-06-23 20:23:01 +0530558 agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -0400559 return err
Maninder2195ccc2021-06-23 20:23:01 +0530560 }
561
khenaidoo6e55d9e2019-12-12 18:26:26 -0500562 if cloned.AdminState == voltha.AdminState_DISABLED {
Maninder9a1bc0d2020-10-26 11:34:02 +0530563 desc = "device-already-disabled"
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400564 agent.requestQueue.RequestComplete()
npujar1d86a522019-11-14 17:11:16 +0530565 return nil
566 }
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530567 if cloned.AdminState == voltha.AdminState_PREPROVISIONED {
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400568 agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -0400569 err = status.Errorf(codes.FailedPrecondition, "deviceId:%s, invalid-admin-state:%s", agent.deviceID, cloned.AdminState)
570 return err
npujar1d86a522019-11-14 17:11:16 +0530571 }
Maninder0aabf0c2021-03-17 14:55:14 +0530572
npujar1d86a522019-11-14 17:11:16 +0530573 // Update the Admin State and operational state before sending the request out
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400574 cloned.AdminState = voltha.AdminState_DISABLED
575 cloned.OperStatus = voltha.OperStatus_UNKNOWN
Maninder9a1bc0d2020-10-26 11:34:02 +0530576
khenaidood948f772021-08-11 17:49:24 -0400577 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
khenaidoo442e7c72020-03-10 16:13:48 -0400578 if err != nil {
khenaidood948f772021-08-11 17:49:24 -0400579 logger.Errorw(ctx, "grpc-client-nil",
580 log.Fields{
581 "error": err,
582 "device-id": agent.deviceID,
583 "device-type": agent.deviceType,
584 "adapter-endpoint": cloned.AdapterEndpoint,
585 })
586 agent.requestQueue.RequestComplete()
npujar1d86a522019-11-14 17:11:16 +0530587 return err
khenaidoo0a822f92019-05-08 15:15:57 -0400588 }
khenaidood948f772021-08-11 17:49:24 -0400589 subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
590 requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS
591 go func() {
592 defer cancel()
593 _, err := client.DisableDevice(subCtx, cloned)
594 if err == nil {
595 agent.onSuccess(subCtx, nil, nil, true)
596 } else {
597 agent.onFailure(subCtx, err, nil, nil, true)
598 }
599 }()
Maninder9a1bc0d2020-10-26 11:34:02 +0530600
khenaidood948f772021-08-11 17:49:24 -0400601 // Update device
602 if err = agent.updateDeviceAndReleaseLock(ctx, cloned); err != nil {
603 return err
604 }
605 currAdminState = cloned.AdminState
khenaidoo0a822f92019-05-08 15:15:57 -0400606
khenaidoo92e62c52018-10-03 14:02:54 -0400607 return nil
608}
609
Kent Hagerman2b216042020-04-03 18:28:56 -0400610func (agent *Agent) rebootDevice(ctx context.Context) error {
Maninder9a1bc0d2020-10-26 11:34:02 +0530611 var desc string
khenaidood948f772021-08-11 17:49:24 -0400612 var err error
613 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
614 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
Maninder9a1bc0d2020-10-26 11:34:02 +0530615
khenaidood948f772021-08-11 17:49:24 -0400616 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
Maninder9a1bc0d2020-10-26 11:34:02 +0530617 desc = err.Error()
npujar1d86a522019-11-14 17:11:16 +0530618 return err
khenaidoo4d4802d2018-10-04 21:59:49 -0400619 }
khenaidoo442e7c72020-03-10 16:13:48 -0400620 defer agent.requestQueue.RequestComplete()
Himani Chawlab4c25912020-11-12 17:16:38 +0530621 logger.Debugw(ctx, "reboot-device", log.Fields{"device-id": agent.deviceID})
khenaidoo442e7c72020-03-10 16:13:48 -0400622
Kent Hagermancba2f302020-07-28 13:37:36 -0400623 device := agent.getDeviceReadOnlyWithoutLock()
Maninder2195ccc2021-06-23 20:23:01 +0530624
625 if !agent.proceedWithRequest(device) {
khenaidood948f772021-08-11 17:49:24 -0400626 err = status.Errorf(codes.FailedPrecondition, "cannot complete operation as device deletion is in progress or reconciling is in progress/failed:%s", agent.deviceID)
khenaidoo442e7c72020-03-10 16:13:48 -0400627 return err
628 }
Maninder9a1bc0d2020-10-26 11:34:02 +0530629
khenaidood948f772021-08-11 17:49:24 -0400630 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
631 if err != nil {
632 logger.Errorw(ctx, "grpc-client-nil",
633 log.Fields{
634 "error": err,
635 "device-id": agent.deviceID,
636 "device-type": agent.deviceType,
637 "adapter-endpoint": device.AdapterEndpoint,
638 })
639 return err
640 }
641 subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
642 requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS
643 go func() {
644 defer cancel()
645 _, err := client.RebootDevice(subCtx, device)
646 if err == nil {
647 agent.onSuccess(subCtx, nil, nil, true)
648 } else {
649 agent.onFailure(subCtx, err, nil, nil, true)
650 }
651 }()
khenaidoo4d4802d2018-10-04 21:59:49 -0400652 return nil
653}
654
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530655func (agent *Agent) deleteDeviceForce(ctx context.Context) error {
Himani Chawlab4c25912020-11-12 17:16:38 +0530656 logger.Debugw(ctx, "delete-device-force", log.Fields{"device-id": agent.deviceID})
Maninder9a1bc0d2020-10-26 11:34:02 +0530657
658 var desc string
khenaidood948f772021-08-11 17:49:24 -0400659 var err error
660 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
661 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
Maninder9a1bc0d2020-10-26 11:34:02 +0530662
khenaidood948f772021-08-11 17:49:24 -0400663 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530664 return err
665 }
666 // Get the device Transient state, return err if it is DELETING
667 previousDeviceTransientState := agent.getTransientState()
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530668 device := agent.cloneDeviceWithoutLock()
khenaidood948f772021-08-11 17:49:24 -0400669 if !agent.isForceDeletingAllowed(previousDeviceTransientState, device) {
670 agent.requestQueue.RequestComplete()
671 err = status.Error(codes.FailedPrecondition, fmt.Sprintf("deviceId:%s, force deletion is in progress", agent.deviceID))
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530672 return err
673 }
Himani Chawlab4c25912020-11-12 17:16:38 +0530674
khenaidood948f772021-08-11 17:49:24 -0400675 previousAdminState := device.AdminState
676 if previousAdminState != common.AdminState_PREPROVISIONED {
khenaidoo9beaaf12021-10-19 17:32:01 -0400677 var client adapter_service.AdapterServiceClient
khenaidood948f772021-08-11 17:49:24 -0400678 client, err = agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530679 if err != nil {
khenaidood948f772021-08-11 17:49:24 -0400680 logger.Errorw(ctx, "grpc-client-nil",
681 log.Fields{
682 "error": err,
683 "device-id": agent.deviceID,
684 "device-type": agent.deviceType,
685 "adapter-endpoint": device.AdapterEndpoint,
686 })
687 agent.requestQueue.RequestComplete()
khenaidoo68a5e0c2021-11-06 13:08:03 -0400688 return fmt.Errorf("remote-not-reachable %w", errNoConnection)
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530689 }
khenaidood948f772021-08-11 17:49:24 -0400690 subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
691 requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS
692 go func() {
693 defer cancel()
694 _, err := client.DeleteDevice(subCtx, device)
695 if err == nil {
696 agent.onSuccess(subCtx, nil, nil, true)
697 } else {
698 agent.onFailure(subCtx, err, nil, nil, true)
699 }
700 }()
701 }
702
703 // Update device
704 if err = agent.updateDeviceWithTransientStateAndReleaseLock(ctx, device,
705 core.DeviceTransientState_FORCE_DELETING, previousDeviceTransientState); err != nil {
706 return err
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530707 }
708 return nil
709}
710
Kent Hagerman2b216042020-04-03 18:28:56 -0400711func (agent *Agent) deleteDevice(ctx context.Context) error {
Himani Chawlab4c25912020-11-12 17:16:38 +0530712 logger.Debugw(ctx, "delete-device", log.Fields{"device-id": agent.deviceID})
Maninder9a1bc0d2020-10-26 11:34:02 +0530713
714 var desc string
khenaidood948f772021-08-11 17:49:24 -0400715 var err error
716 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
717 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
Maninder9a1bc0d2020-10-26 11:34:02 +0530718
khenaidood948f772021-08-11 17:49:24 -0400719 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
Maninder9a1bc0d2020-10-26 11:34:02 +0530720 desc = err.Error()
khenaidoo442e7c72020-03-10 16:13:48 -0400721 return err
722 }
Maninder0aabf0c2021-03-17 14:55:14 +0530723
Maninder2195ccc2021-06-23 20:23:01 +0530724 device := agent.cloneDeviceWithoutLock()
725
726 if !agent.proceedWithRequest(device) {
Maninder0aabf0c2021-03-17 14:55:14 +0530727 agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -0400728 err = status.Errorf(codes.FailedPrecondition, "cannot complete operation as device deletion is in progress or reconciling is in progress/failed: %s", agent.deviceID)
729 return err
Maninder0aabf0c2021-03-17 14:55:14 +0530730 }
731
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530732 // Get the device Transient state, return err if it is DELETING
733 previousDeviceTransientState := agent.getTransientState()
khenaidoo6e55d9e2019-12-12 18:26:26 -0500734
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530735 previousAdminState := device.AdminState
736 // Change the device transient state to DELETING_FROM_ADAPTER state till the device is removed from adapters.
khenaidood948f772021-08-11 17:49:24 -0400737 currentDeviceTransientState := core.DeviceTransientState_DELETING_FROM_ADAPTER
khenaidoo442e7c72020-03-10 16:13:48 -0400738
khenaidood948f772021-08-11 17:49:24 -0400739 if previousAdminState == common.AdminState_PREPROVISIONED {
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530740 // Change the state to DELETING POST ADAPTER RESPONSE directly as adapters have no info of the device.
khenaidood948f772021-08-11 17:49:24 -0400741 currentDeviceTransientState = core.DeviceTransientState_DELETING_POST_ADAPTER_RESPONSE
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530742 }
khenaidood948f772021-08-11 17:49:24 -0400743 // If the device was in pre-prov state (only parent device are in that state) then do not send the request to the
744 // adapter
745 if previousAdminState != common.AdminState_PREPROVISIONED {
khenaidoo9beaaf12021-10-19 17:32:01 -0400746 var client adapter_service.AdapterServiceClient
khenaidood948f772021-08-11 17:49:24 -0400747 client, err = agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
748 if err != nil {
749 logger.Errorw(ctx, "grpc-client-nil",
750 log.Fields{
751 "error": err,
752 "device-id": agent.deviceID,
753 "device-type": agent.deviceType,
754 "adapter-endpoint": device.AdapterEndpoint,
755 })
756 agent.requestQueue.RequestComplete()
757 return err
758 }
759 subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
760 requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS
761 go func() {
762 defer cancel()
763 _, err := client.DeleteDevice(subCtx, device)
764 if err == nil {
765 agent.onDeleteSuccess(subCtx, nil, nil)
766 } else {
767 agent.onDeleteFailure(subCtx, err, nil, nil)
768 }
769 }()
770 }
771
772 // Update device and release lock
773 if err = agent.updateDeviceWithTransientStateAndReleaseLock(ctx, device,
Himani Chawlab4c25912020-11-12 17:16:38 +0530774 currentDeviceTransientState, previousDeviceTransientState); err != nil {
Maninder9a1bc0d2020-10-26 11:34:02 +0530775 desc = err.Error()
npujar1d86a522019-11-14 17:11:16 +0530776 return err
777 }
Himani Chawlab4c25912020-11-12 17:16:38 +0530778
khenaidoo4d4802d2018-10-04 21:59:49 -0400779 return nil
780}
781
Kent Hagerman2b216042020-04-03 18:28:56 -0400782func (agent *Agent) setParentID(ctx context.Context, device *voltha.Device, parentID string) error {
khenaidoo442e7c72020-03-10 16:13:48 -0400783 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
784 return err
785 }
Himani Chawlab4c25912020-11-12 17:16:38 +0530786 logger.Debugw(ctx, "set-parent-id", log.Fields{"device-id": device.Id, "parent-id": parentID})
khenaidoo6e55d9e2019-12-12 18:26:26 -0500787
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400788 cloned := agent.cloneDeviceWithoutLock()
npujar1d86a522019-11-14 17:11:16 +0530789 cloned.ParentId = parentID
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400790 return agent.updateDeviceAndReleaseLock(ctx, cloned)
khenaidooad06fd72019-10-28 12:26:05 -0400791}
792
khenaidoo442e7c72020-03-10 16:13:48 -0400793// getSwitchCapability retrieves the switch capability of a parent device
khenaidoo9beaaf12021-10-19 17:32:01 -0400794func (agent *Agent) getSwitchCapability(ctx context.Context) (*ca.SwitchCapability, error) {
Himani Chawlab4c25912020-11-12 17:16:38 +0530795 logger.Debugw(ctx, "get-switch-capability", log.Fields{"device-id": agent.deviceID})
khenaidoo442e7c72020-03-10 16:13:48 -0400796
Kent Hagermancba2f302020-07-28 13:37:36 -0400797 device, err := agent.getDeviceReadOnly(ctx)
khenaidoo442e7c72020-03-10 16:13:48 -0400798 if err != nil {
khenaidoob9203542018-09-17 22:56:37 -0400799 return nil, err
khenaidoob9203542018-09-17 22:56:37 -0400800 }
khenaidood948f772021-08-11 17:49:24 -0400801
802 // Get the gRPC client
803 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
khenaidoo442e7c72020-03-10 16:13:48 -0400804 if err != nil {
805 return nil, err
806 }
807
khenaidood948f772021-08-11 17:49:24 -0400808 return client.GetOfpDeviceInfo(ctx, device)
khenaidoob9203542018-09-17 22:56:37 -0400809}
810
khenaidood948f772021-08-11 17:49:24 -0400811func (agent *Agent) onPacketFailure(ctx context.Context, err error, packet *ofp.OfpPacketOut) {
812 logger.Errorw(ctx, "packet-out-error", log.Fields{
khenaidoo442e7c72020-03-10 16:13:48 -0400813 "device-id": agent.deviceID,
khenaidood948f772021-08-11 17:49:24 -0400814 "error": err.Error(),
815 "packet": hex.EncodeToString(packet.Data),
khenaidoo442e7c72020-03-10 16:13:48 -0400816 })
khenaidood948f772021-08-11 17:49:24 -0400817 rpce := agent.deviceMgr.NewRPCEvent(ctx, agent.deviceID, err.Error(), nil)
818 go agent.deviceMgr.SendRPCEvent(ctx, "RPC_ERROR_RAISE_EVENT", rpce,
819 voltha.EventCategory_COMMUNICATION, nil, time.Now().Unix())
khenaidoo442e7c72020-03-10 16:13:48 -0400820}
821
Kent Hagerman2b216042020-04-03 18:28:56 -0400822func (agent *Agent) packetOut(ctx context.Context, outPort uint32, packet *ofp.OfpPacketOut) error {
Scott Baker80678602019-11-14 16:57:36 -0800823 if agent.deviceType == "" {
npujar467fe752020-01-16 20:17:45 +0530824 agent.reconcileWithKVStore(ctx)
Scott Baker80678602019-11-14 16:57:36 -0800825 }
khenaidoofdbad6e2018-11-06 22:26:38 -0500826 // Send packet to adapter
khenaidood948f772021-08-11 17:49:24 -0400827 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
khenaidoo442e7c72020-03-10 16:13:48 -0400828 if err != nil {
khenaidood948f772021-08-11 17:49:24 -0400829 logger.Errorw(ctx, "grpc-client-nil",
830 log.Fields{
831 "error": err,
832 "device-id": agent.deviceID,
833 "device-type": agent.deviceType,
834 })
835 return err
khenaidoofdbad6e2018-11-06 22:26:38 -0500836 }
khenaidood948f772021-08-11 17:49:24 -0400837 subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
838 go func() {
839 defer cancel()
khenaidoo9beaaf12021-10-19 17:32:01 -0400840 _, err := client.SendPacketOut(subCtx, &ca.PacketOut{
khenaidood948f772021-08-11 17:49:24 -0400841 DeviceId: agent.deviceID,
842 EgressPortNo: outPort,
843 Packet: packet,
844 })
845 if err == nil {
846 agent.onSuccess(subCtx, nil, nil, false)
847 } else {
848 agent.onPacketFailure(subCtx, err, packet)
849 }
850 }()
khenaidoofdbad6e2018-11-06 22:26:38 -0500851 return nil
852}
853
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400854func (agent *Agent) updateDeviceUsingAdapterData(ctx context.Context, device *voltha.Device) error {
khenaidood948f772021-08-11 17:49:24 -0400855 var err error
856 var desc string
857 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
858 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
859
860 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400861 return err
862 }
Himani Chawlab4c25912020-11-12 17:16:38 +0530863 logger.Debugw(ctx, "update-device-using-adapter-data", log.Fields{"device-id": device.Id})
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400864
865 cloned := agent.cloneDeviceWithoutLock()
Mahir Gunyel8e2707d2019-07-25 00:36:21 -0700866 cloned.Root = device.Root
867 cloned.Vendor = device.Vendor
868 cloned.Model = device.Model
869 cloned.SerialNumber = device.SerialNumber
870 cloned.MacAddress = device.MacAddress
871 cloned.Vlan = device.Vlan
872 cloned.Reason = device.Reason
Andrea Campanella025667e2021-01-14 11:50:07 +0100873 cloned.ImageDownloads = device.ImageDownloads
khenaidood948f772021-08-11 17:49:24 -0400874 cloned.OperStatus = device.OperStatus
875 cloned.ConnectStatus = device.ConnectStatus
876 if err = agent.updateDeviceAndReleaseLock(ctx, cloned); err == nil {
877 requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
878 }
879 return err
khenaidoo43c82122018-11-22 18:38:28 -0500880}
881
Kent Hagerman2b216042020-04-03 18:28:56 -0400882func (agent *Agent) updateDeviceStatus(ctx context.Context, operStatus voltha.OperStatus_Types, connStatus voltha.ConnectStatus_Types) error {
khenaidood948f772021-08-11 17:49:24 -0400883 var err error
884 var desc string
885 opStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
886 defer func() { agent.logDeviceUpdate(ctx, nil, nil, opStatus, err, desc) }()
887
888 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
khenaidoo442e7c72020-03-10 16:13:48 -0400889 return err
890 }
khenaidoo6e55d9e2019-12-12 18:26:26 -0500891
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400892 cloned := agent.cloneDeviceWithoutLock()
npujar1d86a522019-11-14 17:11:16 +0530893 // Ensure the enums passed in are valid - they will be invalid if they are not set when this function is invoked
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400894 if s, ok := voltha.ConnectStatus_Types_name[int32(connStatus)]; ok {
khenaidood948f772021-08-11 17:49:24 -0400895 logger.Debugw(ctx, "update-device-conn-status", log.Fields{"ok": ok, "val": s})
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400896 cloned.ConnectStatus = connStatus
npujar1d86a522019-11-14 17:11:16 +0530897 }
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400898 if s, ok := voltha.OperStatus_Types_name[int32(operStatus)]; ok {
khenaidood948f772021-08-11 17:49:24 -0400899 logger.Debugw(ctx, "update-device-oper-status", log.Fields{"ok": ok, "val": s})
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400900 cloned.OperStatus = operStatus
npujar1d86a522019-11-14 17:11:16 +0530901 }
Himani Chawlab4c25912020-11-12 17:16:38 +0530902 logger.Debugw(ctx, "update-device-status", log.Fields{"device-id": cloned.Id, "oper-status": cloned.OperStatus, "connect-status": cloned.ConnectStatus})
npujar1d86a522019-11-14 17:11:16 +0530903 // Store the device
khenaidood948f772021-08-11 17:49:24 -0400904 if err = agent.updateDeviceAndReleaseLock(ctx, cloned); err == nil {
905 opStatus.Code = common.OperationResp_OPERATION_SUCCESS
906 }
907 return err
khenaidoo92e62c52018-10-03 14:02:54 -0400908}
909
khenaidoob9203542018-09-17 22:56:37 -0400910// TODO: A generic device update by attribute
Kent Hagerman2b216042020-04-03 18:28:56 -0400911func (agent *Agent) updateDeviceAttribute(ctx context.Context, name string, value interface{}) {
khenaidoob9203542018-09-17 22:56:37 -0400912 if value == nil {
913 return
914 }
khenaidoo6e55d9e2019-12-12 18:26:26 -0500915
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400916 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
917 logger.Warnw(ctx, "request-aborted", log.Fields{"device-id": agent.deviceID, "name": name, "error": err})
918 return
919 }
920
921 cloned := agent.cloneDeviceWithoutLock()
khenaidoob9203542018-09-17 22:56:37 -0400922 updated := false
khenaidoo6e55d9e2019-12-12 18:26:26 -0500923 s := reflect.ValueOf(cloned).Elem()
khenaidoob9203542018-09-17 22:56:37 -0400924 if s.Kind() == reflect.Struct {
925 // exported field
926 f := s.FieldByName(name)
927 if f.IsValid() && f.CanSet() {
928 switch f.Kind() {
929 case reflect.String:
930 f.SetString(value.(string))
931 updated = true
932 case reflect.Uint32:
933 f.SetUint(uint64(value.(uint32)))
934 updated = true
935 case reflect.Bool:
936 f.SetBool(value.(bool))
937 updated = true
938 }
939 }
940 }
divyadesaicb8b59d2020-08-18 09:55:47 +0000941 logger.Debugw(ctx, "update-field-status", log.Fields{"device-id": cloned.Id, "name": name, "updated": updated})
khenaidoob9203542018-09-17 22:56:37 -0400942 // Save the data
khenaidoo6e55d9e2019-12-12 18:26:26 -0500943
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400944 if err := agent.updateDeviceAndReleaseLock(ctx, cloned); err != nil {
Rohan Agrawal31f21802020-06-12 05:38:46 +0000945 logger.Warnw(ctx, "attribute-update-failed", log.Fields{"attribute": name, "value": value})
khenaidoob9203542018-09-17 22:56:37 -0400946 }
khenaidoob9203542018-09-17 22:56:37 -0400947}
serkant.uluderya334479d2019-04-10 08:26:15 -0700948
Kent Hagerman45a13e42020-04-13 12:23:50 -0400949func (agent *Agent) simulateAlarm(ctx context.Context, simulateReq *voltha.SimulateAlarmRequest) error {
khenaidood948f772021-08-11 17:49:24 -0400950 var err error
951 var desc string
952 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
953 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
954
955 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
khenaidoo442e7c72020-03-10 16:13:48 -0400956 return err
957 }
958 defer agent.requestQueue.RequestComplete()
Himani Chawlab4c25912020-11-12 17:16:38 +0530959 logger.Debugw(ctx, "simulate-alarm", log.Fields{"device-id": agent.deviceID})
khenaidoo6e55d9e2019-12-12 18:26:26 -0500960
Kent Hagermancba2f302020-07-28 13:37:36 -0400961 device := agent.getDeviceReadOnlyWithoutLock()
khenaidoo6e55d9e2019-12-12 18:26:26 -0500962
khenaidood948f772021-08-11 17:49:24 -0400963 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
khenaidoo442e7c72020-03-10 16:13:48 -0400964 if err != nil {
khenaidood948f772021-08-11 17:49:24 -0400965 logger.Errorw(ctx, "grpc-client-nil",
966 log.Fields{
967 "error": err,
968 "device-id": agent.deviceID,
969 "device-type": agent.deviceType,
970 "adapter-endpoint": device.AdapterEndpoint,
971 })
npujar1d86a522019-11-14 17:11:16 +0530972 return err
serkant.uluderya334479d2019-04-10 08:26:15 -0700973 }
khenaidood948f772021-08-11 17:49:24 -0400974 subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
975 requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS
976 go func() {
977 defer cancel()
khenaidoo9beaaf12021-10-19 17:32:01 -0400978 _, err := client.SimulateAlarm(subCtx, &ca.SimulateAlarmMessage{Device: device, Request: simulateReq})
khenaidood948f772021-08-11 17:49:24 -0400979 if err == nil {
980 agent.onSuccess(subCtx, nil, nil, false)
981 } else {
982 agent.onFailure(subCtx, err, nil, nil, false)
983 }
984 }()
serkant.uluderya334479d2019-04-10 08:26:15 -0700985 return nil
986}
Mahir Gunyelb5851672019-07-24 10:46:26 +0300987
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400988// This function updates the device in the DB, releases the device lock, and runs any state transitions.
989// The calling function MUST hold the device lock. The caller MUST NOT modify the device after this is called.
990func (agent *Agent) updateDeviceAndReleaseLock(ctx context.Context, device *voltha.Device) error {
991 // fail early if this agent is no longer valid
Kent Hagerman4f355f52020-03-30 16:01:33 -0400992 if agent.stopped {
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400993 agent.requestQueue.RequestComplete()
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530994 return errors.New("device-agent-stopped")
Thomas Lee Se5a44012019-11-07 20:32:24 +0530995 }
Kent Hagerman4f355f52020-03-30 16:01:33 -0400996
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400997 // update in db
Kent Hagermanf5a67352020-04-30 15:15:26 -0400998 if err := agent.dbProxy.Set(ctx, agent.deviceID, device); err != nil {
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400999 agent.requestQueue.RequestComplete()
Kent Hagerman4f355f52020-03-30 16:01:33 -04001000 return status.Errorf(codes.Internal, "failed-update-device:%s: %s", agent.deviceID, err)
Mahir Gunyelb5851672019-07-24 10:46:26 +03001001 }
divyadesaicb8b59d2020-08-18 09:55:47 +00001002 logger.Debugw(ctx, "updated-device-in-store", log.Fields{"device-id: ": agent.deviceID})
Mahir Gunyelb5851672019-07-24 10:46:26 +03001003
Kent Hagerman6031aad2020-07-29 16:36:33 -04001004 prevDevice := agent.device
Kent Hagermanf6db9f12020-07-22 17:16:19 -04001005 // update the device
khenaidoo0db4c812020-05-27 15:27:30 -04001006 agent.device = device
Mahir Gunyelb0343bf2021-05-11 14:14:26 -07001007 //If any of the states has chenged, send the change event.
1008 if prevDevice.OperStatus != device.OperStatus || prevDevice.ConnectStatus != device.ConnectStatus || prevDevice.AdminState != device.AdminState {
1009 _ = agent.deviceMgr.Agent.SendDeviceStateChangeEvent(ctx, prevDevice.OperStatus, prevDevice.ConnectStatus, prevDevice.AdminState, device, time.Now().Unix())
1010 }
Maninder0aabf0c2021-03-17 14:55:14 +05301011 deviceTransientState := agent.getTransientState()
1012
Kent Hagermanf6db9f12020-07-22 17:16:19 -04001013 // release lock before processing transition
1014 agent.requestQueue.RequestComplete()
Himani Chawlab4c25912020-11-12 17:16:38 +05301015 subCtx := coreutils.WithSpanAndRPCMetadataFromContext(ctx)
Kent Hagermanf6db9f12020-07-22 17:16:19 -04001016
Himani Chawlab4c25912020-11-12 17:16:38 +05301017 if err := agent.deviceMgr.stateTransitions.ProcessTransition(subCtx,
Maninder0aabf0c2021-03-17 14:55:14 +05301018 device, prevDevice, deviceTransientState, deviceTransientState); err != nil {
Himani Chawlab4c25912020-11-12 17:16:38 +05301019 logger.Errorw(ctx, "failed-process-transition", log.Fields{"device-id": device.Id, "previous-admin-state": prevDevice.AdminState, "current-admin-state": device.AdminState})
1020 // Sending RPC EVENT here
1021 rpce := agent.deviceMgr.NewRPCEvent(ctx, agent.deviceID, err.Error(), nil)
Himani Chawla606a4f02021-03-23 19:45:58 +05301022 agent.deviceMgr.SendRPCEvent(ctx, "RPC_ERROR_RAISE_EVENT", rpce, voltha.EventCategory_COMMUNICATION,
1023 nil, time.Now().Unix())
Himani Chawlab4c25912020-11-12 17:16:38 +05301024
Kent Hagermanf6db9f12020-07-22 17:16:19 -04001025 }
Mahir Gunyelb5851672019-07-24 10:46:26 +03001026 return nil
1027}
Mahir Gunyelfdee9212019-10-16 16:52:21 -07001028
Himani Chawla2ba1c9c2020-10-07 13:19:03 +05301029// This function updates the device transient in the DB through loader, releases the device lock, and runs any state transitions.
1030// The calling function MUST hold the device lock. The caller MUST NOT modify the device after this is called.
1031func (agent *Agent) updateDeviceWithTransientStateAndReleaseLock(ctx context.Context, device *voltha.Device,
khenaidood948f772021-08-11 17:49:24 -04001032 transientState, prevTransientState core.DeviceTransientState_Types) error {
Himani Chawla2ba1c9c2020-10-07 13:19:03 +05301033 // fail early if this agent is no longer valid
1034 if agent.stopped {
1035 agent.requestQueue.RequestComplete()
1036 return errors.New("device-agent-stopped")
1037 }
1038 //update device TransientState
1039 if err := agent.updateTransientState(ctx, transientState); err != nil {
1040 agent.requestQueue.RequestComplete()
1041 return err
1042 }
1043 // update in db
1044 if err := agent.dbProxy.Set(ctx, agent.deviceID, device); err != nil {
1045 //Reverting TransientState update
khenaidood948f772021-08-11 17:49:24 -04001046 if errTransient := agent.updateTransientState(ctx, prevTransientState); errTransient != nil {
1047 logger.Errorw(ctx, "failed-to-revert-transient-state-update-on-error", log.Fields{"device-id": device.Id,
1048 "previous-transient-state": prevTransientState, "current-transient-state": transientState, "error": errTransient})
1049 }
Himani Chawla2ba1c9c2020-10-07 13:19:03 +05301050 agent.requestQueue.RequestComplete()
1051 return status.Errorf(codes.Internal, "failed-update-device:%s: %s", agent.deviceID, err)
1052 }
1053
1054 logger.Debugw(ctx, "updated-device-in-store", log.Fields{"device-id: ": agent.deviceID})
1055
1056 prevDevice := agent.device
1057 // update the device
1058 agent.device = device
Mahir Gunyelb0343bf2021-05-11 14:14:26 -07001059 //If any of the states has chenged, send the change event.
1060 if prevDevice.OperStatus != device.OperStatus || prevDevice.ConnectStatus != device.ConnectStatus || prevDevice.AdminState != device.AdminState {
1061 _ = agent.deviceMgr.Agent.SendDeviceStateChangeEvent(ctx, prevDevice.OperStatus, prevDevice.ConnectStatus, prevDevice.AdminState, device, time.Now().Unix())
1062 }
Himani Chawla2ba1c9c2020-10-07 13:19:03 +05301063
1064 // release lock before processing transition
1065 agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -04001066 go func() {
1067 subCtx := coreutils.WithSpanAndRPCMetadataFromContext(ctx)
1068 if err := agent.deviceMgr.stateTransitions.ProcessTransition(subCtx,
1069 device, prevDevice, transientState, prevTransientState); err != nil {
1070 logger.Errorw(ctx, "failed-process-transition", log.Fields{"device-id": device.Id, "previous-admin-state": prevDevice.AdminState, "current-admin-state": device.AdminState})
1071 // Sending RPC EVENT here
1072 rpce := agent.deviceMgr.NewRPCEvent(ctx, agent.deviceID, err.Error(), nil)
1073 agent.deviceMgr.SendRPCEvent(ctx, "RPC_ERROR_RAISE_EVENT", rpce, voltha.EventCategory_COMMUNICATION,
1074 nil, time.Now().Unix())
1075 }
1076 }()
Himani Chawla2ba1c9c2020-10-07 13:19:03 +05301077 return nil
1078}
Kent Hagerman2b216042020-04-03 18:28:56 -04001079func (agent *Agent) updateDeviceReason(ctx context.Context, reason string) error {
khenaidood948f772021-08-11 17:49:24 -04001080 logger.Debugw(ctx, "update-device-reason", log.Fields{"device-id": agent.deviceID, "reason": reason})
1081
1082 var err error
1083 var desc string
1084 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
1085 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
1086
1087 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
khenaidoo442e7c72020-03-10 16:13:48 -04001088 return err
1089 }
Maninder9a1bc0d2020-10-26 11:34:02 +05301090
Kent Hagermanf6db9f12020-07-22 17:16:19 -04001091 cloned := agent.cloneDeviceWithoutLock()
npujar1d86a522019-11-14 17:11:16 +05301092 cloned.Reason = reason
khenaidood948f772021-08-11 17:49:24 -04001093 if err = agent.updateDeviceAndReleaseLock(ctx, cloned); err == nil {
1094 requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
Maninder9a1bc0d2020-10-26 11:34:02 +05301095 }
khenaidood948f772021-08-11 17:49:24 -04001096 return err
Mahir Gunyelfdee9212019-10-16 16:52:21 -07001097}
kesavandbc2d1622020-01-21 00:42:01 -05001098
Kent Hagerman2b216042020-04-03 18:28:56 -04001099func (agent *Agent) ChildDeviceLost(ctx context.Context, device *voltha.Device) error {
Himani Chawlab4c25912020-11-12 17:16:38 +05301100 logger.Debugw(ctx, "child-device-lost", log.Fields{"child-device-id": device.Id, "parent-device-id": agent.deviceID})
Chaitrashree G S543df3e2020-02-24 22:36:54 -05001101
khenaidood948f772021-08-11 17:49:24 -04001102 var err error
1103 var desc string
1104 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
1105 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
1106
Kent Hagerman2a07b862020-06-19 15:23:07 -04001107 // Remove the associated peer ports on the parent device
1108 for portID := range agent.portLoader.ListIDs() {
1109 if portHandle, have := agent.portLoader.Lock(portID); have {
1110 oldPort := portHandle.GetReadOnly()
1111 updatedPeers := make([]*voltha.Port_PeerPort, 0)
1112 for _, peerPort := range oldPort.Peers {
1113 if peerPort.DeviceId != device.Id {
1114 updatedPeers = append(updatedPeers, peerPort)
1115 }
khenaidoo442e7c72020-03-10 16:13:48 -04001116 }
Kent Hagerman2a07b862020-06-19 15:23:07 -04001117 newPort := *oldPort
1118 newPort.Peers = updatedPeers
1119 if err := portHandle.Update(ctx, &newPort); err != nil {
1120 portHandle.Unlock()
1121 return nil
1122 }
1123 portHandle.Unlock()
khenaidoo442e7c72020-03-10 16:13:48 -04001124 }
Chaitrashree G S543df3e2020-02-24 22:36:54 -05001125 }
1126
khenaidoo442e7c72020-03-10 16:13:48 -04001127 //send request to adapter
khenaidood948f772021-08-11 17:49:24 -04001128 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
khenaidoo442e7c72020-03-10 16:13:48 -04001129 if err != nil {
khenaidood948f772021-08-11 17:49:24 -04001130 logger.Errorw(ctx, "grpc-client-nil",
1131 log.Fields{
1132 "error": err,
1133 "device-id": agent.deviceID,
1134 "device-type": agent.deviceType,
1135 "adapter-endpoint": device.AdapterEndpoint,
1136 })
khenaidoo442e7c72020-03-10 16:13:48 -04001137 return err
Chaitrashree G S543df3e2020-02-24 22:36:54 -05001138 }
khenaidood948f772021-08-11 17:49:24 -04001139 subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
1140 requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS
1141 go func() {
1142 defer cancel()
1143 _, err := client.ChildDeviceLost(subCtx, device)
1144 if err == nil {
1145 agent.onSuccess(subCtx, nil, nil, true)
1146 } else {
1147 agent.onFailure(subCtx, err, nil, nil, true)
1148 }
1149 }()
Chaitrashree G S543df3e2020-02-24 22:36:54 -05001150 return nil
Chaitrashree G S543df3e2020-02-24 22:36:54 -05001151}
onkarkundargi87285252020-01-27 11:34:52 +05301152
khenaidoo9beaaf12021-10-19 17:32:01 -04001153func (agent *Agent) startOmciTest(ctx context.Context, omcitestrequest *omci.OmciTestRequest) (*omci.TestResponse, error) {
khenaidood948f772021-08-11 17:49:24 -04001154 var err error
1155 var desc string
1156 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
1157 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
onkarkundargi87285252020-01-27 11:34:52 +05301158
khenaidood948f772021-08-11 17:49:24 -04001159 // OMCI test may be performed on a pre-provisioned device. If a device is in that state both its device type and endpoint
1160 // may not have been set yet.
1161 // First check if we need to update the type or endpoint
1162 cloned, err := agent.getDeviceReadOnly(ctx)
onkarkundargi87285252020-01-27 11:34:52 +05301163 if err != nil {
1164 return nil, err
1165 }
khenaidood948f772021-08-11 17:49:24 -04001166 if cloned.Type == "" || cloned.AdapterEndpoint == "" {
1167 if err = agent.updateDeviceTypeAndEndpoint(ctx); err != nil {
1168 return nil, err
1169 }
1170 cloned, err = agent.getDeviceReadOnly(ctx)
1171 if err != nil {
1172 return nil, err
1173 }
onkarkundargi87285252020-01-27 11:34:52 +05301174 }
1175
khenaidood948f772021-08-11 17:49:24 -04001176 // Send request to the adapter
1177 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
1178 if err != nil {
1179 logger.Errorw(ctx, "grpc-client-nil",
1180 log.Fields{
1181 "error": err,
1182 "device-id": agent.deviceID,
1183 "device-type": agent.deviceType,
1184 "adapter-endpoint": cloned.AdapterEndpoint,
1185 })
1186 return nil, err
onkarkundargi87285252020-01-27 11:34:52 +05301187 }
khenaidood948f772021-08-11 17:49:24 -04001188
khenaidoo9beaaf12021-10-19 17:32:01 -04001189 res, err := client.StartOmciTest(ctx, &ca.OMCITest{
khenaidood948f772021-08-11 17:49:24 -04001190 Device: cloned,
1191 Request: omcitestrequest,
1192 })
1193 if err == nil {
1194 requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
1195 }
1196 return res, err
onkarkundargi87285252020-01-27 11:34:52 +05301197}
Dinesh Belwalkarc1129f12020-02-27 10:41:33 -08001198
khenaidoo9beaaf12021-10-19 17:32:01 -04001199func (agent *Agent) getExtValue(ctx context.Context, pdevice *voltha.Device, cdevice *voltha.Device, valueparam *extension.ValueSpecifier) (*extension.ReturnValues, error) {
Himani Chawlab4c25912020-11-12 17:16:38 +05301200 logger.Debugw(ctx, "get-ext-value", log.Fields{"device-id": agent.deviceID, "onu-id": valueparam.Id, "value-type": valueparam.Value})
khenaidood948f772021-08-11 17:49:24 -04001201 var err error
1202 var desc string
1203 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
1204 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
1205
1206 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
Dinesh Belwalkarc1129f12020-02-27 10:41:33 -08001207 return nil, err
1208 }
1209
khenaidood948f772021-08-11 17:49:24 -04001210 //send request to adapter synchronously
1211 client, err := agent.adapterMgr.GetAdapterClient(ctx, pdevice.AdapterEndpoint)
Dinesh Belwalkarc1129f12020-02-27 10:41:33 -08001212 if err != nil {
khenaidood948f772021-08-11 17:49:24 -04001213 logger.Errorw(ctx, "grpc-client-nil",
1214 log.Fields{
1215 "error": err,
1216 "device-id": agent.deviceID,
1217 "device-type": agent.deviceType,
1218 "adapter-endpoint": pdevice.AdapterEndpoint,
1219 })
1220 agent.requestQueue.RequestComplete()
Dinesh Belwalkarc1129f12020-02-27 10:41:33 -08001221 return nil, err
1222 }
1223
khenaidood948f772021-08-11 17:49:24 -04001224 // Release lock before sending to adapter
1225 agent.requestQueue.RequestComplete()
Dinesh Belwalkarc1129f12020-02-27 10:41:33 -08001226
khenaidoo9beaaf12021-10-19 17:32:01 -04001227 retVal, err := client.GetExtValue(ctx, &ca.GetExtValueMessage{
khenaidood948f772021-08-11 17:49:24 -04001228 ParentDevice: pdevice,
1229 ChildDevice: cdevice,
1230 ValueType: valueparam.Value,
1231 })
1232 if err == nil {
1233 requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
Dinesh Belwalkarc1129f12020-02-27 10:41:33 -08001234 }
khenaidood948f772021-08-11 17:49:24 -04001235 return retVal, err
Dinesh Belwalkarc1129f12020-02-27 10:41:33 -08001236}
dpaul62686312020-06-23 14:17:36 +05301237
khenaidoo9beaaf12021-10-19 17:32:01 -04001238func (agent *Agent) setExtValue(ctx context.Context, device *voltha.Device, value *extension.ValueSet) (*empty.Empty, error) {
Himani Chawlab4c25912020-11-12 17:16:38 +05301239 logger.Debugw(ctx, "set-ext-value", log.Fields{"device-id": value.Id})
khenaidood948f772021-08-11 17:49:24 -04001240
1241 var err error
1242 var desc string
1243 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
1244 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
1245
1246 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
dpaul62686312020-06-23 14:17:36 +05301247 return nil, err
1248 }
1249
1250 //send request to adapter
khenaidood948f772021-08-11 17:49:24 -04001251 //send request to adapter synchronously
1252 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
dpaul62686312020-06-23 14:17:36 +05301253 if err != nil {
khenaidood948f772021-08-11 17:49:24 -04001254 logger.Errorw(ctx, "grpc-client-nil",
1255 log.Fields{
1256 "error": err,
1257 "device-id": agent.deviceID,
1258 "device-type": agent.deviceType,
1259 "adapter-endpoint": device.AdapterEndpoint,
1260 })
1261 agent.requestQueue.RequestComplete()
dpaul62686312020-06-23 14:17:36 +05301262 return nil, err
1263 }
khenaidood948f772021-08-11 17:49:24 -04001264 // Release lock before sending request to adapter
1265 agent.requestQueue.RequestComplete()
dpaul62686312020-06-23 14:17:36 +05301266
khenaidoo9beaaf12021-10-19 17:32:01 -04001267 retVal, err := client.SetExtValue(ctx, &ca.SetExtValueMessage{
khenaidood948f772021-08-11 17:49:24 -04001268 Device: device,
1269 Value: value,
1270 })
1271 if err == nil {
1272 requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
dpaul62686312020-06-23 14:17:36 +05301273 }
khenaidood948f772021-08-11 17:49:24 -04001274 return retVal, err
dpaul62686312020-06-23 14:17:36 +05301275}
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301276
1277func (agent *Agent) getSingleValue(ctx context.Context, request *extension.SingleGetValueRequest) (*extension.SingleGetValueResponse, error) {
Himani Chawlab4c25912020-11-12 17:16:38 +05301278 logger.Debugw(ctx, "get-single-value", log.Fields{"device-id": request.TargetId})
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301279
khenaidood948f772021-08-11 17:49:24 -04001280 var err error
1281 var desc string
1282 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
1283 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
1284
1285 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301286 return nil, err
1287 }
1288
1289 cloned := agent.cloneDeviceWithoutLock()
1290
1291 //send request to adapter
khenaidood948f772021-08-11 17:49:24 -04001292 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301293 if err != nil {
khenaidood948f772021-08-11 17:49:24 -04001294 logger.Errorw(ctx, "grpc-client-nil",
1295 log.Fields{
1296 "error": err,
1297 "device-id": cloned.Id,
1298 "adapter-endpoint": cloned.AdapterEndpoint,
1299 })
1300 agent.requestQueue.RequestComplete()
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301301 return nil, err
1302 }
khenaidood948f772021-08-11 17:49:24 -04001303 // Release lock before sending request to adapter
1304 agent.requestQueue.RequestComplete()
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301305
khenaidood948f772021-08-11 17:49:24 -04001306 resp, err := client.GetSingleValue(ctx, request)
1307 if err == nil {
1308 requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301309 }
khenaidood948f772021-08-11 17:49:24 -04001310 return resp, err
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301311}
1312
1313func (agent *Agent) setSingleValue(ctx context.Context, request *extension.SingleSetValueRequest) (*extension.SingleSetValueResponse, error) {
Himani Chawlab4c25912020-11-12 17:16:38 +05301314 logger.Debugw(ctx, "set-single-value", log.Fields{"device-id": request.TargetId})
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301315
khenaidood948f772021-08-11 17:49:24 -04001316 var err error
1317 var desc string
1318 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
1319 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
1320
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301321 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
1322 return nil, err
1323 }
1324
1325 cloned := agent.cloneDeviceWithoutLock()
1326
1327 //send request to adapter
khenaidood948f772021-08-11 17:49:24 -04001328 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301329 if err != nil {
khenaidood948f772021-08-11 17:49:24 -04001330 logger.Errorw(ctx, "grpc-client-nil",
1331 log.Fields{
1332 "error": err,
1333 "device-id": agent.deviceID,
1334 "device-type": agent.deviceType,
1335 "adapter-endpoint": cloned.AdapterEndpoint,
1336 })
1337 agent.requestQueue.RequestComplete()
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301338 return nil, err
1339 }
khenaidood948f772021-08-11 17:49:24 -04001340 // Release lock before sending request to adapter
1341 agent.requestQueue.RequestComplete()
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301342
khenaidood948f772021-08-11 17:49:24 -04001343 resp, err := client.SetSingleValue(ctx, request)
1344 if err == nil {
1345 requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301346 }
khenaidood948f772021-08-11 17:49:24 -04001347 return resp, err
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301348}
Maninder0aabf0c2021-03-17 14:55:14 +05301349
Maninder2195ccc2021-06-23 20:23:01 +05301350func (agent *Agent) proceedWithRequest(device *voltha.Device) bool {
1351 return !agent.isDeletionInProgress() && !agent.isInReconcileState(device)
Maninder0aabf0c2021-03-17 14:55:14 +05301352}
1353
1354func (agent *Agent) stopReconcile() {
1355 agent.stopReconcilingMutex.Lock()
1356 if agent.stopReconciling != nil {
1357 agent.stopReconciling <- 0
1358 }
1359 agent.stopReconcilingMutex.Unlock()
1360}
1361
khenaidood948f772021-08-11 17:49:24 -04001362// abortAllProcessing is invoked when an adapter managing this device is restarted
1363func (agent *Agent) abortAllProcessing(ctx context.Context) error {
1364 logger.Infow(ctx, "aborting-current-running-requests", log.Fields{"device-id": agent.deviceID})
1365 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
1366 return err
1367 }
1368 defer agent.requestQueue.RequestComplete()
1369
1370 // If any reconciling is in progress just abort it. The adapter is gone.
1371 agent.stopReconcile()
1372
1373 // Update the Core device transient state accordingly
1374 var updatedState core.DeviceTransientState_Types
1375 switch agent.getTransientState() {
1376 case core.DeviceTransientState_RECONCILE_IN_PROGRESS:
1377 updatedState = core.DeviceTransientState_NONE
1378 case core.DeviceTransientState_FORCE_DELETING:
1379 updatedState = core.DeviceTransientState_DELETE_FAILED
1380 case core.DeviceTransientState_DELETING_FROM_ADAPTER:
1381 updatedState = core.DeviceTransientState_DELETE_FAILED
khenaidoo68a5e0c2021-11-06 13:08:03 -04001382 case core.DeviceTransientState_DELETE_FAILED:
1383 // do not change state
1384 return nil
khenaidood948f772021-08-11 17:49:24 -04001385 default:
1386 updatedState = core.DeviceTransientState_NONE
1387 }
1388 if err := agent.updateTransientState(ctx, updatedState); err != nil {
1389 logger.Errorf(ctx, "transient-state-update-failed", log.Fields{"error": err})
1390 return err
1391 }
1392 return nil
1393}
1394
khenaidoo68a5e0c2021-11-06 13:08:03 -04001395func (agent *Agent) DeleteDevicePostAdapterRestart(ctx context.Context) error {
1396 logger.Debugw(ctx, "delete-post-restart", log.Fields{"device-id": agent.deviceID})
1397 ctx = utils.WithNewSpanAndRPCMetadataContext(ctx, "DelteDevicePostAdapterRestart")
1398
1399 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
1400 return err
1401 }
1402
1403 device := agent.getDeviceReadOnlyWithoutLock()
1404 if device.AdminState == voltha.AdminState_PREPROVISIONED {
1405 logger.Debugw(ctx, "device-in-preprovisioning-state-reconcile-not-needed", log.Fields{"device-id": device.Id})
1406 agent.requestQueue.RequestComplete()
1407 return nil
1408 }
1409 // Change device transient state to FORCE_DELETING
1410 if err := agent.updateTransientState(ctx, core.DeviceTransientState_FORCE_DELETING); err != nil {
1411 logger.Errorw(ctx, "failure-updating-transient-state", log.Fields{"error": err, "device-id": agent.deviceID})
1412 agent.requestQueue.RequestComplete()
1413 return err
1414 }
1415
1416 // Ensure we have a valid grpc client available as we have just restarted
1417 deleteBackoff := backoff.NewExponentialBackOff()
1418 deleteBackoff.InitialInterval = agent.config.BackoffRetryInitialInterval
1419 deleteBackoff.MaxElapsedTime = agent.config.BackoffRetryMaxElapsedTime
1420 deleteBackoff.MaxInterval = agent.config.BackoffRetryMaxInterval
1421 var backoffTimer *time.Timer
1422 var err error
1423 var client adapter_service.AdapterServiceClient
1424retry:
1425 for {
1426 client, err = agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
1427 if err == nil {
1428 break retry
1429 }
1430 duration := deleteBackoff.NextBackOff()
1431 if duration == backoff.Stop {
1432 deleteBackoff.Reset()
1433 duration = deleteBackoff.NextBackOff()
1434 }
1435 backoffTimer = time.NewTimer(duration)
1436 select {
1437 case <-backoffTimer.C:
1438 logger.Debugw(ctx, "backoff-timer-expires", log.Fields{"device-id": agent.deviceID})
1439 case <-ctx.Done():
1440 err = ctx.Err()
1441 break retry
1442 }
1443 }
1444 if backoffTimer != nil && !backoffTimer.Stop() {
1445 select {
1446 case <-backoffTimer.C:
1447 default:
1448 }
1449 }
1450 if err != nil || client == nil {
1451 agent.requestQueue.RequestComplete()
1452 return err
1453 }
1454
1455 // Release the device lock to allow for device state update, if any
1456 agent.requestQueue.RequestComplete()
1457
1458 // Send the delete request to the adapter
1459 subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
1460 defer cancel()
khenaidoo6cd8e862021-11-12 16:33:43 -05001461 _, err = client.DeleteDevice(subCtx, device)
1462 agent.onForceDeleteResponse(subCtx, nil, nil, err)
1463 return err
khenaidoo68a5e0c2021-11-06 13:08:03 -04001464}
1465
khenaidood948f772021-08-11 17:49:24 -04001466func (agent *Agent) ReconcileDevice(ctx context.Context) {
khenaidoo68a5e0c2021-11-06 13:08:03 -04001467 // Do not reconcile if the device was in DELETE_FAILED transient state. Just invoke the force delete on that device.
1468 state := agent.getTransientState()
1469 logger.Debugw(ctx, "starting-reconcile", log.Fields{"device-id": agent.deviceID, "state": state})
1470 if agent.getTransientState() == core.DeviceTransientState_DELETE_FAILED {
1471 if err := agent.DeleteDevicePostAdapterRestart(ctx); err != nil {
1472 logger.Errorw(ctx, "delete-post-restart-failed", log.Fields{"error": err, "device-id": agent.deviceID})
1473 }
1474 return
1475 }
1476
khenaidood948f772021-08-11 17:49:24 -04001477 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
Maninder0aabf0c2021-03-17 14:55:14 +05301478 var desc string
Maninder0aabf0c2021-03-17 14:55:14 +05301479
1480 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
khenaidood948f772021-08-11 17:49:24 -04001481 agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc)
1482 return
1483 }
1484
1485 device := agent.getDeviceReadOnlyWithoutLock()
1486 if device.AdminState == voltha.AdminState_PREPROVISIONED {
1487 agent.requestQueue.RequestComplete()
1488 logger.Debugw(ctx, "device-in-preprovisioning-state-reconcile-not-needed", log.Fields{"device-id": device.Id})
Maninder0aabf0c2021-03-17 14:55:14 +05301489 return
1490 }
1491
Maninder2195ccc2021-06-23 20:23:01 +05301492 if !agent.proceedWithRequest(device) {
Maninder0aabf0c2021-03-17 14:55:14 +05301493 agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -04001494 err := fmt.Errorf("cannot complete operation as device deletion/reconciling is in progress or reconcile failed for device : %s", device.Id)
1495 logger.Errorw(ctx, "reconcile-failed", log.Fields{"error": err})
1496 agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc)
Maninder0aabf0c2021-03-17 14:55:14 +05301497 return
1498 }
1499
1500 //set transient state to RECONCILE IN PROGRESS
khenaidood948f772021-08-11 17:49:24 -04001501 err := agent.updateTransientState(ctx, core.DeviceTransientState_RECONCILE_IN_PROGRESS)
Maninder0aabf0c2021-03-17 14:55:14 +05301502 if err != nil {
1503 agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -04001504 logger.Errorw(ctx, "setting-transient-state-failed", log.Fields{"error": err})
1505 agent.logDeviceUpdate(ctx, nil, nil, requestStatus, nil, desc)
Maninder0aabf0c2021-03-17 14:55:14 +05301506 return
1507 }
1508
Maninder0aabf0c2021-03-17 14:55:14 +05301509 reconcilingBackoff := backoff.NewExponentialBackOff()
1510 reconcilingBackoff.InitialInterval = agent.config.BackoffRetryInitialInterval
1511 reconcilingBackoff.MaxElapsedTime = agent.config.BackoffRetryMaxElapsedTime
1512 reconcilingBackoff.MaxInterval = agent.config.BackoffRetryMaxInterval
1513
1514 //making here to keep lifecycle of this channel within the scope of retryReconcile
1515 agent.stopReconcilingMutex.Lock()
1516 agent.stopReconciling = make(chan int)
1517 agent.stopReconcilingMutex.Unlock()
1518
David K. Bainbridge482e4422021-06-30 12:23:42 -07001519 // defined outside the retry loop so it can be cleaned
1520 // up when the loop breaks
1521 var backoffTimer *time.Timer
1522
1523retry:
Maninder0aabf0c2021-03-17 14:55:14 +05301524 for {
David K. Bainbridge482e4422021-06-30 12:23:42 -07001525 // If the operations state of the device is RECONCILING_FAILED then we do not
1526 // want to continue to attempt reconciliation.
1527 deviceRef := agent.getDeviceReadOnlyWithoutLock()
1528 if deviceRef.OperStatus == common.OperStatus_RECONCILING_FAILED {
1529 logger.Warnw(ctx, "reconciling-failed-halting-retries",
1530 log.Fields{"device-id": device.Id})
1531 agent.requestQueue.RequestComplete()
1532 break retry
1533 }
1534
Maninder0aabf0c2021-03-17 14:55:14 +05301535 // Use an exponential back off to prevent getting into a tight loop
1536 duration := reconcilingBackoff.NextBackOff()
1537 //This case should never occur in default case as max elapsed time for backoff is 0(by default) , so it will never return stop
1538 if duration == backoff.Stop {
1539 // If we reach a maximum then warn and reset the backoff
1540 // timer and keep attempting.
1541 logger.Warnw(ctx, "maximum-reconciling-backoff-reached--resetting-backoff-timer",
1542 log.Fields{"max-reconciling-backoff": reconcilingBackoff.MaxElapsedTime,
1543 "device-id": device.Id})
1544 reconcilingBackoff.Reset()
1545 duration = reconcilingBackoff.NextBackOff()
1546 }
1547
David K. Bainbridge482e4422021-06-30 12:23:42 -07001548 backoffTimer = time.NewTimer(duration)
Maninder0aabf0c2021-03-17 14:55:14 +05301549
khenaidood948f772021-08-11 17:49:24 -04001550 logger.Debugw(ctx, "retrying-reconciling", log.Fields{"deviceID": device.Id, "endpoint": device.AdapterEndpoint})
1551 // Release lock before sending request to adapter
Maninder0aabf0c2021-03-17 14:55:14 +05301552 agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -04001553
1554 // Send a reconcile request to the adapter.
1555 err := agent.sendReconcileRequestToAdapter(ctx, device)
khenaidoo6cd8e862021-11-12 16:33:43 -05001556
1557 // Check the transient state after a response from the adapter. If a device delete
1558 // request was issued due to a callback during that time and failed then just delete
1559 // the device and stop the reconcile loop and invoke the device deletion
1560 if agent.getTransientState() == core.DeviceTransientState_DELETE_FAILED {
1561 if dErr := agent.DeleteDevicePostAdapterRestart(ctx); dErr != nil {
1562 logger.Errorw(ctx, "delete-post-restart-failed", log.Fields{"error": dErr, "device-id": agent.deviceID})
1563 }
1564 break retry
1565 }
1566
khenaidood948f772021-08-11 17:49:24 -04001567 if errors.Is(err, errContextExpired) || errors.Is(err, errReconcileAborted) {
1568 logger.Errorw(ctx, "reconcile-aborted", log.Fields{"error": err})
1569 requestStatus = &common.OperationResp{Code: common.OperationResp_OperationReturnCode(common.OperStatus_FAILED)}
1570 desc = "aborted"
1571 agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc)
1572 break retry
1573 }
Maninder0aabf0c2021-03-17 14:55:14 +05301574 if err != nil {
khenaidood948f772021-08-11 17:49:24 -04001575 agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc)
Maninder0aabf0c2021-03-17 14:55:14 +05301576 <-backoffTimer.C
1577 // backoffTimer expired continue
1578 // Take lock back before retrying
1579 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
khenaidood948f772021-08-11 17:49:24 -04001580 agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc)
David K. Bainbridge482e4422021-06-30 12:23:42 -07001581 break retry
Maninder0aabf0c2021-03-17 14:55:14 +05301582 }
1583 continue
1584 }
khenaidood948f772021-08-11 17:49:24 -04001585 // Success
1586 requestStatus = &common.OperationResp{Code: common.OperationResp_OPERATION_SUCCESS}
1587 desc = "adapter-response"
1588 agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc)
1589 break retry
David K. Bainbridge482e4422021-06-30 12:23:42 -07001590 }
1591
1592 // Retry loop is broken, so stop any timers and drain the channel
1593 if backoffTimer != nil && !backoffTimer.Stop() {
1594
1595 // As per documentation and stack overflow when a timer is stopped its
1596 // channel should be drained. The issue is that Stop returns false
1597 // either if the timer has already been fired "OR" if the timer can be
1598 // stopped before being fired. This means that in some cases the
1599 // channel has already be emptied so attempting to read from it means
1600 // a blocked thread. To get around this use a select so if the
1601 // channel is already empty the default case hits and we are not
1602 // blocked.
1603 select {
1604 case <-backoffTimer.C:
1605 default:
Maninder0aabf0c2021-03-17 14:55:14 +05301606 }
1607 }
1608}
1609
khenaidood948f772021-08-11 17:49:24 -04001610func (agent *Agent) sendReconcileRequestToAdapter(ctx context.Context, device *voltha.Device) error {
1611 logger.Debugw(ctx, "sending-reconcile-to-adapter", log.Fields{"device-id": device.Id, "endpoint": agent.adapterEndpoint})
1612 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
1613 if err != nil {
1614 return err
1615 }
1616 adapterResponse := make(chan error)
1617 go func() {
1618 _, err := client.ReconcileDevice(ctx, device)
1619 adapterResponse <- err
1620 }()
Maninder0aabf0c2021-03-17 14:55:14 +05301621 select {
1622 // wait for response
khenaidood948f772021-08-11 17:49:24 -04001623 case err := <-adapterResponse:
1624 if err != nil {
1625 return err
Maninder0aabf0c2021-03-17 14:55:14 +05301626 }
Maninder0aabf0c2021-03-17 14:55:14 +05301627 //In case of success quit retrying and wait for adapter to reset operation state of device
1628 agent.stopReconcilingMutex.Lock()
1629 agent.stopReconciling = nil
1630 agent.stopReconcilingMutex.Unlock()
1631 return nil
1632
1633 //if reconciling need to be stopped
1634 case _, ok := <-agent.stopReconciling:
1635 agent.stopReconcilingMutex.Lock()
1636 agent.stopReconciling = nil
1637 agent.stopReconcilingMutex.Unlock()
1638 if !ok {
1639 //channel-closed
khenaidood948f772021-08-11 17:49:24 -04001640 return fmt.Errorf("reconcile channel closed:%w", errReconcileAborted)
Maninder0aabf0c2021-03-17 14:55:14 +05301641 }
khenaidood948f772021-08-11 17:49:24 -04001642 return fmt.Errorf("reconciling aborted:%w", errReconcileAborted)
1643 // Context expired
1644 case <-ctx.Done():
1645 return fmt.Errorf("context expired:%s :%w", ctx.Err(), errContextExpired)
Maninder0aabf0c2021-03-17 14:55:14 +05301646 }
Maninder0aabf0c2021-03-17 14:55:14 +05301647}
1648
1649func (agent *Agent) reconcilingCleanup(ctx context.Context) error {
1650 var desc string
khenaidood948f772021-08-11 17:49:24 -04001651 var err error
Maninder0aabf0c2021-03-17 14:55:14 +05301652 operStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
khenaidood948f772021-08-11 17:49:24 -04001653 defer func() { agent.logDeviceUpdate(ctx, nil, nil, operStatus, err, desc) }()
1654
1655 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
1656 desc = "reconcile-cleanup-failed"
Maninder0aabf0c2021-03-17 14:55:14 +05301657 return err
1658 }
1659 defer agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -04001660 err = agent.updateTransientState(ctx, core.DeviceTransientState_NONE)
Maninder0aabf0c2021-03-17 14:55:14 +05301661 if err != nil {
khenaidood948f772021-08-11 17:49:24 -04001662 logger.Errorf(ctx, "transient-state-update-failed", log.Fields{"error": err})
Maninder0aabf0c2021-03-17 14:55:14 +05301663 return err
1664 }
khenaidood948f772021-08-11 17:49:24 -04001665 operStatus.Code = common.OperationResp_OPERATION_SUCCESS
Maninder0aabf0c2021-03-17 14:55:14 +05301666 return nil
1667}
khenaidood948f772021-08-11 17:49:24 -04001668
1669func (agent *Agent) isAdapterConnectionUp(ctx context.Context) bool {
1670 c, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
1671 return c != nil && err == nil
1672}
1673
1674func (agent *Agent) canDeviceRequestProceed(ctx context.Context) error {
1675 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
1676 return err
1677 }
1678 defer agent.requestQueue.RequestComplete()
1679 if agent.proceedWithRequest(agent.device) {
1680 return nil
1681 }
1682 return fmt.Errorf("device-cannot-process-request-%s", agent.deviceID)
1683}