blob: 0713f20798ce35ba67a5709c03c7d8030a28cdf5 [file] [log] [blame]
khenaidoob9203542018-09-17 22:56:37 -04001/*
2 * Copyright 2018-present Open Networking Foundation
3
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7
8 * http://www.apache.org/licenses/LICENSE-2.0
9
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
npujar1d86a522019-11-14 17:11:16 +053016
Kent Hagerman2b216042020-04-03 18:28:56 -040017package device
khenaidoob9203542018-09-17 22:56:37 -040018
19import (
20 "context"
Matteo Scandolo360605d2019-11-05 18:29:17 -080021 "encoding/hex"
Kent Hagerman4f355f52020-03-30 16:01:33 -040022 "errors"
khenaidoo3ab34882019-05-02 21:33:30 -040023 "fmt"
Mahir Gunyelb0343bf2021-05-11 14:14:26 -070024 "reflect"
25 "sync"
26 "time"
27
khenaidoo9beaaf12021-10-19 17:32:01 -040028 "github.com/opencord/voltha-protos/v5/go/adapter_service"
khenaidood948f772021-08-11 17:49:24 -040029 "github.com/opencord/voltha-protos/v5/go/core"
khenaidoo9beaaf12021-10-19 17:32:01 -040030 "github.com/opencord/voltha-protos/v5/go/omci"
khenaidood948f772021-08-11 17:49:24 -040031
Maninder0aabf0c2021-03-17 14:55:14 +053032 "github.com/cenkalti/backoff/v3"
Maninder9a1bc0d2020-10-26 11:34:02 +053033 "github.com/gogo/protobuf/proto"
Maninder9a1bc0d2020-10-26 11:34:02 +053034 "github.com/golang/protobuf/ptypes/empty"
Maninder0aabf0c2021-03-17 14:55:14 +053035 "github.com/opencord/voltha-go/rw_core/config"
khenaidoo68a5e0c2021-11-06 13:08:03 -040036 "github.com/opencord/voltha-go/rw_core/utils"
Maninder9a1bc0d2020-10-26 11:34:02 +053037 "google.golang.org/grpc/codes"
38 "google.golang.org/grpc/status"
Mahir Gunyeladdb66a2020-04-29 18:08:50 -070039
Himani Chawla2ba1c9c2020-10-07 13:19:03 +053040 "github.com/opencord/voltha-go/db/model"
Kent Hagerman2b216042020-04-03 18:28:56 -040041 "github.com/opencord/voltha-go/rw_core/core/adapter"
Mahir Gunyel03de0d32020-06-03 01:36:59 -070042 "github.com/opencord/voltha-go/rw_core/core/device/flow"
43 "github.com/opencord/voltha-go/rw_core/core/device/group"
Kent Hagerman2a07b862020-06-19 15:23:07 -040044 "github.com/opencord/voltha-go/rw_core/core/device/port"
Himani Chawla2ba1c9c2020-10-07 13:19:03 +053045 "github.com/opencord/voltha-go/rw_core/core/device/transientstate"
Scott Bakerb671a862019-10-24 10:53:40 -070046 coreutils "github.com/opencord/voltha-go/rw_core/utils"
khenaidood948f772021-08-11 17:49:24 -040047 "github.com/opencord/voltha-lib-go/v7/pkg/log"
48 "github.com/opencord/voltha-protos/v5/go/common"
khenaidoo9beaaf12021-10-19 17:32:01 -040049 ca "github.com/opencord/voltha-protos/v5/go/core_adapter"
khenaidood948f772021-08-11 17:49:24 -040050 "github.com/opencord/voltha-protos/v5/go/extension"
khenaidood948f772021-08-11 17:49:24 -040051 ofp "github.com/opencord/voltha-protos/v5/go/openflow_13"
52 "github.com/opencord/voltha-protos/v5/go/voltha"
khenaidoob9203542018-09-17 22:56:37 -040053)
54
khenaidood948f772021-08-11 17:49:24 -040055var errReconcileAborted = errors.New("reconcile aborted")
56var errContextExpired = errors.New("context expired")
khenaidoo68a5e0c2021-11-06 13:08:03 -040057var errNoConnection = errors.New("no connection")
khenaidood948f772021-08-11 17:49:24 -040058
Kent Hagerman2b216042020-04-03 18:28:56 -040059// Agent represents device agent attributes
60type Agent struct {
Maninder0aabf0c2021-03-17 14:55:14 +053061 deviceID string
62 parentID string
63 deviceType string
khenaidood948f772021-08-11 17:49:24 -040064 adapterEndpoint string
Maninder0aabf0c2021-03-17 14:55:14 +053065 isRootDevice bool
Maninder0aabf0c2021-03-17 14:55:14 +053066 adapterMgr *adapter.Manager
67 deviceMgr *Manager
68 dbProxy *model.Proxy
69 exitChannel chan int
70 device *voltha.Device
71 requestQueue *coreutils.RequestQueue
khenaidood948f772021-08-11 17:49:24 -040072 internalTimeout time.Duration
73 rpcTimeout time.Duration
Himani Chawla4b4bd252021-11-08 15:59:40 +053074 flowTimeout time.Duration
Maninder0aabf0c2021-03-17 14:55:14 +053075 startOnce sync.Once
76 stopOnce sync.Once
77 stopped bool
78 stopReconciling chan int
79 stopReconcilingMutex sync.RWMutex
80 config *config.RWCoreFlags
Mahir Gunyel03de0d32020-06-03 01:36:59 -070081
khenaidoo7585a962021-06-10 16:15:38 -040082 flowCache *flow.Cache
83 groupCache *group.Cache
Himani Chawla2ba1c9c2020-10-07 13:19:03 +053084 portLoader *port.Loader
85 transientStateLoader *transientstate.Loader
khenaidoob9203542018-09-17 22:56:37 -040086}
87
Kent Hagerman2b216042020-04-03 18:28:56 -040088//newAgent creates a new device agent. The device will be initialized when start() is called.
Himani Chawla4b4bd252021-11-08 15:59:40 +053089func newAgent(device *voltha.Device, deviceMgr *Manager, dbPath *model.Path, deviceProxy *model.Proxy, internalTimeout, rpcTimeout, flowTimeout time.Duration) *Agent {
Kent Hagerman2a07b862020-06-19 15:23:07 -040090 deviceID := device.Id
91 if deviceID == "" {
92 deviceID = coreutils.CreateDeviceID()
Stephane Barbarie1ab43272018-12-08 21:42:13 -050093 }
Scott Baker80678602019-11-14 16:57:36 -080094
Kent Hagerman2a07b862020-06-19 15:23:07 -040095 return &Agent{
Himani Chawla2ba1c9c2020-10-07 13:19:03 +053096 deviceID: deviceID,
Himani Chawla2ba1c9c2020-10-07 13:19:03 +053097 isRootDevice: device.Root,
98 parentID: device.ParentId,
99 deviceType: device.Type,
khenaidood948f772021-08-11 17:49:24 -0400100 adapterEndpoint: device.AdapterEndpoint,
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530101 deviceMgr: deviceMgr,
102 adapterMgr: deviceMgr.adapterMgr,
103 exitChannel: make(chan int, 1),
104 dbProxy: deviceProxy,
khenaidood948f772021-08-11 17:49:24 -0400105 internalTimeout: internalTimeout,
106 rpcTimeout: rpcTimeout,
Himani Chawla4b4bd252021-11-08 15:59:40 +0530107 flowTimeout: flowTimeout,
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530108 device: proto.Clone(device).(*voltha.Device),
109 requestQueue: coreutils.NewRequestQueue(),
Maninder0aabf0c2021-03-17 14:55:14 +0530110 config: deviceMgr.config,
khenaidoo7585a962021-06-10 16:15:38 -0400111 flowCache: flow.NewCache(),
112 groupCache: group.NewCache(),
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530113 portLoader: port.NewLoader(dbPath.SubPath("ports").Proxy(deviceID)),
114 transientStateLoader: transientstate.NewLoader(dbPath.SubPath("core").Proxy("transientstate"), deviceID),
Kent Hagerman2a07b862020-06-19 15:23:07 -0400115 }
khenaidoob9203542018-09-17 22:56:37 -0400116}
117
khenaidoo442e7c72020-03-10 16:13:48 -0400118// start() saves the device to the data model and registers for callbacks on that device if deviceToCreate!=nil.
119// Otherwise, it will load the data from the dB and setup the necessary callbacks and proxies. Returns the device that
Scott Baker80678602019-11-14 16:57:36 -0800120// was started.
khenaidoo7585a962021-06-10 16:15:38 -0400121func (agent *Agent) start(ctx context.Context, deviceExist bool, deviceToCreate *voltha.Device) (*voltha.Device, error) {
khenaidoo442e7c72020-03-10 16:13:48 -0400122 needToStart := false
123 if agent.startOnce.Do(func() { needToStart = true }); !needToStart {
Kent Hagermancba2f302020-07-28 13:37:36 -0400124 return agent.getDeviceReadOnly(ctx)
khenaidoo442e7c72020-03-10 16:13:48 -0400125 }
126 var startSucceeded bool
127 defer func() {
128 if !startSucceeded {
129 if err := agent.stop(ctx); err != nil {
Rohan Agrawal31f21802020-06-12 05:38:46 +0000130 logger.Errorw(ctx, "failed-to-cleanup-after-unsuccessful-start", log.Fields{"device-id": agent.deviceID, "error": err})
khenaidoo442e7c72020-03-10 16:13:48 -0400131 }
132 }
133 }()
khenaidoo7585a962021-06-10 16:15:38 -0400134 if deviceExist {
135 device := deviceToCreate
136 if device == nil {
137 // Load from dB
138 device = &voltha.Device{}
139 have, err := agent.dbProxy.Get(ctx, agent.deviceID, device)
140 if err != nil {
141 return nil, err
142 } else if !have {
143 return nil, status.Errorf(codes.NotFound, "device-%s", agent.deviceID)
144 }
khenaidood948f772021-08-11 17:49:24 -0400145 logger.Infow(ctx, "device-loaded-from-db", log.Fields{"device-id": agent.deviceID, "adapter-endpoint": device.AdapterEndpoint, "type": device.Type})
Thomas Lee Se5a44012019-11-07 20:32:24 +0530146 }
khenaidood948f772021-08-11 17:49:24 -0400147 agent.deviceType = device.Type
148 agent.adapterEndpoint = device.AdapterEndpoint
Kent Hagerman4f355f52020-03-30 16:01:33 -0400149 agent.device = proto.Clone(device).(*voltha.Device)
khenaidoo7585a962021-06-10 16:15:38 -0400150 // load the ports from KV to cache
Kent Hagerman2a07b862020-06-19 15:23:07 -0400151 agent.portLoader.Load(ctx)
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530152 agent.transientStateLoader.Load(ctx)
khenaidoo297cd252019-02-07 22:10:23 -0500153 } else {
Scott Baker80678602019-11-14 16:57:36 -0800154 // Create a new device
Maninder9a1bc0d2020-10-26 11:34:02 +0530155 var desc string
khenaidood948f772021-08-11 17:49:24 -0400156 var err error
Maninder9a1bc0d2020-10-26 11:34:02 +0530157 prevState := common.AdminState_UNKNOWN
158 currState := common.AdminState_UNKNOWN
khenaidood948f772021-08-11 17:49:24 -0400159 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
Maninder9a1bc0d2020-10-26 11:34:02 +0530160
khenaidood948f772021-08-11 17:49:24 -0400161 defer func() { agent.logDeviceUpdate(ctx, &prevState, &currState, requestStatus, err, desc) }()
Maninder9a1bc0d2020-10-26 11:34:02 +0530162
Kent Hagermanf5a67352020-04-30 15:15:26 -0400163 // Assumption is that AdminState, FlowGroups, and Flows are uninitialized since this
164 // is a new device, so populate them here before passing the device to ldProxy.Set.
Kent Hagerman2b216042020-04-03 18:28:56 -0400165 // agent.deviceId will also have been set during newAgent().
khenaidoo7585a962021-06-10 16:15:38 -0400166 device := (proto.Clone(deviceToCreate)).(*voltha.Device)
npujar1d86a522019-11-14 17:11:16 +0530167 device.Id = agent.deviceID
Scott Baker80678602019-11-14 16:57:36 -0800168 device.AdminState = voltha.AdminState_PREPROVISIONED
Maninder9a1bc0d2020-10-26 11:34:02 +0530169 currState = device.AdminState
Scott Baker80678602019-11-14 16:57:36 -0800170 if !deviceToCreate.GetRoot() && deviceToCreate.ProxyAddress != nil {
171 // Set the default vlan ID to the one specified by the parent adapter. It can be
172 // overwritten by the child adapter during a device update request
173 device.Vlan = deviceToCreate.ProxyAddress.ChannelId
174 }
175
khenaidood948f772021-08-11 17:49:24 -0400176 // Save the device to the model
177 if err = agent.dbProxy.Set(ctx, agent.deviceID, device); err != nil {
178 err = status.Errorf(codes.Aborted, "failed-adding-device-%s: %s", agent.deviceID, err)
179 return nil, err
khenaidoo297cd252019-02-07 22:10:23 -0500180 }
Mahir Gunyelb0343bf2021-05-11 14:14:26 -0700181 _ = agent.deviceMgr.Agent.SendDeviceStateChangeEvent(ctx, device.OperStatus, device.ConnectStatus, prevState, device, time.Now().Unix())
khenaidood948f772021-08-11 17:49:24 -0400182 requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
khenaidoo442e7c72020-03-10 16:13:48 -0400183 agent.device = device
khenaidoob9203542018-09-17 22:56:37 -0400184 }
khenaidoo442e7c72020-03-10 16:13:48 -0400185 startSucceeded = true
Rohan Agrawalcf12f202020-08-03 04:42:01 +0000186 log.EnrichSpan(ctx, log.Fields{"device-id": agent.deviceID})
Rohan Agrawal31f21802020-06-12 05:38:46 +0000187 logger.Debugw(ctx, "device-agent-started", log.Fields{"device-id": agent.deviceID})
khenaidoo442e7c72020-03-10 16:13:48 -0400188
Kent Hagermancba2f302020-07-28 13:37:36 -0400189 return agent.getDeviceReadOnly(ctx)
khenaidoob9203542018-09-17 22:56:37 -0400190}
191
khenaidoo4d4802d2018-10-04 21:59:49 -0400192// stop stops the device agent. Not much to do for now
Kent Hagerman2b216042020-04-03 18:28:56 -0400193func (agent *Agent) stop(ctx context.Context) error {
khenaidoo442e7c72020-03-10 16:13:48 -0400194 needToStop := false
195 if agent.stopOnce.Do(func() { needToStop = true }); !needToStop {
196 return nil
197 }
198 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
199 return err
200 }
201 defer agent.requestQueue.RequestComplete()
khenaidoo49085352020-01-13 19:15:43 -0500202
Himani Chawlab4c25912020-11-12 17:16:38 +0530203 logger.Infow(ctx, "stopping-device-agent", log.Fields{"device-id": agent.deviceID, "parent-id": agent.parentID})
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530204 // Remove the device transient loader
205 if err := agent.deleteTransientState(ctx); err != nil {
206 return err
207 }
khenaidoo0a822f92019-05-08 15:15:57 -0400208 // Remove the device from the KV store
Kent Hagermanf5a67352020-04-30 15:15:26 -0400209 if err := agent.dbProxy.Remove(ctx, agent.deviceID); err != nil {
khenaidoo442e7c72020-03-10 16:13:48 -0400210 return err
Thomas Lee Se5a44012019-11-07 20:32:24 +0530211 }
khenaidoo442e7c72020-03-10 16:13:48 -0400212
khenaidoo442e7c72020-03-10 16:13:48 -0400213 close(agent.exitChannel)
214
215 agent.stopped = true
216
Rohan Agrawal31f21802020-06-12 05:38:46 +0000217 logger.Infow(ctx, "device-agent-stopped", log.Fields{"device-id": agent.deviceID, "parent-id": agent.parentID})
khenaidoo442e7c72020-03-10 16:13:48 -0400218
219 return nil
khenaidoob9203542018-09-17 22:56:37 -0400220}
221
Scott Baker80678602019-11-14 16:57:36 -0800222// Load the most recent state from the KVStore for the device.
Kent Hagerman2b216042020-04-03 18:28:56 -0400223func (agent *Agent) reconcileWithKVStore(ctx context.Context) {
khenaidoo442e7c72020-03-10 16:13:48 -0400224 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
Rohan Agrawal31f21802020-06-12 05:38:46 +0000225 logger.Warnw(ctx, "request-aborted", log.Fields{"device-id": agent.deviceID, "error": err})
khenaidoo442e7c72020-03-10 16:13:48 -0400226 return
227 }
228 defer agent.requestQueue.RequestComplete()
Rohan Agrawal31f21802020-06-12 05:38:46 +0000229 logger.Debug(ctx, "reconciling-device-agent-devicetype")
Scott Baker80678602019-11-14 16:57:36 -0800230 // TODO: context timeout
Kent Hagerman4f355f52020-03-30 16:01:33 -0400231 device := &voltha.Device{}
Kent Hagermanf5a67352020-04-30 15:15:26 -0400232 if have, err := agent.dbProxy.Get(ctx, agent.deviceID, device); err != nil {
Rohan Agrawal31f21802020-06-12 05:38:46 +0000233 logger.Errorw(ctx, "kv-get-failed", log.Fields{"device-id": agent.deviceID, "error": err})
Thomas Lee Se5a44012019-11-07 20:32:24 +0530234 return
Kent Hagerman4f355f52020-03-30 16:01:33 -0400235 } else if !have {
236 return // not found in kv
Thomas Lee Se5a44012019-11-07 20:32:24 +0530237 }
Kent Hagerman4f355f52020-03-30 16:01:33 -0400238
khenaidood948f772021-08-11 17:49:24 -0400239 agent.deviceType = device.Type
Kent Hagerman4f355f52020-03-30 16:01:33 -0400240 agent.device = device
khenaidood948f772021-08-11 17:49:24 -0400241 agent.adapterEndpoint = device.AdapterEndpoint
Kent Hagerman2a07b862020-06-19 15:23:07 -0400242 agent.portLoader.Load(ctx)
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530243 agent.transientStateLoader.Load(ctx)
244
Rohan Agrawal31f21802020-06-12 05:38:46 +0000245 logger.Debugw(ctx, "reconciled-device-agent-devicetype", log.Fields{"device-id": agent.deviceID, "type": agent.deviceType})
Scott Baker80678602019-11-14 16:57:36 -0800246}
247
khenaidoo442e7c72020-03-10 16:13:48 -0400248// onSuccess is a common callback for scenarios where we receive a nil response following a request to an adapter
khenaidood948f772021-08-11 17:49:24 -0400249func (agent *Agent) onSuccess(ctx context.Context, prevState, currState *common.AdminState_Types, deviceUpdateLog bool) {
250 if deviceUpdateLog {
251 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_SUCCESS}
252 desc := "adapter-response"
253 agent.logDeviceUpdate(ctx, prevState, currState, requestStatus, nil, desc)
254 return
255 }
256 logger.Debugw(ctx, "successful-operation", log.Fields{"device-id": agent.deviceID, "rpc": coreutils.GetRPCMetadataFromContext(ctx)})
khenaidoo442e7c72020-03-10 16:13:48 -0400257}
258
259// onFailure is a common callback for scenarios where we receive an error response following a request to an adapter
260// and the only action required is to publish the failed result on kafka
khenaidood948f772021-08-11 17:49:24 -0400261func (agent *Agent) onFailure(ctx context.Context, err error, prevState, currState *common.AdminState_Types, deviceUpdateLog bool) {
262 // Send an event on kafka
263 rpce := agent.deviceMgr.NewRPCEvent(ctx, agent.deviceID, err.Error(), nil)
264 go agent.deviceMgr.SendRPCEvent(ctx, "RPC_ERROR_RAISE_EVENT", rpce,
265 voltha.EventCategory_COMMUNICATION, nil, time.Now().Unix())
khenaidoo442e7c72020-03-10 16:13:48 -0400266
khenaidood948f772021-08-11 17:49:24 -0400267 // Log the device update event
268 if deviceUpdateLog {
269 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
270 desc := "adapter-response"
271 agent.logDeviceUpdate(ctx, prevState, currState, requestStatus, err, desc)
272 return
khenaidoo442e7c72020-03-10 16:13:48 -0400273 }
khenaidood948f772021-08-11 17:49:24 -0400274 logger.Errorw(ctx, "failed-operation", log.Fields{"error": err, "device-id": agent.deviceID, "rpc": coreutils.GetRPCMetadataFromContext(ctx)})
khenaidoo442e7c72020-03-10 16:13:48 -0400275}
276
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530277// onDeleteSuccess is a common callback for scenarios where we receive a nil response following a delete request
278// to an adapter.
khenaidood948f772021-08-11 17:49:24 -0400279func (agent *Agent) onDeleteSuccess(ctx context.Context, prevState, currState *common.AdminState_Types) {
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530280 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
khenaidood948f772021-08-11 17:49:24 -0400281 logger.Errorw(ctx, "delete-device-failure", log.Fields{"device-id": agent.deviceID, "error": err})
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530282 }
283 previousDeviceTransientState := agent.getTransientState()
284 newDevice := agent.cloneDeviceWithoutLock()
285 if err := agent.updateDeviceWithTransientStateAndReleaseLock(ctx, newDevice,
khenaidood948f772021-08-11 17:49:24 -0400286 core.DeviceTransientState_DELETING_POST_ADAPTER_RESPONSE, previousDeviceTransientState); err != nil {
287 logger.Errorw(ctx, "delete-device-failure", log.Fields{"device-id": agent.deviceID, "error": err})
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530288 }
khenaidood948f772021-08-11 17:49:24 -0400289 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_SUCCESS}
290 desc := "adapter-response"
291 agent.logDeviceUpdate(ctx, prevState, currState, requestStatus, nil, desc)
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530292}
293
294// onDeleteFailure is a common callback for scenarios where we receive an error response following a delete request
295// to an adapter and the only action required is to return the error response.
khenaidood948f772021-08-11 17:49:24 -0400296func (agent *Agent) onDeleteFailure(ctx context.Context, err error, prevState, currState *common.AdminState_Types) {
297 logger.Errorw(ctx, "rpc-failed", log.Fields{"rpc": coreutils.GetRPCMetadataFromContext(ctx), "device-id": agent.deviceID, "error": err})
298
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530299 //Only updating of transient state is required, no transition.
khenaidood948f772021-08-11 17:49:24 -0400300 if er := agent.updateTransientState(ctx, core.DeviceTransientState_DELETE_FAILED); er != nil {
301 logger.Errorw(ctx, "failed-to-update-transient-state-as-delete-failed", log.Fields{"device-id": agent.deviceID, "error": er})
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530302 }
khenaidood948f772021-08-11 17:49:24 -0400303 rpce := agent.deviceMgr.NewRPCEvent(ctx, agent.deviceID, err.Error(), nil)
304 go agent.deviceMgr.SendRPCEvent(ctx, "RPC_ERROR_RAISE_EVENT", rpce,
305 voltha.EventCategory_COMMUNICATION, nil, time.Now().Unix())
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530306
khenaidood948f772021-08-11 17:49:24 -0400307 // Log the device update event
308 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
309 desc := "adapter-response"
310 agent.logDeviceUpdate(ctx, prevState, currState, requestStatus, err, desc)
Maninder9a1bc0d2020-10-26 11:34:02 +0530311}
312
Kent Hagermancba2f302020-07-28 13:37:36 -0400313// getDeviceReadOnly returns a device which MUST NOT be modified, but is safe to keep forever.
314func (agent *Agent) getDeviceReadOnly(ctx context.Context) (*voltha.Device, error) {
khenaidoo442e7c72020-03-10 16:13:48 -0400315 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
316 return nil, err
317 }
318 defer agent.requestQueue.RequestComplete()
Kent Hagermancba2f302020-07-28 13:37:36 -0400319 return agent.device, nil
khenaidoo92e62c52018-10-03 14:02:54 -0400320}
321
Kent Hagermancba2f302020-07-28 13:37:36 -0400322// getDeviceReadOnlyWithoutLock returns a device which MUST NOT be modified, but is safe to keep forever. This is very efficient.
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400323// The device lock MUST be held by the caller.
Kent Hagermancba2f302020-07-28 13:37:36 -0400324func (agent *Agent) getDeviceReadOnlyWithoutLock() *voltha.Device {
khenaidoo0db4c812020-05-27 15:27:30 -0400325 return agent.device
khenaidoo92e62c52018-10-03 14:02:54 -0400326}
327
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400328// cloneDeviceWithoutLock returns a copy of the device which is safe to modify.
329// The device lock MUST be held by the caller.
330func (agent *Agent) cloneDeviceWithoutLock() *voltha.Device {
331 return proto.Clone(agent.device).(*voltha.Device)
332}
333
khenaidood948f772021-08-11 17:49:24 -0400334func (agent *Agent) updateDeviceTypeAndEndpoint(ctx context.Context) error {
335 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
336 return err
337 }
338 changed := false
339 cloned := agent.cloneDeviceWithoutLock()
340 if cloned.Type == "" {
341 adapterType, err := agent.adapterMgr.GetAdapterType(cloned.Type)
342 if err != nil {
343 agent.requestQueue.RequestComplete()
344 return err
345 }
346 cloned.Type = adapterType
347 changed = true
348 }
349
350 if cloned.AdapterEndpoint == "" {
351 var err error
352 if cloned.AdapterEndpoint, err = agent.adapterMgr.GetAdapterEndpoint(ctx, cloned.Id, cloned.Type); err != nil {
353 agent.requestQueue.RequestComplete()
354 return err
355 }
356 agent.adapterEndpoint = cloned.AdapterEndpoint
357 changed = true
358 }
359
360 if changed {
361 return agent.updateDeviceAndReleaseLock(ctx, cloned)
362 }
363 agent.requestQueue.RequestComplete()
364 return nil
365}
366
khenaidoo3ab34882019-05-02 21:33:30 -0400367// enableDevice activates a preprovisioned or a disable device
Kent Hagerman2b216042020-04-03 18:28:56 -0400368func (agent *Agent) enableDevice(ctx context.Context) error {
Maninder9a1bc0d2020-10-26 11:34:02 +0530369 //To preserve and use oldDevice state as prev state in new device
khenaidood948f772021-08-11 17:49:24 -0400370 var err error
Maninder9a1bc0d2020-10-26 11:34:02 +0530371 var desc string
khenaidood948f772021-08-11 17:49:24 -0400372 var prevAdminState, currAdminState common.AdminState_Types
373 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
Maninder9a1bc0d2020-10-26 11:34:02 +0530374
khenaidood948f772021-08-11 17:49:24 -0400375 defer func() { agent.logDeviceUpdate(ctx, &prevAdminState, &currAdminState, requestStatus, err, desc) }()
Maninder9a1bc0d2020-10-26 11:34:02 +0530376
khenaidood948f772021-08-11 17:49:24 -0400377 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
khenaidoo442e7c72020-03-10 16:13:48 -0400378 return err
379 }
Himani Chawlab4c25912020-11-12 17:16:38 +0530380 logger.Debugw(ctx, "enable-device", log.Fields{"device-id": agent.deviceID})
khenaidoo21d51152019-02-01 13:48:37 -0500381
Kent Hagermancba2f302020-07-28 13:37:36 -0400382 oldDevice := agent.getDeviceReadOnlyWithoutLock()
khenaidood948f772021-08-11 17:49:24 -0400383 prevAdminState = oldDevice.AdminState
Maninder9a1bc0d2020-10-26 11:34:02 +0530384
Maninder2195ccc2021-06-23 20:23:01 +0530385 if !agent.proceedWithRequest(oldDevice) {
386 agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -0400387 err = status.Errorf(codes.FailedPrecondition, "cannot complete operation as device deletion is in progress or reconciling is in progress/failed: %s", agent.deviceID)
388 return err
Maninder2195ccc2021-06-23 20:23:01 +0530389 }
Mahir Gunyel92dd1212021-10-22 11:42:56 -0700390 //vol-4275 TST meeting 08/04/2021: Let EnableDevice to be called again if device is in FAILED operational state,
391 //even the admin state is ENABLED.
392 if oldDevice.AdminState == voltha.AdminState_ENABLED && oldDevice.OperStatus != voltha.OperStatus_FAILED {
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400393 logger.Warnw(ctx, "device-already-enabled", log.Fields{"device-id": agent.deviceID})
394 agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -0400395 err = status.Errorf(codes.FailedPrecondition, fmt.Sprintf("cannot-enable-an-already-enabled-device: %s", oldDevice.Id))
Matteo Scandolod525ae32020-04-02 17:27:29 -0700396 return err
npujar1d86a522019-11-14 17:11:16 +0530397 }
398
khenaidood948f772021-08-11 17:49:24 -0400399 // Verify whether there is a device type that supports this device type
400 _, err = agent.adapterMgr.GetAdapterType(oldDevice.Type)
401 if err != nil {
402 agent.requestQueue.RequestComplete()
403 return err
404 }
405
406 // Update device adapter endpoint if not set. This is set once by the Core and use as is by the adapters. E.g if this is a
407 // child device then the parent adapter will use this device's adapter endpoint (set here) to communicate with it.
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400408 newDevice := agent.cloneDeviceWithoutLock()
khenaidood948f772021-08-11 17:49:24 -0400409 if newDevice.AdapterEndpoint == "" {
410 if newDevice.AdapterEndpoint, err = agent.adapterMgr.GetAdapterEndpoint(ctx, newDevice.Id, newDevice.Type); err != nil {
411 agent.requestQueue.RequestComplete()
412 return err
413 }
414 agent.adapterEndpoint = newDevice.AdapterEndpoint
415 }
npujar1d86a522019-11-14 17:11:16 +0530416
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400417 // Update the Admin State and set the operational state to activating before sending the request to the Adapters
418 newDevice.AdminState = voltha.AdminState_ENABLED
419 newDevice.OperStatus = voltha.OperStatus_ACTIVATING
Maninder9a1bc0d2020-10-26 11:34:02 +0530420
khenaidoo442e7c72020-03-10 16:13:48 -0400421 // Adopt the device if it was in pre-provision state. In all other cases, try to re-enable it.
khenaidood948f772021-08-11 17:49:24 -0400422 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
khenaidoo442e7c72020-03-10 16:13:48 -0400423 if err != nil {
khenaidood948f772021-08-11 17:49:24 -0400424 logger.Errorw(ctx, "grpc-client-nil",
425 log.Fields{
426 "error": err,
427 "device-id": agent.deviceID,
428 "device-type": agent.deviceType,
429 "adapter-endpoint": newDevice.AdapterEndpoint,
430 })
431 agent.requestQueue.RequestComplete()
khenaidoo442e7c72020-03-10 16:13:48 -0400432 return err
433 }
khenaidood948f772021-08-11 17:49:24 -0400434 subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
435 requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS
436 go func() {
437 defer cancel()
438 var err error
439 if oldDevice.AdminState == voltha.AdminState_PREPROVISIONED {
440 _, err = client.AdoptDevice(subCtx, newDevice)
441 } else {
442 _, err = client.ReEnableDevice(subCtx, newDevice)
443 }
444 if err == nil {
445 agent.onSuccess(subCtx, nil, nil, true)
446 } else {
447 agent.onFailure(subCtx, err, nil, nil, true)
Himani Chawlab4c25912020-11-12 17:16:38 +0530448 }
449 }()
khenaidood948f772021-08-11 17:49:24 -0400450
451 // Update device
452 if err = agent.updateDeviceAndReleaseLock(ctx, newDevice); err != nil {
453 return err
khenaidoo2c6a0992019-04-29 13:46:56 -0400454 }
khenaidood948f772021-08-11 17:49:24 -0400455 currAdminState = newDevice.AdminState
456 return nil
khenaidoo2c6a0992019-04-29 13:46:56 -0400457}
458
A R Karthick5c28f552019-12-11 22:47:44 -0800459//addFlowsAndGroups adds the "newFlows" and "newGroups" from the existing flows/groups and sends the update to the
460//adapters
khenaidoo9beaaf12021-10-19 17:32:01 -0400461func (agent *Agent) addFlowsAndGroups(ctx context.Context, newFlows []*ofp.OfpFlowStats, newGroups []*ofp.OfpGroupEntry, flowMetadata *ofp.FlowMetadata) error {
Mahir Gunyel03de0d32020-06-03 01:36:59 -0700462 var flwResponse, grpResponse coreutils.Response
463 var err error
464 //if new flow list is empty then the called function returns quickly
465 if flwResponse, err = agent.addFlowsToAdapter(ctx, newFlows, flowMetadata); err != nil {
A R Karthick5c28f552019-12-11 22:47:44 -0800466 return err
467 }
Mahir Gunyel03de0d32020-06-03 01:36:59 -0700468 //if new group list is empty then the called function returns quickly
469 if grpResponse, err = agent.addGroupsToAdapter(ctx, newGroups, flowMetadata); err != nil {
470 return err
471 }
Himani Chawla4b4bd252021-11-08 15:59:40 +0530472 if errs := coreutils.WaitForNilOrErrorResponses(agent.flowTimeout, flwResponse, grpResponse); errs != nil {
khenaidood948f772021-08-11 17:49:24 -0400473 logger.Warnw(ctx, "adapter-response", log.Fields{"device-id": agent.deviceID, "result": errs})
khenaidoo442e7c72020-03-10 16:13:48 -0400474 return status.Errorf(codes.Aborted, "flow-failure-device-%s", agent.deviceID)
khenaidoo0458db62019-06-20 08:50:36 -0400475 }
khenaidoo0458db62019-06-20 08:50:36 -0400476 return nil
477}
478
A R Karthick5c28f552019-12-11 22:47:44 -0800479//deleteFlowsAndGroups removes the "flowsToDel" and "groupsToDel" from the existing flows/groups and sends the update to the
480//adapters
khenaidoo9beaaf12021-10-19 17:32:01 -0400481func (agent *Agent) deleteFlowsAndGroups(ctx context.Context, flowsToDel []*ofp.OfpFlowStats, groupsToDel []*ofp.OfpGroupEntry, flowMetadata *ofp.FlowMetadata) error {
Mahir Gunyel03de0d32020-06-03 01:36:59 -0700482 var flwResponse, grpResponse coreutils.Response
483 var err error
484 if flwResponse, err = agent.deleteFlowsFromAdapter(ctx, flowsToDel, flowMetadata); err != nil {
A R Karthick5c28f552019-12-11 22:47:44 -0800485 return err
486 }
Mahir Gunyel03de0d32020-06-03 01:36:59 -0700487 if grpResponse, err = agent.deleteGroupsFromAdapter(ctx, groupsToDel, flowMetadata); err != nil {
488 return err
489 }
490
Himani Chawla4b4bd252021-11-08 15:59:40 +0530491 if res := coreutils.WaitForNilOrErrorResponses(agent.flowTimeout, flwResponse, grpResponse); res != nil {
khenaidoo0458db62019-06-20 08:50:36 -0400492 return status.Errorf(codes.Aborted, "errors-%s", res)
493 }
494 return nil
khenaidoo0458db62019-06-20 08:50:36 -0400495}
496
A R Karthick5c28f552019-12-11 22:47:44 -0800497//updateFlowsAndGroups replaces the existing flows and groups with "updatedFlows" and "updatedGroups" respectively. It
498//also sends the updates to the adapters
khenaidoo9beaaf12021-10-19 17:32:01 -0400499func (agent *Agent) updateFlowsAndGroups(ctx context.Context, updatedFlows []*ofp.OfpFlowStats, updatedGroups []*ofp.OfpGroupEntry, flowMetadata *ofp.FlowMetadata) error {
Mahir Gunyel03de0d32020-06-03 01:36:59 -0700500 var flwResponse, grpResponse coreutils.Response
501 var err error
502 if flwResponse, err = agent.updateFlowsToAdapter(ctx, updatedFlows, flowMetadata); err != nil {
A R Karthick5c28f552019-12-11 22:47:44 -0800503 return err
504 }
Mahir Gunyel03de0d32020-06-03 01:36:59 -0700505 if grpResponse, err = agent.updateGroupsToAdapter(ctx, updatedGroups, flowMetadata); err != nil {
506 return err
507 }
508
Himani Chawla4b4bd252021-11-08 15:59:40 +0530509 if res := coreutils.WaitForNilOrErrorResponses(agent.flowTimeout, flwResponse, grpResponse); res != nil {
khenaidoo0458db62019-06-20 08:50:36 -0400510 return status.Errorf(codes.Aborted, "errors-%s", res)
511 }
512 return nil
khenaidoo19d7b632018-10-30 10:49:50 -0400513}
514
khenaidoo4d4802d2018-10-04 21:59:49 -0400515//disableDevice disable a device
Kent Hagerman2b216042020-04-03 18:28:56 -0400516func (agent *Agent) disableDevice(ctx context.Context) error {
khenaidood948f772021-08-11 17:49:24 -0400517 var err error
Maninder9a1bc0d2020-10-26 11:34:02 +0530518 var desc string
khenaidood948f772021-08-11 17:49:24 -0400519 var prevAdminState, currAdminState common.AdminState_Types
520 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
521 defer func() { agent.logDeviceUpdate(ctx, &prevAdminState, &currAdminState, requestStatus, err, desc) }()
Maninder9a1bc0d2020-10-26 11:34:02 +0530522
khenaidood948f772021-08-11 17:49:24 -0400523 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
khenaidoo442e7c72020-03-10 16:13:48 -0400524 return err
525 }
Himani Chawlab4c25912020-11-12 17:16:38 +0530526 logger.Debugw(ctx, "disable-device", log.Fields{"device-id": agent.deviceID})
khenaidoo6e55d9e2019-12-12 18:26:26 -0500527
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400528 cloned := agent.cloneDeviceWithoutLock()
khenaidood948f772021-08-11 17:49:24 -0400529 prevAdminState = agent.device.AdminState
khenaidoo6e55d9e2019-12-12 18:26:26 -0500530
Maninder2195ccc2021-06-23 20:23:01 +0530531 if !agent.proceedWithRequest(cloned) {
khenaidood948f772021-08-11 17:49:24 -0400532 err = status.Errorf(codes.FailedPrecondition, "cannot complete operation as device deletion is in progress or reconciling is in progress/failed: %s", agent.deviceID)
Maninder2195ccc2021-06-23 20:23:01 +0530533 agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -0400534 return err
Maninder2195ccc2021-06-23 20:23:01 +0530535 }
536
khenaidoo6e55d9e2019-12-12 18:26:26 -0500537 if cloned.AdminState == voltha.AdminState_DISABLED {
Maninder9a1bc0d2020-10-26 11:34:02 +0530538 desc = "device-already-disabled"
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400539 agent.requestQueue.RequestComplete()
npujar1d86a522019-11-14 17:11:16 +0530540 return nil
541 }
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530542 if cloned.AdminState == voltha.AdminState_PREPROVISIONED {
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400543 agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -0400544 err = status.Errorf(codes.FailedPrecondition, "deviceId:%s, invalid-admin-state:%s", agent.deviceID, cloned.AdminState)
545 return err
npujar1d86a522019-11-14 17:11:16 +0530546 }
Maninder0aabf0c2021-03-17 14:55:14 +0530547
npujar1d86a522019-11-14 17:11:16 +0530548 // Update the Admin State and operational state before sending the request out
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400549 cloned.AdminState = voltha.AdminState_DISABLED
550 cloned.OperStatus = voltha.OperStatus_UNKNOWN
Maninder9a1bc0d2020-10-26 11:34:02 +0530551
khenaidood948f772021-08-11 17:49:24 -0400552 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
khenaidoo442e7c72020-03-10 16:13:48 -0400553 if err != nil {
khenaidood948f772021-08-11 17:49:24 -0400554 logger.Errorw(ctx, "grpc-client-nil",
555 log.Fields{
556 "error": err,
557 "device-id": agent.deviceID,
558 "device-type": agent.deviceType,
559 "adapter-endpoint": cloned.AdapterEndpoint,
560 })
561 agent.requestQueue.RequestComplete()
npujar1d86a522019-11-14 17:11:16 +0530562 return err
khenaidoo0a822f92019-05-08 15:15:57 -0400563 }
khenaidood948f772021-08-11 17:49:24 -0400564 subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
565 requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS
566 go func() {
567 defer cancel()
568 _, err := client.DisableDevice(subCtx, cloned)
569 if err == nil {
570 agent.onSuccess(subCtx, nil, nil, true)
571 } else {
572 agent.onFailure(subCtx, err, nil, nil, true)
573 }
574 }()
Maninder9a1bc0d2020-10-26 11:34:02 +0530575
khenaidood948f772021-08-11 17:49:24 -0400576 // Update device
577 if err = agent.updateDeviceAndReleaseLock(ctx, cloned); err != nil {
578 return err
579 }
580 currAdminState = cloned.AdminState
khenaidoo0a822f92019-05-08 15:15:57 -0400581
khenaidoo92e62c52018-10-03 14:02:54 -0400582 return nil
583}
584
Kent Hagerman2b216042020-04-03 18:28:56 -0400585func (agent *Agent) rebootDevice(ctx context.Context) error {
Maninder9a1bc0d2020-10-26 11:34:02 +0530586 var desc string
khenaidood948f772021-08-11 17:49:24 -0400587 var err error
588 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
589 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
Maninder9a1bc0d2020-10-26 11:34:02 +0530590
khenaidood948f772021-08-11 17:49:24 -0400591 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
Maninder9a1bc0d2020-10-26 11:34:02 +0530592 desc = err.Error()
npujar1d86a522019-11-14 17:11:16 +0530593 return err
khenaidoo4d4802d2018-10-04 21:59:49 -0400594 }
khenaidoo442e7c72020-03-10 16:13:48 -0400595 defer agent.requestQueue.RequestComplete()
Himani Chawlab4c25912020-11-12 17:16:38 +0530596 logger.Debugw(ctx, "reboot-device", log.Fields{"device-id": agent.deviceID})
khenaidoo442e7c72020-03-10 16:13:48 -0400597
Kent Hagermancba2f302020-07-28 13:37:36 -0400598 device := agent.getDeviceReadOnlyWithoutLock()
Maninder2195ccc2021-06-23 20:23:01 +0530599
600 if !agent.proceedWithRequest(device) {
khenaidood948f772021-08-11 17:49:24 -0400601 err = status.Errorf(codes.FailedPrecondition, "cannot complete operation as device deletion is in progress or reconciling is in progress/failed:%s", agent.deviceID)
khenaidoo442e7c72020-03-10 16:13:48 -0400602 return err
603 }
Maninder9a1bc0d2020-10-26 11:34:02 +0530604
khenaidood948f772021-08-11 17:49:24 -0400605 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
606 if err != nil {
607 logger.Errorw(ctx, "grpc-client-nil",
608 log.Fields{
609 "error": err,
610 "device-id": agent.deviceID,
611 "device-type": agent.deviceType,
612 "adapter-endpoint": device.AdapterEndpoint,
613 })
614 return err
615 }
616 subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
617 requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS
618 go func() {
619 defer cancel()
620 _, err := client.RebootDevice(subCtx, device)
621 if err == nil {
622 agent.onSuccess(subCtx, nil, nil, true)
623 } else {
624 agent.onFailure(subCtx, err, nil, nil, true)
625 }
626 }()
khenaidoo4d4802d2018-10-04 21:59:49 -0400627 return nil
628}
629
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530630func (agent *Agent) deleteDeviceForce(ctx context.Context) error {
Himani Chawlab4c25912020-11-12 17:16:38 +0530631 logger.Debugw(ctx, "delete-device-force", log.Fields{"device-id": agent.deviceID})
Maninder9a1bc0d2020-10-26 11:34:02 +0530632
633 var desc string
khenaidood948f772021-08-11 17:49:24 -0400634 var err error
635 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
636 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
Maninder9a1bc0d2020-10-26 11:34:02 +0530637
khenaidood948f772021-08-11 17:49:24 -0400638 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530639 return err
640 }
641 // Get the device Transient state, return err if it is DELETING
642 previousDeviceTransientState := agent.getTransientState()
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530643 device := agent.cloneDeviceWithoutLock()
khenaidood948f772021-08-11 17:49:24 -0400644 if !agent.isForceDeletingAllowed(previousDeviceTransientState, device) {
645 agent.requestQueue.RequestComplete()
646 err = status.Error(codes.FailedPrecondition, fmt.Sprintf("deviceId:%s, force deletion is in progress", agent.deviceID))
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530647 return err
648 }
Himani Chawlab4c25912020-11-12 17:16:38 +0530649
khenaidood948f772021-08-11 17:49:24 -0400650 previousAdminState := device.AdminState
651 if previousAdminState != common.AdminState_PREPROVISIONED {
khenaidoo9beaaf12021-10-19 17:32:01 -0400652 var client adapter_service.AdapterServiceClient
khenaidood948f772021-08-11 17:49:24 -0400653 client, err = agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530654 if err != nil {
khenaidood948f772021-08-11 17:49:24 -0400655 logger.Errorw(ctx, "grpc-client-nil",
656 log.Fields{
657 "error": err,
658 "device-id": agent.deviceID,
659 "device-type": agent.deviceType,
660 "adapter-endpoint": device.AdapterEndpoint,
661 })
662 agent.requestQueue.RequestComplete()
khenaidoo68a5e0c2021-11-06 13:08:03 -0400663 return fmt.Errorf("remote-not-reachable %w", errNoConnection)
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530664 }
khenaidood948f772021-08-11 17:49:24 -0400665 subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
666 requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS
667 go func() {
668 defer cancel()
669 _, err := client.DeleteDevice(subCtx, device)
670 if err == nil {
671 agent.onSuccess(subCtx, nil, nil, true)
672 } else {
673 agent.onFailure(subCtx, err, nil, nil, true)
674 }
675 }()
676 }
677
678 // Update device
679 if err = agent.updateDeviceWithTransientStateAndReleaseLock(ctx, device,
680 core.DeviceTransientState_FORCE_DELETING, previousDeviceTransientState); err != nil {
681 return err
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530682 }
683 return nil
684}
685
Kent Hagerman2b216042020-04-03 18:28:56 -0400686func (agent *Agent) deleteDevice(ctx context.Context) error {
Himani Chawlab4c25912020-11-12 17:16:38 +0530687 logger.Debugw(ctx, "delete-device", log.Fields{"device-id": agent.deviceID})
Maninder9a1bc0d2020-10-26 11:34:02 +0530688
689 var desc string
khenaidood948f772021-08-11 17:49:24 -0400690 var err error
691 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
692 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
Maninder9a1bc0d2020-10-26 11:34:02 +0530693
khenaidood948f772021-08-11 17:49:24 -0400694 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
Maninder9a1bc0d2020-10-26 11:34:02 +0530695 desc = err.Error()
khenaidoo442e7c72020-03-10 16:13:48 -0400696 return err
697 }
Maninder0aabf0c2021-03-17 14:55:14 +0530698
Maninder2195ccc2021-06-23 20:23:01 +0530699 device := agent.cloneDeviceWithoutLock()
700
701 if !agent.proceedWithRequest(device) {
Maninder0aabf0c2021-03-17 14:55:14 +0530702 agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -0400703 err = status.Errorf(codes.FailedPrecondition, "cannot complete operation as device deletion is in progress or reconciling is in progress/failed: %s", agent.deviceID)
704 return err
Maninder0aabf0c2021-03-17 14:55:14 +0530705 }
706
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530707 // Get the device Transient state, return err if it is DELETING
708 previousDeviceTransientState := agent.getTransientState()
khenaidoo6e55d9e2019-12-12 18:26:26 -0500709
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530710 previousAdminState := device.AdminState
711 // Change the device transient state to DELETING_FROM_ADAPTER state till the device is removed from adapters.
khenaidood948f772021-08-11 17:49:24 -0400712 currentDeviceTransientState := core.DeviceTransientState_DELETING_FROM_ADAPTER
khenaidoo442e7c72020-03-10 16:13:48 -0400713
khenaidood948f772021-08-11 17:49:24 -0400714 if previousAdminState == common.AdminState_PREPROVISIONED {
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530715 // Change the state to DELETING POST ADAPTER RESPONSE directly as adapters have no info of the device.
khenaidood948f772021-08-11 17:49:24 -0400716 currentDeviceTransientState = core.DeviceTransientState_DELETING_POST_ADAPTER_RESPONSE
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530717 }
khenaidood948f772021-08-11 17:49:24 -0400718 // If the device was in pre-prov state (only parent device are in that state) then do not send the request to the
719 // adapter
720 if previousAdminState != common.AdminState_PREPROVISIONED {
khenaidoo9beaaf12021-10-19 17:32:01 -0400721 var client adapter_service.AdapterServiceClient
khenaidood948f772021-08-11 17:49:24 -0400722 client, err = agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
723 if err != nil {
724 logger.Errorw(ctx, "grpc-client-nil",
725 log.Fields{
726 "error": err,
727 "device-id": agent.deviceID,
728 "device-type": agent.deviceType,
729 "adapter-endpoint": device.AdapterEndpoint,
730 })
731 agent.requestQueue.RequestComplete()
732 return err
733 }
734 subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
735 requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS
736 go func() {
737 defer cancel()
738 _, err := client.DeleteDevice(subCtx, device)
739 if err == nil {
740 agent.onDeleteSuccess(subCtx, nil, nil)
741 } else {
742 agent.onDeleteFailure(subCtx, err, nil, nil)
743 }
744 }()
745 }
746
747 // Update device and release lock
748 if err = agent.updateDeviceWithTransientStateAndReleaseLock(ctx, device,
Himani Chawlab4c25912020-11-12 17:16:38 +0530749 currentDeviceTransientState, previousDeviceTransientState); err != nil {
Maninder9a1bc0d2020-10-26 11:34:02 +0530750 desc = err.Error()
npujar1d86a522019-11-14 17:11:16 +0530751 return err
752 }
Himani Chawlab4c25912020-11-12 17:16:38 +0530753
khenaidoo4d4802d2018-10-04 21:59:49 -0400754 return nil
755}
756
Kent Hagerman2b216042020-04-03 18:28:56 -0400757func (agent *Agent) setParentID(ctx context.Context, device *voltha.Device, parentID string) error {
khenaidoo442e7c72020-03-10 16:13:48 -0400758 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
759 return err
760 }
Himani Chawlab4c25912020-11-12 17:16:38 +0530761 logger.Debugw(ctx, "set-parent-id", log.Fields{"device-id": device.Id, "parent-id": parentID})
khenaidoo6e55d9e2019-12-12 18:26:26 -0500762
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400763 cloned := agent.cloneDeviceWithoutLock()
npujar1d86a522019-11-14 17:11:16 +0530764 cloned.ParentId = parentID
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400765 return agent.updateDeviceAndReleaseLock(ctx, cloned)
khenaidooad06fd72019-10-28 12:26:05 -0400766}
767
khenaidoo442e7c72020-03-10 16:13:48 -0400768// getSwitchCapability retrieves the switch capability of a parent device
khenaidoo9beaaf12021-10-19 17:32:01 -0400769func (agent *Agent) getSwitchCapability(ctx context.Context) (*ca.SwitchCapability, error) {
Himani Chawlab4c25912020-11-12 17:16:38 +0530770 logger.Debugw(ctx, "get-switch-capability", log.Fields{"device-id": agent.deviceID})
khenaidoo442e7c72020-03-10 16:13:48 -0400771
Kent Hagermancba2f302020-07-28 13:37:36 -0400772 device, err := agent.getDeviceReadOnly(ctx)
khenaidoo442e7c72020-03-10 16:13:48 -0400773 if err != nil {
khenaidoob9203542018-09-17 22:56:37 -0400774 return nil, err
khenaidoob9203542018-09-17 22:56:37 -0400775 }
khenaidood948f772021-08-11 17:49:24 -0400776
777 // Get the gRPC client
778 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
khenaidoo442e7c72020-03-10 16:13:48 -0400779 if err != nil {
780 return nil, err
781 }
782
khenaidood948f772021-08-11 17:49:24 -0400783 return client.GetOfpDeviceInfo(ctx, device)
khenaidoob9203542018-09-17 22:56:37 -0400784}
785
khenaidood948f772021-08-11 17:49:24 -0400786func (agent *Agent) onPacketFailure(ctx context.Context, err error, packet *ofp.OfpPacketOut) {
787 logger.Errorw(ctx, "packet-out-error", log.Fields{
khenaidoo442e7c72020-03-10 16:13:48 -0400788 "device-id": agent.deviceID,
khenaidood948f772021-08-11 17:49:24 -0400789 "error": err.Error(),
790 "packet": hex.EncodeToString(packet.Data),
khenaidoo442e7c72020-03-10 16:13:48 -0400791 })
khenaidood948f772021-08-11 17:49:24 -0400792 rpce := agent.deviceMgr.NewRPCEvent(ctx, agent.deviceID, err.Error(), nil)
793 go agent.deviceMgr.SendRPCEvent(ctx, "RPC_ERROR_RAISE_EVENT", rpce,
794 voltha.EventCategory_COMMUNICATION, nil, time.Now().Unix())
khenaidoo442e7c72020-03-10 16:13:48 -0400795}
796
Kent Hagerman2b216042020-04-03 18:28:56 -0400797func (agent *Agent) packetOut(ctx context.Context, outPort uint32, packet *ofp.OfpPacketOut) error {
Scott Baker80678602019-11-14 16:57:36 -0800798 if agent.deviceType == "" {
npujar467fe752020-01-16 20:17:45 +0530799 agent.reconcileWithKVStore(ctx)
Scott Baker80678602019-11-14 16:57:36 -0800800 }
khenaidoofdbad6e2018-11-06 22:26:38 -0500801 // Send packet to adapter
khenaidood948f772021-08-11 17:49:24 -0400802 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
khenaidoo442e7c72020-03-10 16:13:48 -0400803 if err != nil {
khenaidood948f772021-08-11 17:49:24 -0400804 logger.Errorw(ctx, "grpc-client-nil",
805 log.Fields{
806 "error": err,
807 "device-id": agent.deviceID,
808 "device-type": agent.deviceType,
809 })
810 return err
khenaidoofdbad6e2018-11-06 22:26:38 -0500811 }
khenaidood948f772021-08-11 17:49:24 -0400812 subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
813 go func() {
814 defer cancel()
khenaidoo9beaaf12021-10-19 17:32:01 -0400815 _, err := client.SendPacketOut(subCtx, &ca.PacketOut{
khenaidood948f772021-08-11 17:49:24 -0400816 DeviceId: agent.deviceID,
817 EgressPortNo: outPort,
818 Packet: packet,
819 })
820 if err == nil {
821 agent.onSuccess(subCtx, nil, nil, false)
822 } else {
823 agent.onPacketFailure(subCtx, err, packet)
824 }
825 }()
khenaidoofdbad6e2018-11-06 22:26:38 -0500826 return nil
827}
828
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400829func (agent *Agent) updateDeviceUsingAdapterData(ctx context.Context, device *voltha.Device) error {
khenaidood948f772021-08-11 17:49:24 -0400830 var err error
831 var desc string
832 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
833 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
834
835 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400836 return err
837 }
Himani Chawlab4c25912020-11-12 17:16:38 +0530838 logger.Debugw(ctx, "update-device-using-adapter-data", log.Fields{"device-id": device.Id})
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400839
840 cloned := agent.cloneDeviceWithoutLock()
Mahir Gunyel8e2707d2019-07-25 00:36:21 -0700841 cloned.Root = device.Root
842 cloned.Vendor = device.Vendor
843 cloned.Model = device.Model
844 cloned.SerialNumber = device.SerialNumber
845 cloned.MacAddress = device.MacAddress
846 cloned.Vlan = device.Vlan
847 cloned.Reason = device.Reason
Andrea Campanella025667e2021-01-14 11:50:07 +0100848 cloned.ImageDownloads = device.ImageDownloads
khenaidood948f772021-08-11 17:49:24 -0400849 cloned.OperStatus = device.OperStatus
850 cloned.ConnectStatus = device.ConnectStatus
851 if err = agent.updateDeviceAndReleaseLock(ctx, cloned); err == nil {
852 requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
853 }
854 return err
khenaidoo43c82122018-11-22 18:38:28 -0500855}
856
Kent Hagerman2b216042020-04-03 18:28:56 -0400857func (agent *Agent) updateDeviceStatus(ctx context.Context, operStatus voltha.OperStatus_Types, connStatus voltha.ConnectStatus_Types) error {
khenaidood948f772021-08-11 17:49:24 -0400858 var err error
859 var desc string
860 opStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
861 defer func() { agent.logDeviceUpdate(ctx, nil, nil, opStatus, err, desc) }()
862
863 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
khenaidoo442e7c72020-03-10 16:13:48 -0400864 return err
865 }
khenaidoo6e55d9e2019-12-12 18:26:26 -0500866
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400867 cloned := agent.cloneDeviceWithoutLock()
npujar1d86a522019-11-14 17:11:16 +0530868 // Ensure the enums passed in are valid - they will be invalid if they are not set when this function is invoked
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400869 if s, ok := voltha.ConnectStatus_Types_name[int32(connStatus)]; ok {
khenaidood948f772021-08-11 17:49:24 -0400870 logger.Debugw(ctx, "update-device-conn-status", log.Fields{"ok": ok, "val": s})
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400871 cloned.ConnectStatus = connStatus
npujar1d86a522019-11-14 17:11:16 +0530872 }
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400873 if s, ok := voltha.OperStatus_Types_name[int32(operStatus)]; ok {
khenaidood948f772021-08-11 17:49:24 -0400874 logger.Debugw(ctx, "update-device-oper-status", log.Fields{"ok": ok, "val": s})
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400875 cloned.OperStatus = operStatus
npujar1d86a522019-11-14 17:11:16 +0530876 }
Himani Chawlab4c25912020-11-12 17:16:38 +0530877 logger.Debugw(ctx, "update-device-status", log.Fields{"device-id": cloned.Id, "oper-status": cloned.OperStatus, "connect-status": cloned.ConnectStatus})
npujar1d86a522019-11-14 17:11:16 +0530878 // Store the device
khenaidood948f772021-08-11 17:49:24 -0400879 if err = agent.updateDeviceAndReleaseLock(ctx, cloned); err == nil {
880 opStatus.Code = common.OperationResp_OPERATION_SUCCESS
881 }
882 return err
khenaidoo92e62c52018-10-03 14:02:54 -0400883}
884
khenaidoob9203542018-09-17 22:56:37 -0400885// TODO: A generic device update by attribute
Kent Hagerman2b216042020-04-03 18:28:56 -0400886func (agent *Agent) updateDeviceAttribute(ctx context.Context, name string, value interface{}) {
khenaidoob9203542018-09-17 22:56:37 -0400887 if value == nil {
888 return
889 }
khenaidoo6e55d9e2019-12-12 18:26:26 -0500890
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400891 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
892 logger.Warnw(ctx, "request-aborted", log.Fields{"device-id": agent.deviceID, "name": name, "error": err})
893 return
894 }
895
896 cloned := agent.cloneDeviceWithoutLock()
khenaidoob9203542018-09-17 22:56:37 -0400897 updated := false
khenaidoo6e55d9e2019-12-12 18:26:26 -0500898 s := reflect.ValueOf(cloned).Elem()
khenaidoob9203542018-09-17 22:56:37 -0400899 if s.Kind() == reflect.Struct {
900 // exported field
901 f := s.FieldByName(name)
902 if f.IsValid() && f.CanSet() {
903 switch f.Kind() {
904 case reflect.String:
905 f.SetString(value.(string))
906 updated = true
907 case reflect.Uint32:
908 f.SetUint(uint64(value.(uint32)))
909 updated = true
910 case reflect.Bool:
911 f.SetBool(value.(bool))
912 updated = true
913 }
914 }
915 }
divyadesaicb8b59d2020-08-18 09:55:47 +0000916 logger.Debugw(ctx, "update-field-status", log.Fields{"device-id": cloned.Id, "name": name, "updated": updated})
khenaidoob9203542018-09-17 22:56:37 -0400917 // Save the data
khenaidoo6e55d9e2019-12-12 18:26:26 -0500918
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400919 if err := agent.updateDeviceAndReleaseLock(ctx, cloned); err != nil {
Rohan Agrawal31f21802020-06-12 05:38:46 +0000920 logger.Warnw(ctx, "attribute-update-failed", log.Fields{"attribute": name, "value": value})
khenaidoob9203542018-09-17 22:56:37 -0400921 }
khenaidoob9203542018-09-17 22:56:37 -0400922}
serkant.uluderya334479d2019-04-10 08:26:15 -0700923
Kent Hagerman45a13e42020-04-13 12:23:50 -0400924func (agent *Agent) simulateAlarm(ctx context.Context, simulateReq *voltha.SimulateAlarmRequest) error {
khenaidood948f772021-08-11 17:49:24 -0400925 var err error
926 var desc string
927 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
928 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
929
930 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
khenaidoo442e7c72020-03-10 16:13:48 -0400931 return err
932 }
933 defer agent.requestQueue.RequestComplete()
Himani Chawlab4c25912020-11-12 17:16:38 +0530934 logger.Debugw(ctx, "simulate-alarm", log.Fields{"device-id": agent.deviceID})
khenaidoo6e55d9e2019-12-12 18:26:26 -0500935
Kent Hagermancba2f302020-07-28 13:37:36 -0400936 device := agent.getDeviceReadOnlyWithoutLock()
khenaidoo6e55d9e2019-12-12 18:26:26 -0500937
khenaidood948f772021-08-11 17:49:24 -0400938 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
khenaidoo442e7c72020-03-10 16:13:48 -0400939 if err != nil {
khenaidood948f772021-08-11 17:49:24 -0400940 logger.Errorw(ctx, "grpc-client-nil",
941 log.Fields{
942 "error": err,
943 "device-id": agent.deviceID,
944 "device-type": agent.deviceType,
945 "adapter-endpoint": device.AdapterEndpoint,
946 })
npujar1d86a522019-11-14 17:11:16 +0530947 return err
serkant.uluderya334479d2019-04-10 08:26:15 -0700948 }
khenaidood948f772021-08-11 17:49:24 -0400949 subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
950 requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS
951 go func() {
952 defer cancel()
khenaidoo9beaaf12021-10-19 17:32:01 -0400953 _, err := client.SimulateAlarm(subCtx, &ca.SimulateAlarmMessage{Device: device, Request: simulateReq})
khenaidood948f772021-08-11 17:49:24 -0400954 if err == nil {
955 agent.onSuccess(subCtx, nil, nil, false)
956 } else {
957 agent.onFailure(subCtx, err, nil, nil, false)
958 }
959 }()
serkant.uluderya334479d2019-04-10 08:26:15 -0700960 return nil
961}
Mahir Gunyelb5851672019-07-24 10:46:26 +0300962
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400963// This function updates the device in the DB, releases the device lock, and runs any state transitions.
964// The calling function MUST hold the device lock. The caller MUST NOT modify the device after this is called.
965func (agent *Agent) updateDeviceAndReleaseLock(ctx context.Context, device *voltha.Device) error {
966 // fail early if this agent is no longer valid
Kent Hagerman4f355f52020-03-30 16:01:33 -0400967 if agent.stopped {
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400968 agent.requestQueue.RequestComplete()
Himani Chawla2ba1c9c2020-10-07 13:19:03 +0530969 return errors.New("device-agent-stopped")
Thomas Lee Se5a44012019-11-07 20:32:24 +0530970 }
Kent Hagerman4f355f52020-03-30 16:01:33 -0400971
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400972 // update in db
Kent Hagermanf5a67352020-04-30 15:15:26 -0400973 if err := agent.dbProxy.Set(ctx, agent.deviceID, device); err != nil {
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400974 agent.requestQueue.RequestComplete()
Kent Hagerman4f355f52020-03-30 16:01:33 -0400975 return status.Errorf(codes.Internal, "failed-update-device:%s: %s", agent.deviceID, err)
Mahir Gunyelb5851672019-07-24 10:46:26 +0300976 }
divyadesaicb8b59d2020-08-18 09:55:47 +0000977 logger.Debugw(ctx, "updated-device-in-store", log.Fields{"device-id: ": agent.deviceID})
Mahir Gunyelb5851672019-07-24 10:46:26 +0300978
Kent Hagerman6031aad2020-07-29 16:36:33 -0400979 prevDevice := agent.device
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400980 // update the device
khenaidoo0db4c812020-05-27 15:27:30 -0400981 agent.device = device
Mahir Gunyelb0343bf2021-05-11 14:14:26 -0700982 //If any of the states has chenged, send the change event.
983 if prevDevice.OperStatus != device.OperStatus || prevDevice.ConnectStatus != device.ConnectStatus || prevDevice.AdminState != device.AdminState {
984 _ = agent.deviceMgr.Agent.SendDeviceStateChangeEvent(ctx, prevDevice.OperStatus, prevDevice.ConnectStatus, prevDevice.AdminState, device, time.Now().Unix())
985 }
Maninder0aabf0c2021-03-17 14:55:14 +0530986 deviceTransientState := agent.getTransientState()
987
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400988 // release lock before processing transition
989 agent.requestQueue.RequestComplete()
Himani Chawlab4c25912020-11-12 17:16:38 +0530990 subCtx := coreutils.WithSpanAndRPCMetadataFromContext(ctx)
Kent Hagermanf6db9f12020-07-22 17:16:19 -0400991
Himani Chawlab4c25912020-11-12 17:16:38 +0530992 if err := agent.deviceMgr.stateTransitions.ProcessTransition(subCtx,
Maninder0aabf0c2021-03-17 14:55:14 +0530993 device, prevDevice, deviceTransientState, deviceTransientState); err != nil {
Himani Chawlab4c25912020-11-12 17:16:38 +0530994 logger.Errorw(ctx, "failed-process-transition", log.Fields{"device-id": device.Id, "previous-admin-state": prevDevice.AdminState, "current-admin-state": device.AdminState})
995 // Sending RPC EVENT here
996 rpce := agent.deviceMgr.NewRPCEvent(ctx, agent.deviceID, err.Error(), nil)
Himani Chawla606a4f02021-03-23 19:45:58 +0530997 agent.deviceMgr.SendRPCEvent(ctx, "RPC_ERROR_RAISE_EVENT", rpce, voltha.EventCategory_COMMUNICATION,
998 nil, time.Now().Unix())
Himani Chawlab4c25912020-11-12 17:16:38 +0530999
Kent Hagermanf6db9f12020-07-22 17:16:19 -04001000 }
Mahir Gunyelb5851672019-07-24 10:46:26 +03001001 return nil
1002}
Mahir Gunyelfdee9212019-10-16 16:52:21 -07001003
Himani Chawla2ba1c9c2020-10-07 13:19:03 +05301004// This function updates the device transient in the DB through loader, releases the device lock, and runs any state transitions.
1005// The calling function MUST hold the device lock. The caller MUST NOT modify the device after this is called.
1006func (agent *Agent) updateDeviceWithTransientStateAndReleaseLock(ctx context.Context, device *voltha.Device,
khenaidood948f772021-08-11 17:49:24 -04001007 transientState, prevTransientState core.DeviceTransientState_Types) error {
Himani Chawla2ba1c9c2020-10-07 13:19:03 +05301008 // fail early if this agent is no longer valid
1009 if agent.stopped {
1010 agent.requestQueue.RequestComplete()
1011 return errors.New("device-agent-stopped")
1012 }
1013 //update device TransientState
1014 if err := agent.updateTransientState(ctx, transientState); err != nil {
1015 agent.requestQueue.RequestComplete()
1016 return err
1017 }
1018 // update in db
1019 if err := agent.dbProxy.Set(ctx, agent.deviceID, device); err != nil {
1020 //Reverting TransientState update
khenaidood948f772021-08-11 17:49:24 -04001021 if errTransient := agent.updateTransientState(ctx, prevTransientState); errTransient != nil {
1022 logger.Errorw(ctx, "failed-to-revert-transient-state-update-on-error", log.Fields{"device-id": device.Id,
1023 "previous-transient-state": prevTransientState, "current-transient-state": transientState, "error": errTransient})
1024 }
Himani Chawla2ba1c9c2020-10-07 13:19:03 +05301025 agent.requestQueue.RequestComplete()
1026 return status.Errorf(codes.Internal, "failed-update-device:%s: %s", agent.deviceID, err)
1027 }
1028
1029 logger.Debugw(ctx, "updated-device-in-store", log.Fields{"device-id: ": agent.deviceID})
1030
1031 prevDevice := agent.device
1032 // update the device
1033 agent.device = device
Mahir Gunyelb0343bf2021-05-11 14:14:26 -07001034 //If any of the states has chenged, send the change event.
1035 if prevDevice.OperStatus != device.OperStatus || prevDevice.ConnectStatus != device.ConnectStatus || prevDevice.AdminState != device.AdminState {
1036 _ = agent.deviceMgr.Agent.SendDeviceStateChangeEvent(ctx, prevDevice.OperStatus, prevDevice.ConnectStatus, prevDevice.AdminState, device, time.Now().Unix())
1037 }
Himani Chawla2ba1c9c2020-10-07 13:19:03 +05301038
1039 // release lock before processing transition
1040 agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -04001041 go func() {
1042 subCtx := coreutils.WithSpanAndRPCMetadataFromContext(ctx)
1043 if err := agent.deviceMgr.stateTransitions.ProcessTransition(subCtx,
1044 device, prevDevice, transientState, prevTransientState); err != nil {
1045 logger.Errorw(ctx, "failed-process-transition", log.Fields{"device-id": device.Id, "previous-admin-state": prevDevice.AdminState, "current-admin-state": device.AdminState})
1046 // Sending RPC EVENT here
1047 rpce := agent.deviceMgr.NewRPCEvent(ctx, agent.deviceID, err.Error(), nil)
1048 agent.deviceMgr.SendRPCEvent(ctx, "RPC_ERROR_RAISE_EVENT", rpce, voltha.EventCategory_COMMUNICATION,
1049 nil, time.Now().Unix())
1050 }
1051 }()
Himani Chawla2ba1c9c2020-10-07 13:19:03 +05301052 return nil
1053}
Kent Hagerman2b216042020-04-03 18:28:56 -04001054func (agent *Agent) updateDeviceReason(ctx context.Context, reason string) error {
khenaidood948f772021-08-11 17:49:24 -04001055 logger.Debugw(ctx, "update-device-reason", log.Fields{"device-id": agent.deviceID, "reason": reason})
1056
1057 var err error
1058 var desc string
1059 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
1060 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
1061
1062 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
khenaidoo442e7c72020-03-10 16:13:48 -04001063 return err
1064 }
Maninder9a1bc0d2020-10-26 11:34:02 +05301065
Kent Hagermanf6db9f12020-07-22 17:16:19 -04001066 cloned := agent.cloneDeviceWithoutLock()
npujar1d86a522019-11-14 17:11:16 +05301067 cloned.Reason = reason
khenaidood948f772021-08-11 17:49:24 -04001068 if err = agent.updateDeviceAndReleaseLock(ctx, cloned); err == nil {
1069 requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
Maninder9a1bc0d2020-10-26 11:34:02 +05301070 }
khenaidood948f772021-08-11 17:49:24 -04001071 return err
Mahir Gunyelfdee9212019-10-16 16:52:21 -07001072}
kesavandbc2d1622020-01-21 00:42:01 -05001073
Kent Hagerman2b216042020-04-03 18:28:56 -04001074func (agent *Agent) ChildDeviceLost(ctx context.Context, device *voltha.Device) error {
Himani Chawlab4c25912020-11-12 17:16:38 +05301075 logger.Debugw(ctx, "child-device-lost", log.Fields{"child-device-id": device.Id, "parent-device-id": agent.deviceID})
Chaitrashree G S543df3e2020-02-24 22:36:54 -05001076
khenaidood948f772021-08-11 17:49:24 -04001077 var err error
1078 var desc string
1079 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
1080 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
1081
Kent Hagerman2a07b862020-06-19 15:23:07 -04001082 // Remove the associated peer ports on the parent device
1083 for portID := range agent.portLoader.ListIDs() {
1084 if portHandle, have := agent.portLoader.Lock(portID); have {
1085 oldPort := portHandle.GetReadOnly()
1086 updatedPeers := make([]*voltha.Port_PeerPort, 0)
1087 for _, peerPort := range oldPort.Peers {
1088 if peerPort.DeviceId != device.Id {
1089 updatedPeers = append(updatedPeers, peerPort)
1090 }
khenaidoo442e7c72020-03-10 16:13:48 -04001091 }
Kent Hagerman2a07b862020-06-19 15:23:07 -04001092 newPort := *oldPort
1093 newPort.Peers = updatedPeers
1094 if err := portHandle.Update(ctx, &newPort); err != nil {
1095 portHandle.Unlock()
1096 return nil
1097 }
1098 portHandle.Unlock()
khenaidoo442e7c72020-03-10 16:13:48 -04001099 }
Chaitrashree G S543df3e2020-02-24 22:36:54 -05001100 }
1101
khenaidoo442e7c72020-03-10 16:13:48 -04001102 //send request to adapter
khenaidood948f772021-08-11 17:49:24 -04001103 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
khenaidoo442e7c72020-03-10 16:13:48 -04001104 if err != nil {
khenaidood948f772021-08-11 17:49:24 -04001105 logger.Errorw(ctx, "grpc-client-nil",
1106 log.Fields{
1107 "error": err,
1108 "device-id": agent.deviceID,
1109 "device-type": agent.deviceType,
1110 "adapter-endpoint": device.AdapterEndpoint,
1111 })
khenaidoo442e7c72020-03-10 16:13:48 -04001112 return err
Chaitrashree G S543df3e2020-02-24 22:36:54 -05001113 }
khenaidood948f772021-08-11 17:49:24 -04001114 subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
1115 requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS
1116 go func() {
1117 defer cancel()
1118 _, err := client.ChildDeviceLost(subCtx, device)
1119 if err == nil {
1120 agent.onSuccess(subCtx, nil, nil, true)
1121 } else {
1122 agent.onFailure(subCtx, err, nil, nil, true)
1123 }
1124 }()
Chaitrashree G S543df3e2020-02-24 22:36:54 -05001125 return nil
Chaitrashree G S543df3e2020-02-24 22:36:54 -05001126}
onkarkundargi87285252020-01-27 11:34:52 +05301127
khenaidoo9beaaf12021-10-19 17:32:01 -04001128func (agent *Agent) startOmciTest(ctx context.Context, omcitestrequest *omci.OmciTestRequest) (*omci.TestResponse, error) {
khenaidood948f772021-08-11 17:49:24 -04001129 var err error
1130 var desc string
1131 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
1132 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
onkarkundargi87285252020-01-27 11:34:52 +05301133
khenaidood948f772021-08-11 17:49:24 -04001134 // OMCI test may be performed on a pre-provisioned device. If a device is in that state both its device type and endpoint
1135 // may not have been set yet.
1136 // First check if we need to update the type or endpoint
1137 cloned, err := agent.getDeviceReadOnly(ctx)
onkarkundargi87285252020-01-27 11:34:52 +05301138 if err != nil {
1139 return nil, err
1140 }
khenaidood948f772021-08-11 17:49:24 -04001141 if cloned.Type == "" || cloned.AdapterEndpoint == "" {
1142 if err = agent.updateDeviceTypeAndEndpoint(ctx); err != nil {
1143 return nil, err
1144 }
1145 cloned, err = agent.getDeviceReadOnly(ctx)
1146 if err != nil {
1147 return nil, err
1148 }
onkarkundargi87285252020-01-27 11:34:52 +05301149 }
1150
khenaidood948f772021-08-11 17:49:24 -04001151 // Send request to the adapter
1152 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
1153 if err != nil {
1154 logger.Errorw(ctx, "grpc-client-nil",
1155 log.Fields{
1156 "error": err,
1157 "device-id": agent.deviceID,
1158 "device-type": agent.deviceType,
1159 "adapter-endpoint": cloned.AdapterEndpoint,
1160 })
1161 return nil, err
onkarkundargi87285252020-01-27 11:34:52 +05301162 }
khenaidood948f772021-08-11 17:49:24 -04001163
khenaidoo9beaaf12021-10-19 17:32:01 -04001164 res, err := client.StartOmciTest(ctx, &ca.OMCITest{
khenaidood948f772021-08-11 17:49:24 -04001165 Device: cloned,
1166 Request: omcitestrequest,
1167 })
1168 if err == nil {
1169 requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
1170 }
1171 return res, err
onkarkundargi87285252020-01-27 11:34:52 +05301172}
Dinesh Belwalkarc1129f12020-02-27 10:41:33 -08001173
khenaidoo9beaaf12021-10-19 17:32:01 -04001174func (agent *Agent) getExtValue(ctx context.Context, pdevice *voltha.Device, cdevice *voltha.Device, valueparam *extension.ValueSpecifier) (*extension.ReturnValues, error) {
Himani Chawlab4c25912020-11-12 17:16:38 +05301175 logger.Debugw(ctx, "get-ext-value", log.Fields{"device-id": agent.deviceID, "onu-id": valueparam.Id, "value-type": valueparam.Value})
khenaidood948f772021-08-11 17:49:24 -04001176 var err error
1177 var desc string
1178 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
1179 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
1180
1181 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
Dinesh Belwalkarc1129f12020-02-27 10:41:33 -08001182 return nil, err
1183 }
1184
khenaidood948f772021-08-11 17:49:24 -04001185 //send request to adapter synchronously
1186 client, err := agent.adapterMgr.GetAdapterClient(ctx, pdevice.AdapterEndpoint)
Dinesh Belwalkarc1129f12020-02-27 10:41:33 -08001187 if err != nil {
khenaidood948f772021-08-11 17:49:24 -04001188 logger.Errorw(ctx, "grpc-client-nil",
1189 log.Fields{
1190 "error": err,
1191 "device-id": agent.deviceID,
1192 "device-type": agent.deviceType,
1193 "adapter-endpoint": pdevice.AdapterEndpoint,
1194 })
1195 agent.requestQueue.RequestComplete()
Dinesh Belwalkarc1129f12020-02-27 10:41:33 -08001196 return nil, err
1197 }
1198
khenaidood948f772021-08-11 17:49:24 -04001199 // Release lock before sending to adapter
1200 agent.requestQueue.RequestComplete()
Dinesh Belwalkarc1129f12020-02-27 10:41:33 -08001201
khenaidoo9beaaf12021-10-19 17:32:01 -04001202 retVal, err := client.GetExtValue(ctx, &ca.GetExtValueMessage{
khenaidood948f772021-08-11 17:49:24 -04001203 ParentDevice: pdevice,
1204 ChildDevice: cdevice,
1205 ValueType: valueparam.Value,
1206 })
1207 if err == nil {
1208 requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
Dinesh Belwalkarc1129f12020-02-27 10:41:33 -08001209 }
khenaidood948f772021-08-11 17:49:24 -04001210 return retVal, err
Dinesh Belwalkarc1129f12020-02-27 10:41:33 -08001211}
dpaul62686312020-06-23 14:17:36 +05301212
khenaidoo9beaaf12021-10-19 17:32:01 -04001213func (agent *Agent) setExtValue(ctx context.Context, device *voltha.Device, value *extension.ValueSet) (*empty.Empty, error) {
Himani Chawlab4c25912020-11-12 17:16:38 +05301214 logger.Debugw(ctx, "set-ext-value", log.Fields{"device-id": value.Id})
khenaidood948f772021-08-11 17:49:24 -04001215
1216 var err error
1217 var desc string
1218 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
1219 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
1220
1221 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
dpaul62686312020-06-23 14:17:36 +05301222 return nil, err
1223 }
1224
1225 //send request to adapter
khenaidood948f772021-08-11 17:49:24 -04001226 //send request to adapter synchronously
1227 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
dpaul62686312020-06-23 14:17:36 +05301228 if err != nil {
khenaidood948f772021-08-11 17:49:24 -04001229 logger.Errorw(ctx, "grpc-client-nil",
1230 log.Fields{
1231 "error": err,
1232 "device-id": agent.deviceID,
1233 "device-type": agent.deviceType,
1234 "adapter-endpoint": device.AdapterEndpoint,
1235 })
1236 agent.requestQueue.RequestComplete()
dpaul62686312020-06-23 14:17:36 +05301237 return nil, err
1238 }
khenaidood948f772021-08-11 17:49:24 -04001239 // Release lock before sending request to adapter
1240 agent.requestQueue.RequestComplete()
dpaul62686312020-06-23 14:17:36 +05301241
khenaidoo9beaaf12021-10-19 17:32:01 -04001242 retVal, err := client.SetExtValue(ctx, &ca.SetExtValueMessage{
khenaidood948f772021-08-11 17:49:24 -04001243 Device: device,
1244 Value: value,
1245 })
1246 if err == nil {
1247 requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
dpaul62686312020-06-23 14:17:36 +05301248 }
khenaidood948f772021-08-11 17:49:24 -04001249 return retVal, err
dpaul62686312020-06-23 14:17:36 +05301250}
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301251
1252func (agent *Agent) getSingleValue(ctx context.Context, request *extension.SingleGetValueRequest) (*extension.SingleGetValueResponse, error) {
Himani Chawlab4c25912020-11-12 17:16:38 +05301253 logger.Debugw(ctx, "get-single-value", log.Fields{"device-id": request.TargetId})
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301254
khenaidood948f772021-08-11 17:49:24 -04001255 var err error
1256 var desc string
1257 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
1258 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
1259
1260 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301261 return nil, err
1262 }
1263
1264 cloned := agent.cloneDeviceWithoutLock()
1265
1266 //send request to adapter
khenaidood948f772021-08-11 17:49:24 -04001267 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301268 if err != nil {
khenaidood948f772021-08-11 17:49:24 -04001269 logger.Errorw(ctx, "grpc-client-nil",
1270 log.Fields{
1271 "error": err,
1272 "device-id": cloned.Id,
1273 "adapter-endpoint": cloned.AdapterEndpoint,
1274 })
1275 agent.requestQueue.RequestComplete()
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301276 return nil, err
1277 }
khenaidood948f772021-08-11 17:49:24 -04001278 // Release lock before sending request to adapter
1279 agent.requestQueue.RequestComplete()
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301280
khenaidood948f772021-08-11 17:49:24 -04001281 resp, err := client.GetSingleValue(ctx, request)
1282 if err == nil {
1283 requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301284 }
khenaidood948f772021-08-11 17:49:24 -04001285 return resp, err
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301286}
1287
1288func (agent *Agent) setSingleValue(ctx context.Context, request *extension.SingleSetValueRequest) (*extension.SingleSetValueResponse, error) {
Himani Chawlab4c25912020-11-12 17:16:38 +05301289 logger.Debugw(ctx, "set-single-value", log.Fields{"device-id": request.TargetId})
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301290
khenaidood948f772021-08-11 17:49:24 -04001291 var err error
1292 var desc string
1293 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
1294 defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()
1295
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301296 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
1297 return nil, err
1298 }
1299
1300 cloned := agent.cloneDeviceWithoutLock()
1301
1302 //send request to adapter
khenaidood948f772021-08-11 17:49:24 -04001303 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301304 if err != nil {
khenaidood948f772021-08-11 17:49:24 -04001305 logger.Errorw(ctx, "grpc-client-nil",
1306 log.Fields{
1307 "error": err,
1308 "device-id": agent.deviceID,
1309 "device-type": agent.deviceType,
1310 "adapter-endpoint": cloned.AdapterEndpoint,
1311 })
1312 agent.requestQueue.RequestComplete()
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301313 return nil, err
1314 }
khenaidood948f772021-08-11 17:49:24 -04001315 // Release lock before sending request to adapter
1316 agent.requestQueue.RequestComplete()
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301317
khenaidood948f772021-08-11 17:49:24 -04001318 resp, err := client.SetSingleValue(ctx, request)
1319 if err == nil {
1320 requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301321 }
khenaidood948f772021-08-11 17:49:24 -04001322 return resp, err
Salman Siddiqui1cf95042020-11-19 00:42:56 +05301323}
Maninder0aabf0c2021-03-17 14:55:14 +05301324
Maninder2195ccc2021-06-23 20:23:01 +05301325func (agent *Agent) proceedWithRequest(device *voltha.Device) bool {
1326 return !agent.isDeletionInProgress() && !agent.isInReconcileState(device)
Maninder0aabf0c2021-03-17 14:55:14 +05301327}
1328
1329func (agent *Agent) stopReconcile() {
1330 agent.stopReconcilingMutex.Lock()
1331 if agent.stopReconciling != nil {
1332 agent.stopReconciling <- 0
1333 }
1334 agent.stopReconcilingMutex.Unlock()
1335}
1336
khenaidood948f772021-08-11 17:49:24 -04001337// abortAllProcessing is invoked when an adapter managing this device is restarted
1338func (agent *Agent) abortAllProcessing(ctx context.Context) error {
1339 logger.Infow(ctx, "aborting-current-running-requests", log.Fields{"device-id": agent.deviceID})
1340 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
1341 return err
1342 }
1343 defer agent.requestQueue.RequestComplete()
1344
1345 // If any reconciling is in progress just abort it. The adapter is gone.
1346 agent.stopReconcile()
1347
1348 // Update the Core device transient state accordingly
1349 var updatedState core.DeviceTransientState_Types
1350 switch agent.getTransientState() {
1351 case core.DeviceTransientState_RECONCILE_IN_PROGRESS:
1352 updatedState = core.DeviceTransientState_NONE
1353 case core.DeviceTransientState_FORCE_DELETING:
1354 updatedState = core.DeviceTransientState_DELETE_FAILED
1355 case core.DeviceTransientState_DELETING_FROM_ADAPTER:
1356 updatedState = core.DeviceTransientState_DELETE_FAILED
khenaidoo68a5e0c2021-11-06 13:08:03 -04001357 case core.DeviceTransientState_DELETE_FAILED:
1358 // do not change state
1359 return nil
khenaidood948f772021-08-11 17:49:24 -04001360 default:
1361 updatedState = core.DeviceTransientState_NONE
1362 }
1363 if err := agent.updateTransientState(ctx, updatedState); err != nil {
1364 logger.Errorf(ctx, "transient-state-update-failed", log.Fields{"error": err})
1365 return err
1366 }
1367 return nil
1368}
1369
khenaidoo68a5e0c2021-11-06 13:08:03 -04001370func (agent *Agent) DeleteDevicePostAdapterRestart(ctx context.Context) error {
1371 logger.Debugw(ctx, "delete-post-restart", log.Fields{"device-id": agent.deviceID})
1372 ctx = utils.WithNewSpanAndRPCMetadataContext(ctx, "DelteDevicePostAdapterRestart")
1373
1374 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
1375 return err
1376 }
1377
1378 device := agent.getDeviceReadOnlyWithoutLock()
1379 if device.AdminState == voltha.AdminState_PREPROVISIONED {
1380 logger.Debugw(ctx, "device-in-preprovisioning-state-reconcile-not-needed", log.Fields{"device-id": device.Id})
1381 agent.requestQueue.RequestComplete()
1382 return nil
1383 }
1384 // Change device transient state to FORCE_DELETING
1385 if err := agent.updateTransientState(ctx, core.DeviceTransientState_FORCE_DELETING); err != nil {
1386 logger.Errorw(ctx, "failure-updating-transient-state", log.Fields{"error": err, "device-id": agent.deviceID})
1387 agent.requestQueue.RequestComplete()
1388 return err
1389 }
1390
1391 // Ensure we have a valid grpc client available as we have just restarted
1392 deleteBackoff := backoff.NewExponentialBackOff()
1393 deleteBackoff.InitialInterval = agent.config.BackoffRetryInitialInterval
1394 deleteBackoff.MaxElapsedTime = agent.config.BackoffRetryMaxElapsedTime
1395 deleteBackoff.MaxInterval = agent.config.BackoffRetryMaxInterval
1396 var backoffTimer *time.Timer
1397 var err error
1398 var client adapter_service.AdapterServiceClient
1399retry:
1400 for {
1401 client, err = agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
1402 if err == nil {
1403 break retry
1404 }
1405 duration := deleteBackoff.NextBackOff()
1406 if duration == backoff.Stop {
1407 deleteBackoff.Reset()
1408 duration = deleteBackoff.NextBackOff()
1409 }
1410 backoffTimer = time.NewTimer(duration)
1411 select {
1412 case <-backoffTimer.C:
1413 logger.Debugw(ctx, "backoff-timer-expires", log.Fields{"device-id": agent.deviceID})
1414 case <-ctx.Done():
1415 err = ctx.Err()
1416 break retry
1417 }
1418 }
1419 if backoffTimer != nil && !backoffTimer.Stop() {
1420 select {
1421 case <-backoffTimer.C:
1422 default:
1423 }
1424 }
1425 if err != nil || client == nil {
1426 agent.requestQueue.RequestComplete()
1427 return err
1428 }
1429
1430 // Release the device lock to allow for device state update, if any
1431 agent.requestQueue.RequestComplete()
1432
1433 // Send the delete request to the adapter
1434 subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
1435 defer cancel()
1436 if _, err = client.DeleteDevice(subCtx, device); err != nil {
1437 agent.onDeleteFailure(subCtx, err, nil, nil)
1438 } else {
1439 agent.onDeleteSuccess(subCtx, nil, nil)
1440 }
1441 return nil
1442}
1443
khenaidood948f772021-08-11 17:49:24 -04001444func (agent *Agent) ReconcileDevice(ctx context.Context) {
khenaidoo68a5e0c2021-11-06 13:08:03 -04001445 // Do not reconcile if the device was in DELETE_FAILED transient state. Just invoke the force delete on that device.
1446 state := agent.getTransientState()
1447 logger.Debugw(ctx, "starting-reconcile", log.Fields{"device-id": agent.deviceID, "state": state})
1448 if agent.getTransientState() == core.DeviceTransientState_DELETE_FAILED {
1449 if err := agent.DeleteDevicePostAdapterRestart(ctx); err != nil {
1450 logger.Errorw(ctx, "delete-post-restart-failed", log.Fields{"error": err, "device-id": agent.deviceID})
1451 }
1452 return
1453 }
1454
khenaidood948f772021-08-11 17:49:24 -04001455 requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
Maninder0aabf0c2021-03-17 14:55:14 +05301456 var desc string
Maninder0aabf0c2021-03-17 14:55:14 +05301457
1458 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
khenaidood948f772021-08-11 17:49:24 -04001459 agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc)
1460 return
1461 }
1462
1463 device := agent.getDeviceReadOnlyWithoutLock()
1464 if device.AdminState == voltha.AdminState_PREPROVISIONED {
1465 agent.requestQueue.RequestComplete()
1466 logger.Debugw(ctx, "device-in-preprovisioning-state-reconcile-not-needed", log.Fields{"device-id": device.Id})
Maninder0aabf0c2021-03-17 14:55:14 +05301467 return
1468 }
1469
Maninder2195ccc2021-06-23 20:23:01 +05301470 if !agent.proceedWithRequest(device) {
Maninder0aabf0c2021-03-17 14:55:14 +05301471 agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -04001472 err := fmt.Errorf("cannot complete operation as device deletion/reconciling is in progress or reconcile failed for device : %s", device.Id)
1473 logger.Errorw(ctx, "reconcile-failed", log.Fields{"error": err})
1474 agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc)
Maninder0aabf0c2021-03-17 14:55:14 +05301475 return
1476 }
1477
1478 //set transient state to RECONCILE IN PROGRESS
khenaidood948f772021-08-11 17:49:24 -04001479 err := agent.updateTransientState(ctx, core.DeviceTransientState_RECONCILE_IN_PROGRESS)
Maninder0aabf0c2021-03-17 14:55:14 +05301480 if err != nil {
1481 agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -04001482 logger.Errorw(ctx, "setting-transient-state-failed", log.Fields{"error": err})
1483 agent.logDeviceUpdate(ctx, nil, nil, requestStatus, nil, desc)
Maninder0aabf0c2021-03-17 14:55:14 +05301484 return
1485 }
1486
Maninder0aabf0c2021-03-17 14:55:14 +05301487 reconcilingBackoff := backoff.NewExponentialBackOff()
1488 reconcilingBackoff.InitialInterval = agent.config.BackoffRetryInitialInterval
1489 reconcilingBackoff.MaxElapsedTime = agent.config.BackoffRetryMaxElapsedTime
1490 reconcilingBackoff.MaxInterval = agent.config.BackoffRetryMaxInterval
1491
1492 //making here to keep lifecycle of this channel within the scope of retryReconcile
1493 agent.stopReconcilingMutex.Lock()
1494 agent.stopReconciling = make(chan int)
1495 agent.stopReconcilingMutex.Unlock()
1496
David K. Bainbridge482e4422021-06-30 12:23:42 -07001497 // defined outside the retry loop so it can be cleaned
1498 // up when the loop breaks
1499 var backoffTimer *time.Timer
1500
1501retry:
Maninder0aabf0c2021-03-17 14:55:14 +05301502 for {
David K. Bainbridge482e4422021-06-30 12:23:42 -07001503 // If the operations state of the device is RECONCILING_FAILED then we do not
1504 // want to continue to attempt reconciliation.
1505 deviceRef := agent.getDeviceReadOnlyWithoutLock()
1506 if deviceRef.OperStatus == common.OperStatus_RECONCILING_FAILED {
1507 logger.Warnw(ctx, "reconciling-failed-halting-retries",
1508 log.Fields{"device-id": device.Id})
1509 agent.requestQueue.RequestComplete()
1510 break retry
1511 }
1512
Maninder0aabf0c2021-03-17 14:55:14 +05301513 // Use an exponential back off to prevent getting into a tight loop
1514 duration := reconcilingBackoff.NextBackOff()
1515 //This case should never occur in default case as max elapsed time for backoff is 0(by default) , so it will never return stop
1516 if duration == backoff.Stop {
1517 // If we reach a maximum then warn and reset the backoff
1518 // timer and keep attempting.
1519 logger.Warnw(ctx, "maximum-reconciling-backoff-reached--resetting-backoff-timer",
1520 log.Fields{"max-reconciling-backoff": reconcilingBackoff.MaxElapsedTime,
1521 "device-id": device.Id})
1522 reconcilingBackoff.Reset()
1523 duration = reconcilingBackoff.NextBackOff()
1524 }
1525
David K. Bainbridge482e4422021-06-30 12:23:42 -07001526 backoffTimer = time.NewTimer(duration)
Maninder0aabf0c2021-03-17 14:55:14 +05301527
khenaidood948f772021-08-11 17:49:24 -04001528 logger.Debugw(ctx, "retrying-reconciling", log.Fields{"deviceID": device.Id, "endpoint": device.AdapterEndpoint})
1529 // Release lock before sending request to adapter
Maninder0aabf0c2021-03-17 14:55:14 +05301530 agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -04001531
1532 // Send a reconcile request to the adapter.
1533 err := agent.sendReconcileRequestToAdapter(ctx, device)
1534 if errors.Is(err, errContextExpired) || errors.Is(err, errReconcileAborted) {
1535 logger.Errorw(ctx, "reconcile-aborted", log.Fields{"error": err})
1536 requestStatus = &common.OperationResp{Code: common.OperationResp_OperationReturnCode(common.OperStatus_FAILED)}
1537 desc = "aborted"
1538 agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc)
1539 break retry
1540 }
Maninder0aabf0c2021-03-17 14:55:14 +05301541 if err != nil {
khenaidood948f772021-08-11 17:49:24 -04001542 agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc)
Maninder0aabf0c2021-03-17 14:55:14 +05301543 <-backoffTimer.C
1544 // backoffTimer expired continue
1545 // Take lock back before retrying
1546 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
khenaidood948f772021-08-11 17:49:24 -04001547 agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc)
David K. Bainbridge482e4422021-06-30 12:23:42 -07001548 break retry
Maninder0aabf0c2021-03-17 14:55:14 +05301549 }
1550 continue
1551 }
khenaidood948f772021-08-11 17:49:24 -04001552 // Success
1553 requestStatus = &common.OperationResp{Code: common.OperationResp_OPERATION_SUCCESS}
1554 desc = "adapter-response"
1555 agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc)
1556 break retry
David K. Bainbridge482e4422021-06-30 12:23:42 -07001557 }
1558
1559 // Retry loop is broken, so stop any timers and drain the channel
1560 if backoffTimer != nil && !backoffTimer.Stop() {
1561
1562 // As per documentation and stack overflow when a timer is stopped its
1563 // channel should be drained. The issue is that Stop returns false
1564 // either if the timer has already been fired "OR" if the timer can be
1565 // stopped before being fired. This means that in some cases the
1566 // channel has already be emptied so attempting to read from it means
1567 // a blocked thread. To get around this use a select so if the
1568 // channel is already empty the default case hits and we are not
1569 // blocked.
1570 select {
1571 case <-backoffTimer.C:
1572 default:
Maninder0aabf0c2021-03-17 14:55:14 +05301573 }
1574 }
1575}
1576
khenaidood948f772021-08-11 17:49:24 -04001577func (agent *Agent) sendReconcileRequestToAdapter(ctx context.Context, device *voltha.Device) error {
1578 logger.Debugw(ctx, "sending-reconcile-to-adapter", log.Fields{"device-id": device.Id, "endpoint": agent.adapterEndpoint})
1579 client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
1580 if err != nil {
1581 return err
1582 }
1583 adapterResponse := make(chan error)
1584 go func() {
1585 _, err := client.ReconcileDevice(ctx, device)
1586 adapterResponse <- err
1587 }()
Maninder0aabf0c2021-03-17 14:55:14 +05301588 select {
1589 // wait for response
khenaidood948f772021-08-11 17:49:24 -04001590 case err := <-adapterResponse:
1591 if err != nil {
1592 return err
Maninder0aabf0c2021-03-17 14:55:14 +05301593 }
Maninder0aabf0c2021-03-17 14:55:14 +05301594 //In case of success quit retrying and wait for adapter to reset operation state of device
1595 agent.stopReconcilingMutex.Lock()
1596 agent.stopReconciling = nil
1597 agent.stopReconcilingMutex.Unlock()
1598 return nil
1599
1600 //if reconciling need to be stopped
1601 case _, ok := <-agent.stopReconciling:
1602 agent.stopReconcilingMutex.Lock()
1603 agent.stopReconciling = nil
1604 agent.stopReconcilingMutex.Unlock()
1605 if !ok {
1606 //channel-closed
khenaidood948f772021-08-11 17:49:24 -04001607 return fmt.Errorf("reconcile channel closed:%w", errReconcileAborted)
Maninder0aabf0c2021-03-17 14:55:14 +05301608 }
khenaidood948f772021-08-11 17:49:24 -04001609 return fmt.Errorf("reconciling aborted:%w", errReconcileAborted)
1610 // Context expired
1611 case <-ctx.Done():
1612 return fmt.Errorf("context expired:%s :%w", ctx.Err(), errContextExpired)
Maninder0aabf0c2021-03-17 14:55:14 +05301613 }
Maninder0aabf0c2021-03-17 14:55:14 +05301614}
1615
1616func (agent *Agent) reconcilingCleanup(ctx context.Context) error {
1617 var desc string
khenaidood948f772021-08-11 17:49:24 -04001618 var err error
Maninder0aabf0c2021-03-17 14:55:14 +05301619 operStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
khenaidood948f772021-08-11 17:49:24 -04001620 defer func() { agent.logDeviceUpdate(ctx, nil, nil, operStatus, err, desc) }()
1621
1622 if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
1623 desc = "reconcile-cleanup-failed"
Maninder0aabf0c2021-03-17 14:55:14 +05301624 return err
1625 }
1626 defer agent.requestQueue.RequestComplete()
khenaidood948f772021-08-11 17:49:24 -04001627 err = agent.updateTransientState(ctx, core.DeviceTransientState_NONE)
Maninder0aabf0c2021-03-17 14:55:14 +05301628 if err != nil {
khenaidood948f772021-08-11 17:49:24 -04001629 logger.Errorf(ctx, "transient-state-update-failed", log.Fields{"error": err})
Maninder0aabf0c2021-03-17 14:55:14 +05301630 return err
1631 }
khenaidood948f772021-08-11 17:49:24 -04001632 operStatus.Code = common.OperationResp_OPERATION_SUCCESS
Maninder0aabf0c2021-03-17 14:55:14 +05301633 return nil
1634}
khenaidood948f772021-08-11 17:49:24 -04001635
1636func (agent *Agent) isAdapterConnectionUp(ctx context.Context) bool {
1637 c, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
1638 return c != nil && err == nil
1639}
1640
1641func (agent *Agent) canDeviceRequestProceed(ctx context.Context) error {
1642 if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
1643 return err
1644 }
1645 defer agent.requestQueue.RequestComplete()
1646 if agent.proceedWithRequest(agent.device) {
1647 return nil
1648 }
1649 return fmt.Errorf("device-cannot-process-request-%s", agent.deviceID)
1650}