blob: 69cd3c8af70653482cf834ab08d7034d04300131 [file] [log] [blame]
khenaidoob9203542018-09-17 22:56:37 -04001/*
2 * Copyright 2018-present Open Networking Foundation
3
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7
8 * http://www.apache.org/licenses/LICENSE-2.0
9
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
npujar1d86a522019-11-14 17:11:16 +053016
khenaidoob9203542018-09-17 22:56:37 -040017package core
18
19import (
20 "context"
Thomas Lee Se5a44012019-11-07 20:32:24 +053021 "fmt"
Scott Baker2d87ee32020-03-03 13:04:01 -080022 "sync"
npujar1d86a522019-11-14 17:11:16 +053023 "time"
24
sbarbari17d7e222019-11-05 10:02:29 -050025 "github.com/opencord/voltha-go/db/model"
khenaidoob9203542018-09-17 22:56:37 -040026 "github.com/opencord/voltha-go/rw_core/config"
serkant.uluderya2ae470f2020-01-21 11:13:09 -080027 "github.com/opencord/voltha-lib-go/v3/pkg/db"
28 "github.com/opencord/voltha-lib-go/v3/pkg/db/kvstore"
29 grpcserver "github.com/opencord/voltha-lib-go/v3/pkg/grpc"
30 "github.com/opencord/voltha-lib-go/v3/pkg/kafka"
31 "github.com/opencord/voltha-lib-go/v3/pkg/log"
32 "github.com/opencord/voltha-lib-go/v3/pkg/probe"
33 "github.com/opencord/voltha-protos/v3/go/voltha"
khenaidoob9203542018-09-17 22:56:37 -040034 "google.golang.org/grpc"
khenaidoob3244212019-08-27 14:32:27 -040035 "google.golang.org/grpc/codes"
36 "google.golang.org/grpc/status"
khenaidoob9203542018-09-17 22:56:37 -040037)
38
npujar1d86a522019-11-14 17:11:16 +053039// Core represent read,write core attributes
khenaidoob9203542018-09-17 22:56:37 -040040type Core struct {
npujar1d86a522019-11-14 17:11:16 +053041 instanceID string
khenaidoob9203542018-09-17 22:56:37 -040042 deviceMgr *DeviceManager
43 logicalDeviceMgr *LogicalDeviceManager
44 grpcServer *grpcserver.GrpcServer
Richard Jankowskidbab94a2018-12-06 16:20:25 -050045 grpcNBIAPIHandler *APIHandler
khenaidoo2c6a0992019-04-29 13:46:56 -040046 adapterMgr *AdapterManager
khenaidoob9203542018-09-17 22:56:37 -040047 config *config.RWCoreFlags
npujar467fe752020-01-16 20:17:45 +053048 kmp kafka.InterContainerProxy
khenaidoo92e62c52018-10-03 14:02:54 -040049 clusterDataRoot model.Root
50 localDataRoot model.Root
khenaidoob9203542018-09-17 22:56:37 -040051 clusterDataProxy *model.Proxy
52 localDataProxy *model.Proxy
Scott Baker2d87ee32020-03-03 13:04:01 -080053 exitChannel chan struct{}
54 stopOnce sync.Once
Richard Jankowskie4d77662018-10-17 13:53:21 -040055 kvClient kvstore.Client
Girish Kumar4d3887d2019-11-22 14:22:05 +000056 backend db.Backend
khenaidoo43c82122018-11-22 18:38:28 -050057 kafkaClient kafka.Client
khenaidoob9203542018-09-17 22:56:37 -040058}
59
npujar1d86a522019-11-14 17:11:16 +053060// NewCore creates instance of rw core
Thomas Lee Se5a44012019-11-07 20:32:24 +053061func NewCore(ctx context.Context, id string, cf *config.RWCoreFlags, kvClient kvstore.Client, kafkaClient kafka.Client) *Core {
khenaidoob9203542018-09-17 22:56:37 -040062 var core Core
npujar1d86a522019-11-14 17:11:16 +053063 core.instanceID = id
Scott Baker2d87ee32020-03-03 13:04:01 -080064 core.exitChannel = make(chan struct{})
khenaidoob9203542018-09-17 22:56:37 -040065 core.config = cf
Richard Jankowskie4d77662018-10-17 13:53:21 -040066 core.kvClient = kvClient
khenaidoo43c82122018-11-22 18:38:28 -050067 core.kafkaClient = kafkaClient
Richard Jankowskie4d77662018-10-17 13:53:21 -040068
Girish Kumar4d3887d2019-11-22 14:22:05 +000069 // Configure backend to push Liveness Status at least every (cf.LiveProbeInterval / 2) seconds
70 // so as to avoid trigger of Liveness check (due to Liveness timeout) when backend is alive
71 livenessChannelInterval := cf.LiveProbeInterval / 2
72
Richard Jankowskie4d77662018-10-17 13:53:21 -040073 // Setup the KV store
Girish Kumar4d3887d2019-11-22 14:22:05 +000074 core.backend = db.Backend{
75 Client: kvClient,
76 StoreType: cf.KVStoreType,
77 Host: cf.KVStoreHost,
78 Port: cf.KVStorePort,
79 Timeout: cf.KVStoreTimeout,
80 LivenessChannelInterval: livenessChannelInterval,
81 PathPrefix: cf.KVStoreDataPrefix}
82 core.clusterDataRoot = model.NewRoot(&voltha.Voltha{}, &core.backend)
83 core.localDataRoot = model.NewRoot(&voltha.CoreInstance{}, &core.backend)
khenaidoob9203542018-09-17 22:56:37 -040084 return &core
85}
86
npujar1d86a522019-11-14 17:11:16 +053087// Start brings up core services
Thomas Lee Se5a44012019-11-07 20:32:24 +053088func (core *Core) Start(ctx context.Context) error {
David K. Bainbridgeb4a9ab02019-09-20 15:12:16 -070089
90 // If the context has a probe then fetch it and register our services
91 var p *probe.Probe
92 if value := ctx.Value(probe.ProbeContextKey); value != nil {
93 if _, ok := value.(*probe.Probe); ok {
94 p = value.(*probe.Probe)
95 p.RegisterService(
96 "message-bus",
97 "kv-store",
98 "device-manager",
99 "logical-device-manager",
100 "adapter-manager",
101 "grpc-service",
102 )
103 }
104 }
105
Girish Kumarf56a4682020-03-20 20:07:46 +0000106 logger.Info("starting-core-services", log.Fields{"coreId": core.instanceID})
khenaidoob3244212019-08-27 14:32:27 -0400107
108 // Wait until connection to KV Store is up
109 if err := core.waitUntilKVStoreReachableOrMaxTries(ctx, core.config.MaxConnectionRetries, core.config.ConnectionRetryInterval); err != nil {
Girish Kumarf56a4682020-03-20 20:07:46 +0000110 logger.Fatal("Unable-to-connect-to-KV-store")
khenaidoob3244212019-08-27 14:32:27 -0400111 }
David K. Bainbridgeb4a9ab02019-09-20 15:12:16 -0700112 if p != nil {
113 p.UpdateStatus("kv-store", probe.ServiceStatusRunning)
114 }
Thomas Lee Se5a44012019-11-07 20:32:24 +0530115 var err error
116
npujar467fe752020-01-16 20:17:45 +0530117 core.clusterDataProxy, err = core.clusterDataRoot.CreateProxy(ctx, "/", false)
Thomas Lee Se5a44012019-11-07 20:32:24 +0530118 if err != nil {
119 probe.UpdateStatusFromContext(ctx, "kv-store", probe.ServiceStatusNotReady)
120 return fmt.Errorf("Failed to create cluster data proxy")
121 }
npujar467fe752020-01-16 20:17:45 +0530122 core.localDataProxy, err = core.localDataRoot.CreateProxy(ctx, "/", false)
Thomas Lee Se5a44012019-11-07 20:32:24 +0530123 if err != nil {
124 probe.UpdateStatusFromContext(ctx, "kv-store", probe.ServiceStatusNotReady)
125 return fmt.Errorf("Failed to create local data proxy")
126 }
khenaidoob3244212019-08-27 14:32:27 -0400127
Scott Bakeree6a0872019-10-29 15:59:52 -0700128 // core.kmp must be created before deviceMgr and adapterMgr, as they will make
129 // private copies of the poiner to core.kmp.
npujar467fe752020-01-16 20:17:45 +0530130 core.initKafkaManager(ctx)
khenaidoob3244212019-08-27 14:32:27 -0400131
Girish Kumarf56a4682020-03-20 20:07:46 +0000132 logger.Debugw("values", log.Fields{"kmp": core.kmp})
Richard Jankowski199fd862019-03-18 14:49:51 -0400133 core.deviceMgr = newDeviceManager(core)
Kent Hagerman16ce36a2019-12-17 13:40:53 -0500134 core.adapterMgr = newAdapterManager(core.clusterDataProxy, core.instanceID, core.kafkaClient, core.deviceMgr)
khenaidooba6b6c42019-08-02 09:11:56 -0400135 core.deviceMgr.adapterMgr = core.adapterMgr
khenaidoo2c6a0992019-04-29 13:46:56 -0400136 core.logicalDeviceMgr = newLogicalDeviceManager(core, core.deviceMgr, core.kmp, core.clusterDataProxy, core.config.DefaultCoreTimeout)
khenaidoo54e0ddf2019-02-27 16:21:33 -0500137
Scott Bakeree6a0872019-10-29 15:59:52 -0700138 // Start the KafkaManager. This must be done after the deviceMgr, adapterMgr, and
139 // logicalDeviceMgr have been created, as once the kmp is started, it will register
140 // the above with the kmp.
141
142 go core.startKafkaManager(ctx,
143 core.config.ConnectionRetryInterval,
144 core.config.LiveProbeInterval,
145 core.config.NotLiveProbeInterval)
khenaidoob3244212019-08-27 14:32:27 -0400146
khenaidoob9203542018-09-17 22:56:37 -0400147 go core.startDeviceManager(ctx)
148 go core.startLogicalDeviceManager(ctx)
149 go core.startGRPCService(ctx)
khenaidoo21d51152019-02-01 13:48:37 -0500150 go core.startAdapterManager(ctx)
Girish Kumar4d3887d2019-11-22 14:22:05 +0000151 go core.monitorKvstoreLiveness(ctx)
khenaidoob9203542018-09-17 22:56:37 -0400152
Girish Kumarf56a4682020-03-20 20:07:46 +0000153 logger.Info("core-services-started")
Thomas Lee Se5a44012019-11-07 20:32:24 +0530154 return nil
khenaidoob9203542018-09-17 22:56:37 -0400155}
156
npujar1d86a522019-11-14 17:11:16 +0530157// Stop brings down core services
khenaidoob9203542018-09-17 22:56:37 -0400158func (core *Core) Stop(ctx context.Context) {
Scott Baker2d87ee32020-03-03 13:04:01 -0800159 core.stopOnce.Do(func() {
Girish Kumarf56a4682020-03-20 20:07:46 +0000160 logger.Info("stopping-adaptercore")
Scott Baker2d87ee32020-03-03 13:04:01 -0800161 // Signal to the KVStoreMonitor that we are stopping.
162 close(core.exitChannel)
163 // Stop all the started services
164 if core.grpcServer != nil {
165 core.grpcServer.Stop()
166 }
167 if core.logicalDeviceMgr != nil {
168 core.logicalDeviceMgr.stop(ctx)
169 }
170 if core.deviceMgr != nil {
171 core.deviceMgr.stop(ctx)
172 }
173 if core.kmp != nil {
174 core.kmp.Stop()
175 }
Girish Kumarf56a4682020-03-20 20:07:46 +0000176 logger.Info("adaptercore-stopped")
Scott Baker2d87ee32020-03-03 13:04:01 -0800177 })
khenaidoob9203542018-09-17 22:56:37 -0400178}
179
khenaidoo631fe542019-05-31 15:44:43 -0400180//startGRPCService creates the grpc service handlers, registers it to the grpc server and starts the server
khenaidoob9203542018-09-17 22:56:37 -0400181func (core *Core) startGRPCService(ctx context.Context) {
182 // create an insecure gserver server
Scott Bakeree6a0872019-10-29 15:59:52 -0700183 core.grpcServer = grpcserver.NewGrpcServer(core.config.GrpcHost, core.config.GrpcPort, nil, false, probe.GetProbeFromContext(ctx))
Girish Kumarf56a4682020-03-20 20:07:46 +0000184 logger.Info("grpc-server-created")
khenaidoob9203542018-09-17 22:56:37 -0400185
khenaidoo54e0ddf2019-02-27 16:21:33 -0500186 core.grpcNBIAPIHandler = NewAPIHandler(core)
Girish Kumarf56a4682020-03-20 20:07:46 +0000187 logger.Infow("grpc-handler", log.Fields{"core_binding_key": core.config.CoreBindingKey})
Richard Jankowskidbab94a2018-12-06 16:20:25 -0500188 core.logicalDeviceMgr.setGrpcNbiHandler(core.grpcNBIAPIHandler)
khenaidoob9203542018-09-17 22:56:37 -0400189 // Create a function to register the core GRPC service with the GRPC server
190 f := func(gs *grpc.Server) {
191 voltha.RegisterVolthaServiceServer(
192 gs,
Richard Jankowskidbab94a2018-12-06 16:20:25 -0500193 core.grpcNBIAPIHandler,
khenaidoob9203542018-09-17 22:56:37 -0400194 )
195 }
196
197 core.grpcServer.AddService(f)
Girish Kumarf56a4682020-03-20 20:07:46 +0000198 logger.Info("grpc-service-added")
khenaidoob9203542018-09-17 22:56:37 -0400199
David K. Bainbridgeb4a9ab02019-09-20 15:12:16 -0700200 /*
201 * Start the GRPC server
202 *
203 * This is a bit sub-optimal here as the grpcServer.Start call does not return (blocks)
204 * until something fails, but we want to send a "start" status update. As written this
205 * means that we are actually sending the "start" status update before the server is
206 * started, which means it is possible that the status is "running" before it actually is.
207 *
208 * This means that there is a small window in which the core could return its status as
209 * ready, when it really isn't.
210 */
211 probe.UpdateStatusFromContext(ctx, "grpc-service", probe.ServiceStatusRunning)
Girish Kumarf56a4682020-03-20 20:07:46 +0000212 logger.Info("grpc-server-started")
npujar467fe752020-01-16 20:17:45 +0530213 core.grpcServer.Start(ctx)
David K. Bainbridgeb4a9ab02019-09-20 15:12:16 -0700214 probe.UpdateStatusFromContext(ctx, "grpc-service", probe.ServiceStatusStopped)
khenaidoob9203542018-09-17 22:56:37 -0400215}
216
Scott Bakeree6a0872019-10-29 15:59:52 -0700217// Initialize the kafka manager, but we will start it later
npujar467fe752020-01-16 20:17:45 +0530218func (core *Core) initKafkaManager(ctx context.Context) {
Girish Kumarf56a4682020-03-20 20:07:46 +0000219 logger.Infow("initialize-kafka-manager", log.Fields{"host": core.config.KafkaAdapterHost,
khenaidoob9203542018-09-17 22:56:37 -0400220 "port": core.config.KafkaAdapterPort, "topic": core.config.CoreTopic})
Scott Bakeree6a0872019-10-29 15:59:52 -0700221
222 probe.UpdateStatusFromContext(ctx, "message-bus", probe.ServiceStatusPreparing)
223
224 // create the proxy
npujar467fe752020-01-16 20:17:45 +0530225 core.kmp = kafka.NewInterContainerProxy(
khenaidoo43c82122018-11-22 18:38:28 -0500226 kafka.InterContainerHost(core.config.KafkaAdapterHost),
227 kafka.InterContainerPort(core.config.KafkaAdapterPort),
228 kafka.MsgClient(core.kafkaClient),
khenaidoo79232702018-12-04 11:00:41 -0500229 kafka.DefaultTopic(&kafka.Topic{Name: core.config.CoreTopic}),
npujar467fe752020-01-16 20:17:45 +0530230 kafka.DeviceDiscoveryTopic(&kafka.Topic{Name: core.config.AffinityRouterTopic}))
Scott Bakeree6a0872019-10-29 15:59:52 -0700231
232 probe.UpdateStatusFromContext(ctx, "message-bus", probe.ServiceStatusPrepared)
Scott Bakeree6a0872019-10-29 15:59:52 -0700233}
234
235/*
236 * KafkaMonitorThread
237 *
npujar1d86a522019-11-14 17:11:16 +0530238 * Responsible for starting the Kafka Interadapter Proxy and monitoring its liveness
Scott Bakeree6a0872019-10-29 15:59:52 -0700239 * state.
240 *
241 * Any producer that fails to send will cause KafkaInterContainerProxy to
242 * post a false event on its liveness channel. Any producer that succeeds in sending
243 * will cause KafkaInterContainerProxy to post a true event on its liveness
npujar1d86a522019-11-14 17:11:16 +0530244 * channel. Group receivers also update liveness state, and a receiver will typically
Scott Bakeree6a0872019-10-29 15:59:52 -0700245 * indicate a loss of liveness within 3-5 seconds of Kafka going down. Receivers
246 * only indicate restoration of liveness if a message is received. During normal
247 * operation, messages will be routinely produced and received, automatically
248 * indicating liveness state. These routine liveness indications are rate-limited
249 * inside sarama_client.
250 *
251 * This thread monitors the status of KafkaInterContainerProxy's liveness and pushes
252 * that state to the core's readiness probes. If no liveness event has been seen
253 * within a timeout, then the thread will make an attempt to produce a "liveness"
254 * message, which will in turn trigger a liveness event on the liveness channel, true
255 * or false depending on whether the attempt succeeded.
256 *
257 * The gRPC server in turn monitors the state of the readiness probe and will
258 * start issuing UNAVAILABLE response while the probe is not ready.
259 *
260 * startupRetryInterval -- interval between attempts to start
261 * liveProbeInterval -- interval between liveness checks when in a live state
262 * notLiveProbeInterval -- interval between liveness checks when in a notLive state
263 *
264 * liveProbeInterval and notLiveProbeInterval can be configured separately,
265 * though the current default is that both are set to 60 seconds.
266 */
267
Girish Kumar4d3887d2019-11-22 14:22:05 +0000268func (core *Core) startKafkaManager(ctx context.Context, startupRetryInterval time.Duration, liveProbeInterval time.Duration, notLiveProbeInterval time.Duration) {
Girish Kumarf56a4682020-03-20 20:07:46 +0000269 logger.Infow("starting-kafka-manager-thread", log.Fields{"host": core.config.KafkaAdapterHost,
Scott Bakeree6a0872019-10-29 15:59:52 -0700270 "port": core.config.KafkaAdapterPort, "topic": core.config.CoreTopic})
271
272 started := false
273 for !started {
274 // If we haven't started yet, then try to start
Girish Kumarf56a4682020-03-20 20:07:46 +0000275 logger.Infow("starting-kafka-proxy", log.Fields{})
Scott Bakeree6a0872019-10-29 15:59:52 -0700276 if err := core.kmp.Start(); err != nil {
277 // We failed to start. Delay and then try again later.
278 // Don't worry about liveness, as we can't be live until we've started.
279 probe.UpdateStatusFromContext(ctx, "message-bus", probe.ServiceStatusNotReady)
Girish Kumarf56a4682020-03-20 20:07:46 +0000280 logger.Infow("error-starting-kafka-messaging-proxy", log.Fields{"error": err})
Girish Kumar4d3887d2019-11-22 14:22:05 +0000281 time.Sleep(startupRetryInterval)
khenaidoob3244212019-08-27 14:32:27 -0400282 } else {
Scott Bakeree6a0872019-10-29 15:59:52 -0700283 // We started. We only need to do this once.
284 // Next we'll fall through and start checking liveness.
Girish Kumarf56a4682020-03-20 20:07:46 +0000285 logger.Infow("started-kafka-proxy", log.Fields{})
Scott Bakeree6a0872019-10-29 15:59:52 -0700286
287 // cannot do this until after the kmp is started
npujar1d86a522019-11-14 17:11:16 +0530288 if err := core.registerAdapterRequestHandlers(ctx, core.instanceID, core.deviceMgr, core.logicalDeviceMgr, core.adapterMgr, core.clusterDataProxy, core.localDataProxy); err != nil {
Girish Kumarf56a4682020-03-20 20:07:46 +0000289 logger.Fatal("Failure-registering-adapterRequestHandler")
Scott Bakeree6a0872019-10-29 15:59:52 -0700290 }
291
292 started = true
khenaidoob3244212019-08-27 14:32:27 -0400293 }
khenaidoob9203542018-09-17 22:56:37 -0400294 }
Scott Bakeree6a0872019-10-29 15:59:52 -0700295
Girish Kumarf56a4682020-03-20 20:07:46 +0000296 logger.Info("started-kafka-message-proxy")
Scott Bakeree6a0872019-10-29 15:59:52 -0700297
298 livenessChannel := core.kmp.EnableLivenessChannel(true)
299
Girish Kumarf56a4682020-03-20 20:07:46 +0000300 logger.Info("enabled-kafka-liveness-channel")
Scott Bakeree6a0872019-10-29 15:59:52 -0700301
Girish Kumar4d3887d2019-11-22 14:22:05 +0000302 timeout := liveProbeInterval
Scott Bakeree6a0872019-10-29 15:59:52 -0700303 for {
304 timeoutTimer := time.NewTimer(timeout)
305 select {
306 case liveness := <-livenessChannel:
Girish Kumarf56a4682020-03-20 20:07:46 +0000307 logger.Infow("kafka-manager-thread-liveness-event", log.Fields{"liveness": liveness})
Scott Bakeree6a0872019-10-29 15:59:52 -0700308 // there was a state change in Kafka liveness
309 if !liveness {
310 probe.UpdateStatusFromContext(ctx, "message-bus", probe.ServiceStatusNotReady)
311
312 if core.grpcServer != nil {
Girish Kumarf56a4682020-03-20 20:07:46 +0000313 logger.Info("kafka-manager-thread-set-server-notready")
Scott Bakeree6a0872019-10-29 15:59:52 -0700314 }
315
316 // retry frequently while life is bad
Girish Kumar4d3887d2019-11-22 14:22:05 +0000317 timeout = notLiveProbeInterval
Scott Bakeree6a0872019-10-29 15:59:52 -0700318 } else {
319 probe.UpdateStatusFromContext(ctx, "message-bus", probe.ServiceStatusRunning)
320
321 if core.grpcServer != nil {
Girish Kumarf56a4682020-03-20 20:07:46 +0000322 logger.Info("kafka-manager-thread-set-server-ready")
Scott Bakeree6a0872019-10-29 15:59:52 -0700323 }
324
325 // retry infrequently while life is good
Girish Kumar4d3887d2019-11-22 14:22:05 +0000326 timeout = liveProbeInterval
Scott Bakeree6a0872019-10-29 15:59:52 -0700327 }
328 if !timeoutTimer.Stop() {
329 <-timeoutTimer.C
330 }
331 case <-timeoutTimer.C:
Girish Kumarf56a4682020-03-20 20:07:46 +0000332 logger.Info("kafka-proxy-liveness-recheck")
Scott Bakeree6a0872019-10-29 15:59:52 -0700333 // send the liveness probe in a goroutine; we don't want to deadlock ourselves as
334 // the liveness probe may wait (and block) writing to our channel.
335 go func() {
336 err := core.kmp.SendLiveness()
337 if err != nil {
338 // Catch possible error case if sending liveness after Sarama has been stopped.
Girish Kumarf56a4682020-03-20 20:07:46 +0000339 logger.Warnw("error-kafka-send-liveness", log.Fields{"error": err})
Scott Bakeree6a0872019-10-29 15:59:52 -0700340 }
341 }()
342 }
343 }
khenaidoob9203542018-09-17 22:56:37 -0400344}
345
khenaidoob3244212019-08-27 14:32:27 -0400346// waitUntilKVStoreReachableOrMaxTries will wait until it can connect to a KV store or until maxtries has been reached
Girish Kumar4d3887d2019-11-22 14:22:05 +0000347func (core *Core) waitUntilKVStoreReachableOrMaxTries(ctx context.Context, maxRetries int, retryInterval time.Duration) error {
Girish Kumarf56a4682020-03-20 20:07:46 +0000348 logger.Infow("verifying-KV-store-connectivity", log.Fields{"host": core.config.KVStoreHost,
khenaidoob3244212019-08-27 14:32:27 -0400349 "port": core.config.KVStorePort, "retries": maxRetries, "retryInterval": retryInterval})
khenaidoob3244212019-08-27 14:32:27 -0400350 count := 0
351 for {
npujar467fe752020-01-16 20:17:45 +0530352 if !core.kvClient.IsConnectionUp(ctx) {
Girish Kumarf56a4682020-03-20 20:07:46 +0000353 logger.Info("KV-store-unreachable")
khenaidoob3244212019-08-27 14:32:27 -0400354 if maxRetries != -1 {
355 if count >= maxRetries {
356 return status.Error(codes.Unavailable, "kv store unreachable")
357 }
358 }
npujar1d86a522019-11-14 17:11:16 +0530359 count++
khenaidoob3244212019-08-27 14:32:27 -0400360 // Take a nap before retrying
Girish Kumar4d3887d2019-11-22 14:22:05 +0000361 time.Sleep(retryInterval)
Girish Kumarf56a4682020-03-20 20:07:46 +0000362 logger.Infow("retry-KV-store-connectivity", log.Fields{"retryCount": count, "maxRetries": maxRetries, "retryInterval": retryInterval})
khenaidoob3244212019-08-27 14:32:27 -0400363
364 } else {
365 break
366 }
367 }
Girish Kumarf56a4682020-03-20 20:07:46 +0000368 logger.Info("KV-store-reachable")
khenaidoob3244212019-08-27 14:32:27 -0400369 return nil
370}
371
npujar1d86a522019-11-14 17:11:16 +0530372func (core *Core) registerAdapterRequestHandlers(ctx context.Context, coreInstanceID string, dMgr *DeviceManager,
khenaidoo297cd252019-02-07 22:10:23 -0500373 ldMgr *LogicalDeviceManager, aMgr *AdapterManager, cdProxy *model.Proxy, ldProxy *model.Proxy,
khenaidoo54e0ddf2019-02-27 16:21:33 -0500374) error {
npujar1d86a522019-11-14 17:11:16 +0530375 requestProxy := NewAdapterRequestHandlerProxy(core, coreInstanceID, dMgr, ldMgr, aMgr, cdProxy, ldProxy,
David Bainbridgefd27f4b2020-03-26 18:27:41 -0700376 core.config.LongRunningRequestTimeout, core.config.DefaultRequestTimeout)
khenaidoob9203542018-09-17 22:56:37 -0400377
khenaidoo54e0ddf2019-02-27 16:21:33 -0500378 // Register the broadcast topic to handle any core-bound broadcast requests
379 if err := core.kmp.SubscribeWithRequestHandlerInterface(kafka.Topic{Name: core.config.CoreTopic}, requestProxy); err != nil {
Girish Kumarf56a4682020-03-20 20:07:46 +0000380 logger.Fatalw("Failed-registering-broadcast-handler", log.Fields{"topic": core.config.CoreTopic})
khenaidoo54e0ddf2019-02-27 16:21:33 -0500381 return err
382 }
383
Kent Hagermana6d0c362019-07-30 12:50:21 -0400384 // Register the core-pair topic to handle core-bound requests destined to the core pair
385 if err := core.kmp.SubscribeWithDefaultRequestHandler(kafka.Topic{Name: core.config.CorePairTopic}, kafka.OffsetNewest); err != nil {
Girish Kumarf56a4682020-03-20 20:07:46 +0000386 logger.Fatalw("Failed-registering-pair-handler", log.Fields{"topic": core.config.CorePairTopic})
Kent Hagermana6d0c362019-07-30 12:50:21 -0400387 return err
388 }
389
Girish Kumarf56a4682020-03-20 20:07:46 +0000390 logger.Info("request-handler-registered")
khenaidoob9203542018-09-17 22:56:37 -0400391 return nil
392}
393
394func (core *Core) startDeviceManager(ctx context.Context) {
Girish Kumarf56a4682020-03-20 20:07:46 +0000395 logger.Info("DeviceManager-Starting...")
khenaidoo4d4802d2018-10-04 21:59:49 -0400396 core.deviceMgr.start(ctx, core.logicalDeviceMgr)
Girish Kumarf56a4682020-03-20 20:07:46 +0000397 logger.Info("DeviceManager-Started")
khenaidoob9203542018-09-17 22:56:37 -0400398}
399
400func (core *Core) startLogicalDeviceManager(ctx context.Context) {
Girish Kumarf56a4682020-03-20 20:07:46 +0000401 logger.Info("Logical-DeviceManager-Starting...")
khenaidoo4d4802d2018-10-04 21:59:49 -0400402 core.logicalDeviceMgr.start(ctx)
Girish Kumarf56a4682020-03-20 20:07:46 +0000403 logger.Info("Logical-DeviceManager-Started")
khenaidoob9203542018-09-17 22:56:37 -0400404}
khenaidoo21d51152019-02-01 13:48:37 -0500405
406func (core *Core) startAdapterManager(ctx context.Context) {
Girish Kumarf56a4682020-03-20 20:07:46 +0000407 logger.Info("Adapter-Manager-Starting...")
Thomas Lee Se5a44012019-11-07 20:32:24 +0530408 err := core.adapterMgr.start(ctx)
409 if err != nil {
Girish Kumarf56a4682020-03-20 20:07:46 +0000410 logger.Fatalf("failed-to-start-adapter-manager: error %v ", err)
Thomas Lee Se5a44012019-11-07 20:32:24 +0530411 }
Girish Kumarf56a4682020-03-20 20:07:46 +0000412 logger.Info("Adapter-Manager-Started")
William Kurkiandaa6bb22019-03-07 12:26:28 -0500413}
Girish Kumar4d3887d2019-11-22 14:22:05 +0000414
415/*
416* Thread to monitor kvstore Liveness (connection status)
417*
418* This function constantly monitors Liveness State of kvstore as reported
419* periodically by backend and updates the Status of kv-store service registered
420* with rw_core probe.
421*
422* If no liveness event has been seen within a timeout, then the thread will
423* perform a "liveness" check attempt, which will in turn trigger a liveness event on
424* the liveness channel, true or false depending on whether the attempt succeeded.
425*
426* The gRPC server in turn monitors the state of the readiness probe and will
427* start issuing UNAVAILABLE response while the probe is not ready.
428 */
429func (core *Core) monitorKvstoreLiveness(ctx context.Context) {
Girish Kumarf56a4682020-03-20 20:07:46 +0000430 logger.Info("start-monitoring-kvstore-liveness")
Girish Kumar4d3887d2019-11-22 14:22:05 +0000431
432 // Instruct backend to create Liveness channel for transporting state updates
433 livenessChannel := core.backend.EnableLivenessChannel()
434
Girish Kumarf56a4682020-03-20 20:07:46 +0000435 logger.Debug("enabled-kvstore-liveness-channel")
Girish Kumar4d3887d2019-11-22 14:22:05 +0000436
437 // Default state for kvstore is alive for rw_core
438 timeout := core.config.LiveProbeInterval
Scott Baker2d87ee32020-03-03 13:04:01 -0800439loop:
Girish Kumar4d3887d2019-11-22 14:22:05 +0000440 for {
441 timeoutTimer := time.NewTimer(timeout)
442 select {
443
444 case liveness := <-livenessChannel:
Girish Kumarf56a4682020-03-20 20:07:46 +0000445 logger.Debugw("received-liveness-change-notification", log.Fields{"liveness": liveness})
Girish Kumar4d3887d2019-11-22 14:22:05 +0000446
447 if !liveness {
448 probe.UpdateStatusFromContext(ctx, "kv-store", probe.ServiceStatusNotReady)
449
450 if core.grpcServer != nil {
Girish Kumarf56a4682020-03-20 20:07:46 +0000451 logger.Info("kvstore-set-server-notready")
Girish Kumar4d3887d2019-11-22 14:22:05 +0000452 }
453
454 timeout = core.config.NotLiveProbeInterval
455
456 } else {
457 probe.UpdateStatusFromContext(ctx, "kv-store", probe.ServiceStatusRunning)
458
459 if core.grpcServer != nil {
Girish Kumarf56a4682020-03-20 20:07:46 +0000460 logger.Info("kvstore-set-server-ready")
Girish Kumar4d3887d2019-11-22 14:22:05 +0000461 }
462
463 timeout = core.config.LiveProbeInterval
464 }
465
466 if !timeoutTimer.Stop() {
467 <-timeoutTimer.C
468 }
469
Scott Baker2d87ee32020-03-03 13:04:01 -0800470 case <-core.exitChannel:
471 break loop
472
Girish Kumar4d3887d2019-11-22 14:22:05 +0000473 case <-timeoutTimer.C:
Girish Kumarf56a4682020-03-20 20:07:46 +0000474 logger.Info("kvstore-perform-liveness-check-on-timeout")
Girish Kumar4d3887d2019-11-22 14:22:05 +0000475
476 // Trigger Liveness check if no liveness update received within the timeout period.
477 // The Liveness check will push Live state to same channel which this routine is
478 // reading and processing. This, do it asynchronously to avoid blocking for
479 // backend response and avoid any possibility of deadlock
npujar467fe752020-01-16 20:17:45 +0530480 go core.backend.PerformLivenessCheck(ctx)
Girish Kumar4d3887d2019-11-22 14:22:05 +0000481 }
482 }
483}