khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2018-present Open Networking Foundation |
| 3 | |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
npujar | 1d86a52 | 2019-11-14 17:11:16 +0530 | [diff] [blame] | 16 | |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 17 | package core |
| 18 | |
| 19 | import ( |
| 20 | "context" |
Scott Baker | 2d87ee3 | 2020-03-03 13:04:01 -0800 | [diff] [blame] | 21 | "sync" |
npujar | 1d86a52 | 2019-11-14 17:11:16 +0530 | [diff] [blame] | 22 | "time" |
| 23 | |
sbarbari | 17d7e22 | 2019-11-05 10:02:29 -0500 | [diff] [blame] | 24 | "github.com/opencord/voltha-go/db/model" |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 25 | "github.com/opencord/voltha-go/rw_core/config" |
Kent Hagerman | 2b21604 | 2020-04-03 18:28:56 -0400 | [diff] [blame] | 26 | "github.com/opencord/voltha-go/rw_core/core/adapter" |
| 27 | "github.com/opencord/voltha-go/rw_core/core/api" |
| 28 | "github.com/opencord/voltha-go/rw_core/core/device" |
serkant.uluderya | 2ae470f | 2020-01-21 11:13:09 -0800 | [diff] [blame] | 29 | "github.com/opencord/voltha-lib-go/v3/pkg/db" |
| 30 | "github.com/opencord/voltha-lib-go/v3/pkg/db/kvstore" |
| 31 | grpcserver "github.com/opencord/voltha-lib-go/v3/pkg/grpc" |
| 32 | "github.com/opencord/voltha-lib-go/v3/pkg/kafka" |
| 33 | "github.com/opencord/voltha-lib-go/v3/pkg/log" |
| 34 | "github.com/opencord/voltha-lib-go/v3/pkg/probe" |
| 35 | "github.com/opencord/voltha-protos/v3/go/voltha" |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 36 | "google.golang.org/grpc" |
khenaidoo | b324421 | 2019-08-27 14:32:27 -0400 | [diff] [blame] | 37 | "google.golang.org/grpc/codes" |
| 38 | "google.golang.org/grpc/status" |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 39 | ) |
| 40 | |
npujar | 1d86a52 | 2019-11-14 17:11:16 +0530 | [diff] [blame] | 41 | // Core represent read,write core attributes |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 42 | type Core struct { |
npujar | 1d86a52 | 2019-11-14 17:11:16 +0530 | [diff] [blame] | 43 | instanceID string |
Kent Hagerman | 2b21604 | 2020-04-03 18:28:56 -0400 | [diff] [blame] | 44 | deviceMgr *device.Manager |
| 45 | logicalDeviceMgr *device.LogicalManager |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 46 | grpcServer *grpcserver.GrpcServer |
Kent Hagerman | 2b21604 | 2020-04-03 18:28:56 -0400 | [diff] [blame] | 47 | grpcNBIAPIHandler *api.NBIHandler |
| 48 | adapterMgr *adapter.Manager |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 49 | config *config.RWCoreFlags |
npujar | 467fe75 | 2020-01-16 20:17:45 +0530 | [diff] [blame] | 50 | kmp kafka.InterContainerProxy |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 51 | clusterDataProxy *model.Proxy |
| 52 | localDataProxy *model.Proxy |
Scott Baker | 2d87ee3 | 2020-03-03 13:04:01 -0800 | [diff] [blame] | 53 | exitChannel chan struct{} |
| 54 | stopOnce sync.Once |
Richard Jankowski | e4d7766 | 2018-10-17 13:53:21 -0400 | [diff] [blame] | 55 | kvClient kvstore.Client |
Girish Kumar | 4d3887d | 2019-11-22 14:22:05 +0000 | [diff] [blame] | 56 | backend db.Backend |
khenaidoo | 43c8212 | 2018-11-22 18:38:28 -0500 | [diff] [blame] | 57 | kafkaClient kafka.Client |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 58 | } |
| 59 | |
npujar | 1d86a52 | 2019-11-14 17:11:16 +0530 | [diff] [blame] | 60 | // NewCore creates instance of rw core |
Thomas Lee S | e5a4401 | 2019-11-07 20:32:24 +0530 | [diff] [blame] | 61 | func NewCore(ctx context.Context, id string, cf *config.RWCoreFlags, kvClient kvstore.Client, kafkaClient kafka.Client) *Core { |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 62 | var core Core |
npujar | 1d86a52 | 2019-11-14 17:11:16 +0530 | [diff] [blame] | 63 | core.instanceID = id |
Scott Baker | 2d87ee3 | 2020-03-03 13:04:01 -0800 | [diff] [blame] | 64 | core.exitChannel = make(chan struct{}) |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 65 | core.config = cf |
Richard Jankowski | e4d7766 | 2018-10-17 13:53:21 -0400 | [diff] [blame] | 66 | core.kvClient = kvClient |
khenaidoo | 43c8212 | 2018-11-22 18:38:28 -0500 | [diff] [blame] | 67 | core.kafkaClient = kafkaClient |
Richard Jankowski | e4d7766 | 2018-10-17 13:53:21 -0400 | [diff] [blame] | 68 | |
Girish Kumar | 4d3887d | 2019-11-22 14:22:05 +0000 | [diff] [blame] | 69 | // Configure backend to push Liveness Status at least every (cf.LiveProbeInterval / 2) seconds |
| 70 | // so as to avoid trigger of Liveness check (due to Liveness timeout) when backend is alive |
| 71 | livenessChannelInterval := cf.LiveProbeInterval / 2 |
| 72 | |
Richard Jankowski | e4d7766 | 2018-10-17 13:53:21 -0400 | [diff] [blame] | 73 | // Setup the KV store |
Girish Kumar | 4d3887d | 2019-11-22 14:22:05 +0000 | [diff] [blame] | 74 | core.backend = db.Backend{ |
| 75 | Client: kvClient, |
| 76 | StoreType: cf.KVStoreType, |
| 77 | Host: cf.KVStoreHost, |
| 78 | Port: cf.KVStorePort, |
| 79 | Timeout: cf.KVStoreTimeout, |
| 80 | LivenessChannelInterval: livenessChannelInterval, |
| 81 | PathPrefix: cf.KVStoreDataPrefix} |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 82 | return &core |
| 83 | } |
| 84 | |
npujar | 1d86a52 | 2019-11-14 17:11:16 +0530 | [diff] [blame] | 85 | // Start brings up core services |
Thomas Lee S | e5a4401 | 2019-11-07 20:32:24 +0530 | [diff] [blame] | 86 | func (core *Core) Start(ctx context.Context) error { |
David K. Bainbridge | b4a9ab0 | 2019-09-20 15:12:16 -0700 | [diff] [blame] | 87 | |
| 88 | // If the context has a probe then fetch it and register our services |
| 89 | var p *probe.Probe |
| 90 | if value := ctx.Value(probe.ProbeContextKey); value != nil { |
| 91 | if _, ok := value.(*probe.Probe); ok { |
| 92 | p = value.(*probe.Probe) |
| 93 | p.RegisterService( |
| 94 | "message-bus", |
| 95 | "kv-store", |
| 96 | "device-manager", |
| 97 | "logical-device-manager", |
| 98 | "adapter-manager", |
| 99 | "grpc-service", |
| 100 | ) |
| 101 | } |
| 102 | } |
| 103 | |
Girish Kumar | f56a468 | 2020-03-20 20:07:46 +0000 | [diff] [blame] | 104 | logger.Info("starting-core-services", log.Fields{"coreId": core.instanceID}) |
khenaidoo | b324421 | 2019-08-27 14:32:27 -0400 | [diff] [blame] | 105 | |
| 106 | // Wait until connection to KV Store is up |
| 107 | if err := core.waitUntilKVStoreReachableOrMaxTries(ctx, core.config.MaxConnectionRetries, core.config.ConnectionRetryInterval); err != nil { |
Girish Kumar | f56a468 | 2020-03-20 20:07:46 +0000 | [diff] [blame] | 108 | logger.Fatal("Unable-to-connect-to-KV-store") |
khenaidoo | b324421 | 2019-08-27 14:32:27 -0400 | [diff] [blame] | 109 | } |
David K. Bainbridge | b4a9ab0 | 2019-09-20 15:12:16 -0700 | [diff] [blame] | 110 | if p != nil { |
| 111 | p.UpdateStatus("kv-store", probe.ServiceStatusRunning) |
| 112 | } |
Thomas Lee S | e5a4401 | 2019-11-07 20:32:24 +0530 | [diff] [blame] | 113 | |
Kent Hagerman | 2b21604 | 2020-04-03 18:28:56 -0400 | [diff] [blame] | 114 | endpointMgr := kafka.NewEndpointManager(&core.backend) |
| 115 | |
Kent Hagerman | 4f355f5 | 2020-03-30 16:01:33 -0400 | [diff] [blame] | 116 | core.clusterDataProxy = model.NewProxy(&core.backend, "/") |
| 117 | core.localDataProxy = model.NewProxy(&core.backend, "/") |
khenaidoo | b324421 | 2019-08-27 14:32:27 -0400 | [diff] [blame] | 118 | |
Scott Baker | ee6a087 | 2019-10-29 15:59:52 -0700 | [diff] [blame] | 119 | // core.kmp must be created before deviceMgr and adapterMgr, as they will make |
| 120 | // private copies of the poiner to core.kmp. |
npujar | 467fe75 | 2020-01-16 20:17:45 +0530 | [diff] [blame] | 121 | core.initKafkaManager(ctx) |
khenaidoo | b324421 | 2019-08-27 14:32:27 -0400 | [diff] [blame] | 122 | |
Girish Kumar | f56a468 | 2020-03-20 20:07:46 +0000 | [diff] [blame] | 123 | logger.Debugw("values", log.Fields{"kmp": core.kmp}) |
Kent Hagerman | 2b21604 | 2020-04-03 18:28:56 -0400 | [diff] [blame] | 124 | core.adapterMgr = adapter.NewAdapterManager(core.clusterDataProxy, core.instanceID, core.kafkaClient) |
Kent Hagerman | 45a13e4 | 2020-04-13 12:23:50 -0400 | [diff] [blame^] | 125 | core.deviceMgr, core.logicalDeviceMgr = device.NewManagers(core.clusterDataProxy, core.adapterMgr, core.kmp, endpointMgr, core.config.CorePairTopic, core.instanceID, core.config.DefaultCoreTimeout) |
khenaidoo | 54e0ddf | 2019-02-27 16:21:33 -0500 | [diff] [blame] | 126 | |
Scott Baker | ee6a087 | 2019-10-29 15:59:52 -0700 | [diff] [blame] | 127 | // Start the KafkaManager. This must be done after the deviceMgr, adapterMgr, and |
| 128 | // logicalDeviceMgr have been created, as once the kmp is started, it will register |
| 129 | // the above with the kmp. |
| 130 | |
| 131 | go core.startKafkaManager(ctx, |
| 132 | core.config.ConnectionRetryInterval, |
| 133 | core.config.LiveProbeInterval, |
| 134 | core.config.NotLiveProbeInterval) |
khenaidoo | b324421 | 2019-08-27 14:32:27 -0400 | [diff] [blame] | 135 | |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 136 | go core.startDeviceManager(ctx) |
| 137 | go core.startLogicalDeviceManager(ctx) |
| 138 | go core.startGRPCService(ctx) |
khenaidoo | 21d5115 | 2019-02-01 13:48:37 -0500 | [diff] [blame] | 139 | go core.startAdapterManager(ctx) |
Girish Kumar | 4d3887d | 2019-11-22 14:22:05 +0000 | [diff] [blame] | 140 | go core.monitorKvstoreLiveness(ctx) |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 141 | |
Girish Kumar | f56a468 | 2020-03-20 20:07:46 +0000 | [diff] [blame] | 142 | logger.Info("core-services-started") |
Thomas Lee S | e5a4401 | 2019-11-07 20:32:24 +0530 | [diff] [blame] | 143 | return nil |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 144 | } |
| 145 | |
npujar | 1d86a52 | 2019-11-14 17:11:16 +0530 | [diff] [blame] | 146 | // Stop brings down core services |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 147 | func (core *Core) Stop(ctx context.Context) { |
Scott Baker | 2d87ee3 | 2020-03-03 13:04:01 -0800 | [diff] [blame] | 148 | core.stopOnce.Do(func() { |
Girish Kumar | f56a468 | 2020-03-20 20:07:46 +0000 | [diff] [blame] | 149 | logger.Info("stopping-adaptercore") |
Scott Baker | 2d87ee3 | 2020-03-03 13:04:01 -0800 | [diff] [blame] | 150 | // Signal to the KVStoreMonitor that we are stopping. |
| 151 | close(core.exitChannel) |
| 152 | // Stop all the started services |
| 153 | if core.grpcServer != nil { |
| 154 | core.grpcServer.Stop() |
| 155 | } |
| 156 | if core.logicalDeviceMgr != nil { |
Kent Hagerman | 2b21604 | 2020-04-03 18:28:56 -0400 | [diff] [blame] | 157 | core.logicalDeviceMgr.Stop(ctx) |
Scott Baker | 2d87ee3 | 2020-03-03 13:04:01 -0800 | [diff] [blame] | 158 | } |
| 159 | if core.deviceMgr != nil { |
Kent Hagerman | 2b21604 | 2020-04-03 18:28:56 -0400 | [diff] [blame] | 160 | core.deviceMgr.Stop(ctx) |
Scott Baker | 2d87ee3 | 2020-03-03 13:04:01 -0800 | [diff] [blame] | 161 | } |
| 162 | if core.kmp != nil { |
| 163 | core.kmp.Stop() |
| 164 | } |
Girish Kumar | f56a468 | 2020-03-20 20:07:46 +0000 | [diff] [blame] | 165 | logger.Info("adaptercore-stopped") |
Scott Baker | 2d87ee3 | 2020-03-03 13:04:01 -0800 | [diff] [blame] | 166 | }) |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 167 | } |
| 168 | |
khenaidoo | 631fe54 | 2019-05-31 15:44:43 -0400 | [diff] [blame] | 169 | //startGRPCService creates the grpc service handlers, registers it to the grpc server and starts the server |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 170 | func (core *Core) startGRPCService(ctx context.Context) { |
| 171 | // create an insecure gserver server |
Scott Baker | ee6a087 | 2019-10-29 15:59:52 -0700 | [diff] [blame] | 172 | core.grpcServer = grpcserver.NewGrpcServer(core.config.GrpcHost, core.config.GrpcPort, nil, false, probe.GetProbeFromContext(ctx)) |
Girish Kumar | f56a468 | 2020-03-20 20:07:46 +0000 | [diff] [blame] | 173 | logger.Info("grpc-server-created") |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 174 | |
Kent Hagerman | 45a13e4 | 2020-04-13 12:23:50 -0400 | [diff] [blame^] | 175 | core.grpcNBIAPIHandler = api.NewNBIHandler(core.deviceMgr, core.logicalDeviceMgr, core.adapterMgr) |
Girish Kumar | f56a468 | 2020-03-20 20:07:46 +0000 | [diff] [blame] | 176 | logger.Infow("grpc-handler", log.Fields{"core_binding_key": core.config.CoreBindingKey}) |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 177 | // Create a function to register the core GRPC service with the GRPC server |
| 178 | f := func(gs *grpc.Server) { |
| 179 | voltha.RegisterVolthaServiceServer( |
| 180 | gs, |
Richard Jankowski | dbab94a | 2018-12-06 16:20:25 -0500 | [diff] [blame] | 181 | core.grpcNBIAPIHandler, |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 182 | ) |
| 183 | } |
| 184 | |
| 185 | core.grpcServer.AddService(f) |
Girish Kumar | f56a468 | 2020-03-20 20:07:46 +0000 | [diff] [blame] | 186 | logger.Info("grpc-service-added") |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 187 | |
David K. Bainbridge | b4a9ab0 | 2019-09-20 15:12:16 -0700 | [diff] [blame] | 188 | /* |
| 189 | * Start the GRPC server |
| 190 | * |
| 191 | * This is a bit sub-optimal here as the grpcServer.Start call does not return (blocks) |
| 192 | * until something fails, but we want to send a "start" status update. As written this |
| 193 | * means that we are actually sending the "start" status update before the server is |
| 194 | * started, which means it is possible that the status is "running" before it actually is. |
| 195 | * |
| 196 | * This means that there is a small window in which the core could return its status as |
| 197 | * ready, when it really isn't. |
| 198 | */ |
| 199 | probe.UpdateStatusFromContext(ctx, "grpc-service", probe.ServiceStatusRunning) |
Girish Kumar | f56a468 | 2020-03-20 20:07:46 +0000 | [diff] [blame] | 200 | logger.Info("grpc-server-started") |
npujar | 467fe75 | 2020-01-16 20:17:45 +0530 | [diff] [blame] | 201 | core.grpcServer.Start(ctx) |
David K. Bainbridge | b4a9ab0 | 2019-09-20 15:12:16 -0700 | [diff] [blame] | 202 | probe.UpdateStatusFromContext(ctx, "grpc-service", probe.ServiceStatusStopped) |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 203 | } |
| 204 | |
Scott Baker | ee6a087 | 2019-10-29 15:59:52 -0700 | [diff] [blame] | 205 | // Initialize the kafka manager, but we will start it later |
npujar | 467fe75 | 2020-01-16 20:17:45 +0530 | [diff] [blame] | 206 | func (core *Core) initKafkaManager(ctx context.Context) { |
Girish Kumar | f56a468 | 2020-03-20 20:07:46 +0000 | [diff] [blame] | 207 | logger.Infow("initialize-kafka-manager", log.Fields{"host": core.config.KafkaAdapterHost, |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 208 | "port": core.config.KafkaAdapterPort, "topic": core.config.CoreTopic}) |
Scott Baker | ee6a087 | 2019-10-29 15:59:52 -0700 | [diff] [blame] | 209 | |
| 210 | probe.UpdateStatusFromContext(ctx, "message-bus", probe.ServiceStatusPreparing) |
| 211 | |
| 212 | // create the proxy |
npujar | 467fe75 | 2020-01-16 20:17:45 +0530 | [diff] [blame] | 213 | core.kmp = kafka.NewInterContainerProxy( |
khenaidoo | 43c8212 | 2018-11-22 18:38:28 -0500 | [diff] [blame] | 214 | kafka.InterContainerHost(core.config.KafkaAdapterHost), |
| 215 | kafka.InterContainerPort(core.config.KafkaAdapterPort), |
| 216 | kafka.MsgClient(core.kafkaClient), |
khenaidoo | 7923270 | 2018-12-04 11:00:41 -0500 | [diff] [blame] | 217 | kafka.DefaultTopic(&kafka.Topic{Name: core.config.CoreTopic}), |
npujar | 467fe75 | 2020-01-16 20:17:45 +0530 | [diff] [blame] | 218 | kafka.DeviceDiscoveryTopic(&kafka.Topic{Name: core.config.AffinityRouterTopic})) |
Scott Baker | ee6a087 | 2019-10-29 15:59:52 -0700 | [diff] [blame] | 219 | |
| 220 | probe.UpdateStatusFromContext(ctx, "message-bus", probe.ServiceStatusPrepared) |
Scott Baker | ee6a087 | 2019-10-29 15:59:52 -0700 | [diff] [blame] | 221 | } |
| 222 | |
| 223 | /* |
| 224 | * KafkaMonitorThread |
| 225 | * |
npujar | 1d86a52 | 2019-11-14 17:11:16 +0530 | [diff] [blame] | 226 | * Responsible for starting the Kafka Interadapter Proxy and monitoring its liveness |
Scott Baker | ee6a087 | 2019-10-29 15:59:52 -0700 | [diff] [blame] | 227 | * state. |
| 228 | * |
| 229 | * Any producer that fails to send will cause KafkaInterContainerProxy to |
| 230 | * post a false event on its liveness channel. Any producer that succeeds in sending |
| 231 | * will cause KafkaInterContainerProxy to post a true event on its liveness |
npujar | 1d86a52 | 2019-11-14 17:11:16 +0530 | [diff] [blame] | 232 | * channel. Group receivers also update liveness state, and a receiver will typically |
Scott Baker | ee6a087 | 2019-10-29 15:59:52 -0700 | [diff] [blame] | 233 | * indicate a loss of liveness within 3-5 seconds of Kafka going down. Receivers |
| 234 | * only indicate restoration of liveness if a message is received. During normal |
| 235 | * operation, messages will be routinely produced and received, automatically |
| 236 | * indicating liveness state. These routine liveness indications are rate-limited |
| 237 | * inside sarama_client. |
| 238 | * |
| 239 | * This thread monitors the status of KafkaInterContainerProxy's liveness and pushes |
| 240 | * that state to the core's readiness probes. If no liveness event has been seen |
| 241 | * within a timeout, then the thread will make an attempt to produce a "liveness" |
| 242 | * message, which will in turn trigger a liveness event on the liveness channel, true |
| 243 | * or false depending on whether the attempt succeeded. |
| 244 | * |
| 245 | * The gRPC server in turn monitors the state of the readiness probe and will |
| 246 | * start issuing UNAVAILABLE response while the probe is not ready. |
| 247 | * |
| 248 | * startupRetryInterval -- interval between attempts to start |
| 249 | * liveProbeInterval -- interval between liveness checks when in a live state |
| 250 | * notLiveProbeInterval -- interval between liveness checks when in a notLive state |
| 251 | * |
| 252 | * liveProbeInterval and notLiveProbeInterval can be configured separately, |
| 253 | * though the current default is that both are set to 60 seconds. |
| 254 | */ |
| 255 | |
Girish Kumar | 4d3887d | 2019-11-22 14:22:05 +0000 | [diff] [blame] | 256 | func (core *Core) startKafkaManager(ctx context.Context, startupRetryInterval time.Duration, liveProbeInterval time.Duration, notLiveProbeInterval time.Duration) { |
Girish Kumar | f56a468 | 2020-03-20 20:07:46 +0000 | [diff] [blame] | 257 | logger.Infow("starting-kafka-manager-thread", log.Fields{"host": core.config.KafkaAdapterHost, |
Scott Baker | ee6a087 | 2019-10-29 15:59:52 -0700 | [diff] [blame] | 258 | "port": core.config.KafkaAdapterPort, "topic": core.config.CoreTopic}) |
| 259 | |
| 260 | started := false |
| 261 | for !started { |
| 262 | // If we haven't started yet, then try to start |
Girish Kumar | f56a468 | 2020-03-20 20:07:46 +0000 | [diff] [blame] | 263 | logger.Infow("starting-kafka-proxy", log.Fields{}) |
Scott Baker | ee6a087 | 2019-10-29 15:59:52 -0700 | [diff] [blame] | 264 | if err := core.kmp.Start(); err != nil { |
| 265 | // We failed to start. Delay and then try again later. |
| 266 | // Don't worry about liveness, as we can't be live until we've started. |
| 267 | probe.UpdateStatusFromContext(ctx, "message-bus", probe.ServiceStatusNotReady) |
Girish Kumar | f56a468 | 2020-03-20 20:07:46 +0000 | [diff] [blame] | 268 | logger.Infow("error-starting-kafka-messaging-proxy", log.Fields{"error": err}) |
Girish Kumar | 4d3887d | 2019-11-22 14:22:05 +0000 | [diff] [blame] | 269 | time.Sleep(startupRetryInterval) |
khenaidoo | b324421 | 2019-08-27 14:32:27 -0400 | [diff] [blame] | 270 | } else { |
Scott Baker | ee6a087 | 2019-10-29 15:59:52 -0700 | [diff] [blame] | 271 | // We started. We only need to do this once. |
| 272 | // Next we'll fall through and start checking liveness. |
Girish Kumar | f56a468 | 2020-03-20 20:07:46 +0000 | [diff] [blame] | 273 | logger.Infow("started-kafka-proxy", log.Fields{}) |
Scott Baker | ee6a087 | 2019-10-29 15:59:52 -0700 | [diff] [blame] | 274 | |
| 275 | // cannot do this until after the kmp is started |
npujar | 1d86a52 | 2019-11-14 17:11:16 +0530 | [diff] [blame] | 276 | if err := core.registerAdapterRequestHandlers(ctx, core.instanceID, core.deviceMgr, core.logicalDeviceMgr, core.adapterMgr, core.clusterDataProxy, core.localDataProxy); err != nil { |
Girish Kumar | f56a468 | 2020-03-20 20:07:46 +0000 | [diff] [blame] | 277 | logger.Fatal("Failure-registering-adapterRequestHandler") |
Scott Baker | ee6a087 | 2019-10-29 15:59:52 -0700 | [diff] [blame] | 278 | } |
| 279 | |
| 280 | started = true |
khenaidoo | b324421 | 2019-08-27 14:32:27 -0400 | [diff] [blame] | 281 | } |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 282 | } |
Scott Baker | ee6a087 | 2019-10-29 15:59:52 -0700 | [diff] [blame] | 283 | |
Girish Kumar | f56a468 | 2020-03-20 20:07:46 +0000 | [diff] [blame] | 284 | logger.Info("started-kafka-message-proxy") |
Scott Baker | ee6a087 | 2019-10-29 15:59:52 -0700 | [diff] [blame] | 285 | |
| 286 | livenessChannel := core.kmp.EnableLivenessChannel(true) |
| 287 | |
Girish Kumar | f56a468 | 2020-03-20 20:07:46 +0000 | [diff] [blame] | 288 | logger.Info("enabled-kafka-liveness-channel") |
Scott Baker | ee6a087 | 2019-10-29 15:59:52 -0700 | [diff] [blame] | 289 | |
Girish Kumar | 4d3887d | 2019-11-22 14:22:05 +0000 | [diff] [blame] | 290 | timeout := liveProbeInterval |
Scott Baker | ee6a087 | 2019-10-29 15:59:52 -0700 | [diff] [blame] | 291 | for { |
| 292 | timeoutTimer := time.NewTimer(timeout) |
| 293 | select { |
| 294 | case liveness := <-livenessChannel: |
Girish Kumar | f56a468 | 2020-03-20 20:07:46 +0000 | [diff] [blame] | 295 | logger.Infow("kafka-manager-thread-liveness-event", log.Fields{"liveness": liveness}) |
Scott Baker | ee6a087 | 2019-10-29 15:59:52 -0700 | [diff] [blame] | 296 | // there was a state change in Kafka liveness |
| 297 | if !liveness { |
| 298 | probe.UpdateStatusFromContext(ctx, "message-bus", probe.ServiceStatusNotReady) |
| 299 | |
| 300 | if core.grpcServer != nil { |
Girish Kumar | f56a468 | 2020-03-20 20:07:46 +0000 | [diff] [blame] | 301 | logger.Info("kafka-manager-thread-set-server-notready") |
Scott Baker | ee6a087 | 2019-10-29 15:59:52 -0700 | [diff] [blame] | 302 | } |
| 303 | |
| 304 | // retry frequently while life is bad |
Girish Kumar | 4d3887d | 2019-11-22 14:22:05 +0000 | [diff] [blame] | 305 | timeout = notLiveProbeInterval |
Scott Baker | ee6a087 | 2019-10-29 15:59:52 -0700 | [diff] [blame] | 306 | } else { |
| 307 | probe.UpdateStatusFromContext(ctx, "message-bus", probe.ServiceStatusRunning) |
| 308 | |
| 309 | if core.grpcServer != nil { |
Girish Kumar | f56a468 | 2020-03-20 20:07:46 +0000 | [diff] [blame] | 310 | logger.Info("kafka-manager-thread-set-server-ready") |
Scott Baker | ee6a087 | 2019-10-29 15:59:52 -0700 | [diff] [blame] | 311 | } |
| 312 | |
| 313 | // retry infrequently while life is good |
Girish Kumar | 4d3887d | 2019-11-22 14:22:05 +0000 | [diff] [blame] | 314 | timeout = liveProbeInterval |
Scott Baker | ee6a087 | 2019-10-29 15:59:52 -0700 | [diff] [blame] | 315 | } |
| 316 | if !timeoutTimer.Stop() { |
| 317 | <-timeoutTimer.C |
| 318 | } |
| 319 | case <-timeoutTimer.C: |
Girish Kumar | f56a468 | 2020-03-20 20:07:46 +0000 | [diff] [blame] | 320 | logger.Info("kafka-proxy-liveness-recheck") |
Scott Baker | ee6a087 | 2019-10-29 15:59:52 -0700 | [diff] [blame] | 321 | // send the liveness probe in a goroutine; we don't want to deadlock ourselves as |
| 322 | // the liveness probe may wait (and block) writing to our channel. |
| 323 | go func() { |
| 324 | err := core.kmp.SendLiveness() |
| 325 | if err != nil { |
| 326 | // Catch possible error case if sending liveness after Sarama has been stopped. |
Girish Kumar | f56a468 | 2020-03-20 20:07:46 +0000 | [diff] [blame] | 327 | logger.Warnw("error-kafka-send-liveness", log.Fields{"error": err}) |
Scott Baker | ee6a087 | 2019-10-29 15:59:52 -0700 | [diff] [blame] | 328 | } |
| 329 | }() |
| 330 | } |
| 331 | } |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 332 | } |
| 333 | |
khenaidoo | b324421 | 2019-08-27 14:32:27 -0400 | [diff] [blame] | 334 | // waitUntilKVStoreReachableOrMaxTries will wait until it can connect to a KV store or until maxtries has been reached |
Girish Kumar | 4d3887d | 2019-11-22 14:22:05 +0000 | [diff] [blame] | 335 | func (core *Core) waitUntilKVStoreReachableOrMaxTries(ctx context.Context, maxRetries int, retryInterval time.Duration) error { |
Girish Kumar | f56a468 | 2020-03-20 20:07:46 +0000 | [diff] [blame] | 336 | logger.Infow("verifying-KV-store-connectivity", log.Fields{"host": core.config.KVStoreHost, |
khenaidoo | b324421 | 2019-08-27 14:32:27 -0400 | [diff] [blame] | 337 | "port": core.config.KVStorePort, "retries": maxRetries, "retryInterval": retryInterval}) |
khenaidoo | b324421 | 2019-08-27 14:32:27 -0400 | [diff] [blame] | 338 | count := 0 |
| 339 | for { |
npujar | 467fe75 | 2020-01-16 20:17:45 +0530 | [diff] [blame] | 340 | if !core.kvClient.IsConnectionUp(ctx) { |
Girish Kumar | f56a468 | 2020-03-20 20:07:46 +0000 | [diff] [blame] | 341 | logger.Info("KV-store-unreachable") |
khenaidoo | b324421 | 2019-08-27 14:32:27 -0400 | [diff] [blame] | 342 | if maxRetries != -1 { |
| 343 | if count >= maxRetries { |
| 344 | return status.Error(codes.Unavailable, "kv store unreachable") |
| 345 | } |
| 346 | } |
npujar | 1d86a52 | 2019-11-14 17:11:16 +0530 | [diff] [blame] | 347 | count++ |
khenaidoo | b324421 | 2019-08-27 14:32:27 -0400 | [diff] [blame] | 348 | // Take a nap before retrying |
Girish Kumar | 4d3887d | 2019-11-22 14:22:05 +0000 | [diff] [blame] | 349 | time.Sleep(retryInterval) |
Girish Kumar | f56a468 | 2020-03-20 20:07:46 +0000 | [diff] [blame] | 350 | logger.Infow("retry-KV-store-connectivity", log.Fields{"retryCount": count, "maxRetries": maxRetries, "retryInterval": retryInterval}) |
khenaidoo | b324421 | 2019-08-27 14:32:27 -0400 | [diff] [blame] | 351 | |
| 352 | } else { |
| 353 | break |
| 354 | } |
| 355 | } |
Girish Kumar | f56a468 | 2020-03-20 20:07:46 +0000 | [diff] [blame] | 356 | logger.Info("KV-store-reachable") |
khenaidoo | b324421 | 2019-08-27 14:32:27 -0400 | [diff] [blame] | 357 | return nil |
| 358 | } |
| 359 | |
Kent Hagerman | 2b21604 | 2020-04-03 18:28:56 -0400 | [diff] [blame] | 360 | func (core *Core) registerAdapterRequestHandlers(ctx context.Context, coreInstanceID string, dMgr *device.Manager, |
| 361 | ldMgr *device.LogicalManager, aMgr *adapter.Manager, cdProxy *model.Proxy, ldProxy *model.Proxy, |
khenaidoo | 54e0ddf | 2019-02-27 16:21:33 -0500 | [diff] [blame] | 362 | ) error { |
Kent Hagerman | 2b21604 | 2020-04-03 18:28:56 -0400 | [diff] [blame] | 363 | requestProxy := api.NewAdapterRequestHandlerProxy(coreInstanceID, dMgr, aMgr, cdProxy, ldProxy, |
David Bainbridge | d1afd66 | 2020-03-26 18:27:41 -0700 | [diff] [blame] | 364 | core.config.LongRunningRequestTimeout, core.config.DefaultRequestTimeout) |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 365 | |
khenaidoo | 54e0ddf | 2019-02-27 16:21:33 -0500 | [diff] [blame] | 366 | // Register the broadcast topic to handle any core-bound broadcast requests |
| 367 | if err := core.kmp.SubscribeWithRequestHandlerInterface(kafka.Topic{Name: core.config.CoreTopic}, requestProxy); err != nil { |
Girish Kumar | f56a468 | 2020-03-20 20:07:46 +0000 | [diff] [blame] | 368 | logger.Fatalw("Failed-registering-broadcast-handler", log.Fields{"topic": core.config.CoreTopic}) |
khenaidoo | 54e0ddf | 2019-02-27 16:21:33 -0500 | [diff] [blame] | 369 | return err |
| 370 | } |
| 371 | |
Kent Hagerman | a6d0c36 | 2019-07-30 12:50:21 -0400 | [diff] [blame] | 372 | // Register the core-pair topic to handle core-bound requests destined to the core pair |
| 373 | if err := core.kmp.SubscribeWithDefaultRequestHandler(kafka.Topic{Name: core.config.CorePairTopic}, kafka.OffsetNewest); err != nil { |
Girish Kumar | f56a468 | 2020-03-20 20:07:46 +0000 | [diff] [blame] | 374 | logger.Fatalw("Failed-registering-pair-handler", log.Fields{"topic": core.config.CorePairTopic}) |
Kent Hagerman | a6d0c36 | 2019-07-30 12:50:21 -0400 | [diff] [blame] | 375 | return err |
| 376 | } |
| 377 | |
Girish Kumar | f56a468 | 2020-03-20 20:07:46 +0000 | [diff] [blame] | 378 | logger.Info("request-handler-registered") |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 379 | return nil |
| 380 | } |
| 381 | |
| 382 | func (core *Core) startDeviceManager(ctx context.Context) { |
Girish Kumar | f56a468 | 2020-03-20 20:07:46 +0000 | [diff] [blame] | 383 | logger.Info("DeviceManager-Starting...") |
Kent Hagerman | 2b21604 | 2020-04-03 18:28:56 -0400 | [diff] [blame] | 384 | core.deviceMgr.Start(ctx) |
Girish Kumar | f56a468 | 2020-03-20 20:07:46 +0000 | [diff] [blame] | 385 | logger.Info("DeviceManager-Started") |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 386 | } |
| 387 | |
| 388 | func (core *Core) startLogicalDeviceManager(ctx context.Context) { |
Girish Kumar | f56a468 | 2020-03-20 20:07:46 +0000 | [diff] [blame] | 389 | logger.Info("Logical-DeviceManager-Starting...") |
Kent Hagerman | 2b21604 | 2020-04-03 18:28:56 -0400 | [diff] [blame] | 390 | core.logicalDeviceMgr.Start(ctx) |
Girish Kumar | f56a468 | 2020-03-20 20:07:46 +0000 | [diff] [blame] | 391 | logger.Info("Logical-DeviceManager-Started") |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 392 | } |
khenaidoo | 21d5115 | 2019-02-01 13:48:37 -0500 | [diff] [blame] | 393 | |
| 394 | func (core *Core) startAdapterManager(ctx context.Context) { |
Girish Kumar | f56a468 | 2020-03-20 20:07:46 +0000 | [diff] [blame] | 395 | logger.Info("Adapter-Manager-Starting...") |
Kent Hagerman | 2b21604 | 2020-04-03 18:28:56 -0400 | [diff] [blame] | 396 | err := core.adapterMgr.Start(ctx) |
Thomas Lee S | e5a4401 | 2019-11-07 20:32:24 +0530 | [diff] [blame] | 397 | if err != nil { |
Girish Kumar | f56a468 | 2020-03-20 20:07:46 +0000 | [diff] [blame] | 398 | logger.Fatalf("failed-to-start-adapter-manager: error %v ", err) |
Thomas Lee S | e5a4401 | 2019-11-07 20:32:24 +0530 | [diff] [blame] | 399 | } |
Girish Kumar | f56a468 | 2020-03-20 20:07:46 +0000 | [diff] [blame] | 400 | logger.Info("Adapter-Manager-Started") |
William Kurkian | daa6bb2 | 2019-03-07 12:26:28 -0500 | [diff] [blame] | 401 | } |
Girish Kumar | 4d3887d | 2019-11-22 14:22:05 +0000 | [diff] [blame] | 402 | |
| 403 | /* |
| 404 | * Thread to monitor kvstore Liveness (connection status) |
| 405 | * |
| 406 | * This function constantly monitors Liveness State of kvstore as reported |
| 407 | * periodically by backend and updates the Status of kv-store service registered |
| 408 | * with rw_core probe. |
| 409 | * |
| 410 | * If no liveness event has been seen within a timeout, then the thread will |
| 411 | * perform a "liveness" check attempt, which will in turn trigger a liveness event on |
| 412 | * the liveness channel, true or false depending on whether the attempt succeeded. |
| 413 | * |
| 414 | * The gRPC server in turn monitors the state of the readiness probe and will |
| 415 | * start issuing UNAVAILABLE response while the probe is not ready. |
| 416 | */ |
| 417 | func (core *Core) monitorKvstoreLiveness(ctx context.Context) { |
Girish Kumar | f56a468 | 2020-03-20 20:07:46 +0000 | [diff] [blame] | 418 | logger.Info("start-monitoring-kvstore-liveness") |
Girish Kumar | 4d3887d | 2019-11-22 14:22:05 +0000 | [diff] [blame] | 419 | |
| 420 | // Instruct backend to create Liveness channel for transporting state updates |
| 421 | livenessChannel := core.backend.EnableLivenessChannel() |
| 422 | |
Girish Kumar | f56a468 | 2020-03-20 20:07:46 +0000 | [diff] [blame] | 423 | logger.Debug("enabled-kvstore-liveness-channel") |
Girish Kumar | 4d3887d | 2019-11-22 14:22:05 +0000 | [diff] [blame] | 424 | |
| 425 | // Default state for kvstore is alive for rw_core |
| 426 | timeout := core.config.LiveProbeInterval |
Scott Baker | 2d87ee3 | 2020-03-03 13:04:01 -0800 | [diff] [blame] | 427 | loop: |
Girish Kumar | 4d3887d | 2019-11-22 14:22:05 +0000 | [diff] [blame] | 428 | for { |
| 429 | timeoutTimer := time.NewTimer(timeout) |
| 430 | select { |
| 431 | |
| 432 | case liveness := <-livenessChannel: |
Girish Kumar | f56a468 | 2020-03-20 20:07:46 +0000 | [diff] [blame] | 433 | logger.Debugw("received-liveness-change-notification", log.Fields{"liveness": liveness}) |
Girish Kumar | 4d3887d | 2019-11-22 14:22:05 +0000 | [diff] [blame] | 434 | |
| 435 | if !liveness { |
| 436 | probe.UpdateStatusFromContext(ctx, "kv-store", probe.ServiceStatusNotReady) |
| 437 | |
| 438 | if core.grpcServer != nil { |
Girish Kumar | f56a468 | 2020-03-20 20:07:46 +0000 | [diff] [blame] | 439 | logger.Info("kvstore-set-server-notready") |
Girish Kumar | 4d3887d | 2019-11-22 14:22:05 +0000 | [diff] [blame] | 440 | } |
| 441 | |
| 442 | timeout = core.config.NotLiveProbeInterval |
| 443 | |
| 444 | } else { |
| 445 | probe.UpdateStatusFromContext(ctx, "kv-store", probe.ServiceStatusRunning) |
| 446 | |
| 447 | if core.grpcServer != nil { |
Girish Kumar | f56a468 | 2020-03-20 20:07:46 +0000 | [diff] [blame] | 448 | logger.Info("kvstore-set-server-ready") |
Girish Kumar | 4d3887d | 2019-11-22 14:22:05 +0000 | [diff] [blame] | 449 | } |
| 450 | |
| 451 | timeout = core.config.LiveProbeInterval |
| 452 | } |
| 453 | |
| 454 | if !timeoutTimer.Stop() { |
| 455 | <-timeoutTimer.C |
| 456 | } |
| 457 | |
Scott Baker | 2d87ee3 | 2020-03-03 13:04:01 -0800 | [diff] [blame] | 458 | case <-core.exitChannel: |
| 459 | break loop |
| 460 | |
Girish Kumar | 4d3887d | 2019-11-22 14:22:05 +0000 | [diff] [blame] | 461 | case <-timeoutTimer.C: |
Girish Kumar | f56a468 | 2020-03-20 20:07:46 +0000 | [diff] [blame] | 462 | logger.Info("kvstore-perform-liveness-check-on-timeout") |
Girish Kumar | 4d3887d | 2019-11-22 14:22:05 +0000 | [diff] [blame] | 463 | |
| 464 | // Trigger Liveness check if no liveness update received within the timeout period. |
| 465 | // The Liveness check will push Live state to same channel which this routine is |
| 466 | // reading and processing. This, do it asynchronously to avoid blocking for |
| 467 | // backend response and avoid any possibility of deadlock |
npujar | 467fe75 | 2020-01-16 20:17:45 +0530 | [diff] [blame] | 468 | go core.backend.PerformLivenessCheck(ctx) |
Girish Kumar | 4d3887d | 2019-11-22 14:22:05 +0000 | [diff] [blame] | 469 | } |
| 470 | } |
| 471 | } |