blob: 9e97c86c58cd84b38676a1b8f9af4675badd8942 [file] [log] [blame]
Kent Hagerman2f0d0552020-04-23 17:28:52 -04001/*
Joey Armstrong5f51f2e2023-01-17 17:06:26 -05002 * Copyright 2018-2023 Open Networking Foundation (ONF) and the ONF Contributors
Kent Hagerman2f0d0552020-04-23 17:28:52 -04003
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7
8 * http://www.apache.org/licenses/LICENSE-2.0
9
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17package core
18
19import (
20 "context"
21 "errors"
22 "time"
23
khenaidood948f772021-08-11 17:49:24 -040024 "github.com/opencord/voltha-lib-go/v7/pkg/db"
25 "github.com/opencord/voltha-lib-go/v7/pkg/db/kvstore"
26 "github.com/opencord/voltha-lib-go/v7/pkg/log"
27 "github.com/opencord/voltha-lib-go/v7/pkg/probe"
Kent Hagerman2f0d0552020-04-23 17:28:52 -040028 "google.golang.org/grpc/codes"
29 "google.golang.org/grpc/status"
30)
31
Rohan Agrawal31f21802020-06-12 05:38:46 +000032func newKVClient(ctx context.Context, storeType string, address string, timeout time.Duration) (kvstore.Client, error) {
33 logger.Infow(ctx, "kv-store-type", log.Fields{"store": storeType})
Andrey Pozolotin34dd63f2021-05-31 21:26:40 +030034 if storeType == "etcd" {
Rohan Agrawal31f21802020-06-12 05:38:46 +000035 return kvstore.NewEtcdClient(ctx, address, timeout, log.FatalLevel)
Kent Hagerman2f0d0552020-04-23 17:28:52 -040036 }
37 return nil, errors.New("unsupported-kv-store")
38}
39
40func stopKVClient(ctx context.Context, kvClient kvstore.Client) {
41 // Release all reservations
42 if err := kvClient.ReleaseAllReservations(ctx); err != nil {
Rohan Agrawal31f21802020-06-12 05:38:46 +000043 logger.Infow(ctx, "fail-to-release-all-reservations", log.Fields{"error": err})
Kent Hagerman2f0d0552020-04-23 17:28:52 -040044 }
45 // Close the DB connection
Rohan Agrawal31f21802020-06-12 05:38:46 +000046 kvClient.Close(ctx)
Kent Hagerman2f0d0552020-04-23 17:28:52 -040047}
48
49// waitUntilKVStoreReachableOrMaxTries will wait until it can connect to a KV store or until maxtries has been reached
khenaidood948f772021-08-11 17:49:24 -040050func waitUntilKVStoreReachableOrMaxTries(ctx context.Context, kvClient kvstore.Client, maxRetries int, retryInterval time.Duration, serviceName string) error {
Rohan Agrawal31f21802020-06-12 05:38:46 +000051 logger.Infow(ctx, "verifying-KV-store-connectivity", log.Fields{"retries": maxRetries, "retryInterval": retryInterval})
Kent Hagerman2f0d0552020-04-23 17:28:52 -040052 count := 0
53 for {
54 if !kvClient.IsConnectionUp(ctx) {
Rohan Agrawal31f21802020-06-12 05:38:46 +000055 logger.Info(ctx, "KV-store-unreachable")
Kent Hagerman2f0d0552020-04-23 17:28:52 -040056 if maxRetries != -1 {
57 if count >= maxRetries {
58 return status.Error(codes.Unavailable, "kv store unreachable")
59 }
60 }
61 count++
62
63 // Take a nap before retrying
64 select {
65 case <-ctx.Done():
66 //ctx canceled
67 return ctx.Err()
68 case <-time.After(retryInterval):
69 }
Rohan Agrawal31f21802020-06-12 05:38:46 +000070 logger.Infow(ctx, "retry-KV-store-connectivity", log.Fields{"retryCount": count, "maxRetries": maxRetries, "retryInterval": retryInterval})
Kent Hagerman2f0d0552020-04-23 17:28:52 -040071 } else {
72 break
73 }
74 }
khenaidood948f772021-08-11 17:49:24 -040075 probe.UpdateStatusFromContext(ctx, serviceName, probe.ServiceStatusRunning)
Rohan Agrawal31f21802020-06-12 05:38:46 +000076 logger.Info(ctx, "KV-store-reachable")
Kent Hagerman2f0d0552020-04-23 17:28:52 -040077 return nil
78}
79
80/*
81 * Thread to monitor kvstore Liveness (connection status)
82 *
83 * This function constantly monitors Liveness State of kvstore as reported
84 * periodically by backend and updates the Status of kv-store service registered
85 * with rw_core probe.
86 *
87 * If no liveness event has been seen within a timeout, then the thread will
88 * perform a "liveness" check attempt, which will in turn trigger a liveness event on
89 * the liveness channel, true or false depending on whether the attempt succeeded.
90 *
91 * The gRPC server in turn monitors the state of the readiness probe and will
92 * start issuing UNAVAILABLE response while the probe is not ready.
93 */
khenaidood948f772021-08-11 17:49:24 -040094func monitorKVStoreLiveness(ctx context.Context, backend *db.Backend, serviceName string, liveProbeInterval, notLiveProbeInterval time.Duration) {
Rohan Agrawal31f21802020-06-12 05:38:46 +000095 logger.Info(ctx, "start-monitoring-kvstore-liveness")
Kent Hagerman2f0d0552020-04-23 17:28:52 -040096
97 // Instruct backend to create Liveness channel for transporting state updates
Rohan Agrawal31f21802020-06-12 05:38:46 +000098 livenessChannel := backend.EnableLivenessChannel(ctx)
Kent Hagerman2f0d0552020-04-23 17:28:52 -040099
khenaidood948f772021-08-11 17:49:24 -0400100 logger.Infow(ctx, "enabled-liveness-channel", log.Fields{"service-name": serviceName})
Kent Hagerman2f0d0552020-04-23 17:28:52 -0400101
102 // Default state for kvstore is alive for rw_core
103 timeout := liveProbeInterval
104loop:
105 for {
106 timeoutTimer := time.NewTimer(timeout)
107 select {
108
109 case liveness := <-livenessChannel:
khenaidood948f772021-08-11 17:49:24 -0400110 logger.Debugw(ctx, "received-liveness-change-notification", log.Fields{"liveness": liveness, "service-name": serviceName})
Kent Hagerman2f0d0552020-04-23 17:28:52 -0400111
112 if !liveness {
khenaidood948f772021-08-11 17:49:24 -0400113 probe.UpdateStatusFromContext(ctx, serviceName, probe.ServiceStatusNotReady)
114 logger.Infow(ctx, "service-not-ready", log.Fields{"service-name": serviceName})
Kent Hagerman2f0d0552020-04-23 17:28:52 -0400115
116 timeout = notLiveProbeInterval
117
118 } else {
khenaidood948f772021-08-11 17:49:24 -0400119 probe.UpdateStatusFromContext(ctx, serviceName, probe.ServiceStatusRunning)
120 logger.Infow(ctx, "service-ready", log.Fields{"service-name": serviceName})
Kent Hagerman2f0d0552020-04-23 17:28:52 -0400121
122 timeout = liveProbeInterval
123 }
124
125 if !timeoutTimer.Stop() {
126 <-timeoutTimer.C
127 }
128
129 case <-ctx.Done():
130 break loop
131
132 case <-timeoutTimer.C:
khenaidood948f772021-08-11 17:49:24 -0400133 logger.Infow(ctx, "perform-liveness-check-on-timeout", log.Fields{"service-name": serviceName})
Kent Hagerman2f0d0552020-04-23 17:28:52 -0400134
135 // Trigger Liveness check if no liveness update received within the timeout period.
136 // The Liveness check will push Live state to same channel which this routine is
137 // reading and processing. This, do it asynchronously to avoid blocking for
138 // backend response and avoid any possibility of deadlock
139 go backend.PerformLivenessCheck(ctx)
140 }
141 }
142}