divyadesai | 81bb7ba | 2020-03-11 11:45:23 +0000 | [diff] [blame] | 1 | /* |
Mahir Gunyel | 82f98dd | 2023-07-24 10:29:49 +0300 | [diff] [blame] | 2 | * Copyright 2018-2023 Open Networking Foundation (ONF) and the ONF Contributors |
divyadesai | 81bb7ba | 2020-03-11 11:45:23 +0000 | [diff] [blame] | 3 | |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
| 17 | package db |
| 18 | |
| 19 | import ( |
| 20 | "context" |
| 21 | "errors" |
| 22 | "fmt" |
Girish Kumar | cd40201 | 2020-08-18 12:17:38 +0000 | [diff] [blame] | 23 | "sync" |
divyadesai | 81bb7ba | 2020-03-11 11:45:23 +0000 | [diff] [blame] | 24 | "time" |
| 25 | |
David K. Bainbridge | e05cf0c | 2021-08-19 03:16:50 +0000 | [diff] [blame] | 26 | "github.com/opencord/voltha-lib-go/v7/pkg/db/kvstore" |
| 27 | "github.com/opencord/voltha-lib-go/v7/pkg/log" |
divyadesai | 81bb7ba | 2020-03-11 11:45:23 +0000 | [diff] [blame] | 28 | "google.golang.org/grpc/codes" |
| 29 | "google.golang.org/grpc/status" |
| 30 | ) |
| 31 | |
| 32 | const ( |
| 33 | // Default Minimal Interval for posting alive state of backend kvstore on Liveness Channel |
| 34 | DefaultLivenessChannelInterval = time.Second * 30 |
| 35 | ) |
| 36 | |
| 37 | // Backend structure holds details for accessing the kv store |
| 38 | type Backend struct { |
divyadesai | 81bb7ba | 2020-03-11 11:45:23 +0000 | [diff] [blame] | 39 | Client kvstore.Client |
| 40 | StoreType string |
Neha Sharma | 87d43d7 | 2020-04-08 14:10:40 +0000 | [diff] [blame] | 41 | Timeout time.Duration |
Neha Sharma | 318a129 | 2020-05-14 19:53:26 +0000 | [diff] [blame] | 42 | Address string |
divyadesai | 81bb7ba | 2020-03-11 11:45:23 +0000 | [diff] [blame] | 43 | PathPrefix string |
Girish Kumar | cd40201 | 2020-08-18 12:17:38 +0000 | [diff] [blame] | 44 | alive bool // Is this backend connection alive? |
| 45 | livenessMutex sync.Mutex |
divyadesai | 81bb7ba | 2020-03-11 11:45:23 +0000 | [diff] [blame] | 46 | liveness chan bool // channel to post alive state |
| 47 | LivenessChannelInterval time.Duration // regularly push alive state beyond this interval |
| 48 | lastLivenessTime time.Time // Instant of last alive state push |
| 49 | } |
| 50 | |
| 51 | // NewBackend creates a new instance of a Backend structure |
Rohan Agrawal | c32d993 | 2020-06-15 11:01:47 +0000 | [diff] [blame] | 52 | func NewBackend(ctx context.Context, storeType string, address string, timeout time.Duration, pathPrefix string) *Backend { |
divyadesai | 81bb7ba | 2020-03-11 11:45:23 +0000 | [diff] [blame] | 53 | var err error |
| 54 | |
| 55 | b := &Backend{ |
| 56 | StoreType: storeType, |
Neha Sharma | 318a129 | 2020-05-14 19:53:26 +0000 | [diff] [blame] | 57 | Address: address, |
divyadesai | 81bb7ba | 2020-03-11 11:45:23 +0000 | [diff] [blame] | 58 | Timeout: timeout, |
| 59 | LivenessChannelInterval: DefaultLivenessChannelInterval, |
| 60 | PathPrefix: pathPrefix, |
| 61 | alive: false, // connection considered down at start |
| 62 | } |
| 63 | |
Rohan Agrawal | c32d993 | 2020-06-15 11:01:47 +0000 | [diff] [blame] | 64 | if b.Client, err = b.newClient(ctx, address, timeout); err != nil { |
| 65 | logger.Errorw(ctx, "failed-to-create-kv-client", |
divyadesai | 81bb7ba | 2020-03-11 11:45:23 +0000 | [diff] [blame] | 66 | log.Fields{ |
Neha Sharma | 318a129 | 2020-05-14 19:53:26 +0000 | [diff] [blame] | 67 | "type": storeType, "address": address, |
divyadesai | 81bb7ba | 2020-03-11 11:45:23 +0000 | [diff] [blame] | 68 | "timeout": timeout, "prefix": pathPrefix, |
| 69 | "error": err.Error(), |
| 70 | }) |
| 71 | } |
| 72 | |
| 73 | return b |
| 74 | } |
| 75 | |
Rohan Agrawal | c32d993 | 2020-06-15 11:01:47 +0000 | [diff] [blame] | 76 | func (b *Backend) newClient(ctx context.Context, address string, timeout time.Duration) (kvstore.Client, error) { |
divyadesai | 81bb7ba | 2020-03-11 11:45:23 +0000 | [diff] [blame] | 77 | switch b.StoreType { |
Mahir Gunyel | 82f98dd | 2023-07-24 10:29:49 +0300 | [diff] [blame] | 78 | case "redis": |
| 79 | return kvstore.NewRedisClient(address, timeout, false) |
| 80 | case "redis-sentinel": |
| 81 | return kvstore.NewRedisClient(address, timeout, true) |
divyadesai | 81bb7ba | 2020-03-11 11:45:23 +0000 | [diff] [blame] | 82 | case "etcd": |
Rohan Agrawal | c32d993 | 2020-06-15 11:01:47 +0000 | [diff] [blame] | 83 | return kvstore.NewEtcdClient(ctx, address, timeout, log.WarnLevel) |
divyadesai | 81bb7ba | 2020-03-11 11:45:23 +0000 | [diff] [blame] | 84 | } |
| 85 | return nil, errors.New("unsupported-kv-store") |
| 86 | } |
| 87 | |
Rohan Agrawal | c32d993 | 2020-06-15 11:01:47 +0000 | [diff] [blame] | 88 | func (b *Backend) makePath(ctx context.Context, key string) string { |
divyadesai | 81bb7ba | 2020-03-11 11:45:23 +0000 | [diff] [blame] | 89 | path := fmt.Sprintf("%s/%s", b.PathPrefix, key) |
| 90 | return path |
| 91 | } |
| 92 | |
Rohan Agrawal | c32d993 | 2020-06-15 11:01:47 +0000 | [diff] [blame] | 93 | func (b *Backend) updateLiveness(ctx context.Context, alive bool) { |
divyadesai | 81bb7ba | 2020-03-11 11:45:23 +0000 | [diff] [blame] | 94 | // Periodically push stream of liveness data to the channel, |
| 95 | // so that in a live state, the core does not timeout and |
| 96 | // send a forced liveness message. Push alive state if the |
| 97 | // last push to channel was beyond livenessChannelInterval |
Girish Kumar | cd40201 | 2020-08-18 12:17:38 +0000 | [diff] [blame] | 98 | b.livenessMutex.Lock() |
| 99 | defer b.livenessMutex.Unlock() |
divyadesai | 81bb7ba | 2020-03-11 11:45:23 +0000 | [diff] [blame] | 100 | if b.liveness != nil { |
divyadesai | 81bb7ba | 2020-03-11 11:45:23 +0000 | [diff] [blame] | 101 | if b.alive != alive { |
Rohan Agrawal | c32d993 | 2020-06-15 11:01:47 +0000 | [diff] [blame] | 102 | logger.Debug(ctx, "update-liveness-channel-reason-change") |
divyadesai | 81bb7ba | 2020-03-11 11:45:23 +0000 | [diff] [blame] | 103 | b.liveness <- alive |
| 104 | b.lastLivenessTime = time.Now() |
| 105 | } else if time.Since(b.lastLivenessTime) > b.LivenessChannelInterval { |
Rohan Agrawal | c32d993 | 2020-06-15 11:01:47 +0000 | [diff] [blame] | 106 | logger.Debug(ctx, "update-liveness-channel-reason-interval") |
divyadesai | 81bb7ba | 2020-03-11 11:45:23 +0000 | [diff] [blame] | 107 | b.liveness <- alive |
| 108 | b.lastLivenessTime = time.Now() |
| 109 | } |
| 110 | } |
| 111 | |
| 112 | // Emit log message only for alive state change |
| 113 | if b.alive != alive { |
Rohan Agrawal | c32d993 | 2020-06-15 11:01:47 +0000 | [diff] [blame] | 114 | logger.Debugw(ctx, "change-kvstore-alive-status", log.Fields{"alive": alive}) |
divyadesai | 81bb7ba | 2020-03-11 11:45:23 +0000 | [diff] [blame] | 115 | b.alive = alive |
| 116 | } |
| 117 | } |
| 118 | |
| 119 | // Perform a dummy Key Lookup on kvstore to test Connection Liveness and |
| 120 | // post on Liveness channel |
| 121 | func (b *Backend) PerformLivenessCheck(ctx context.Context) bool { |
| 122 | alive := b.Client.IsConnectionUp(ctx) |
Rohan Agrawal | c32d993 | 2020-06-15 11:01:47 +0000 | [diff] [blame] | 123 | logger.Debugw(ctx, "kvstore-liveness-check-result", log.Fields{"alive": alive}) |
divyadesai | 81bb7ba | 2020-03-11 11:45:23 +0000 | [diff] [blame] | 124 | |
Rohan Agrawal | c32d993 | 2020-06-15 11:01:47 +0000 | [diff] [blame] | 125 | b.updateLiveness(ctx, alive) |
divyadesai | 81bb7ba | 2020-03-11 11:45:23 +0000 | [diff] [blame] | 126 | return alive |
| 127 | } |
| 128 | |
| 129 | // Enable the liveness monitor channel. This channel will report |
| 130 | // a "true" or "false" on every kvstore operation which indicates whether |
| 131 | // or not the connection is still Live. This channel is then picked up |
| 132 | // by the service (i.e. rw_core / ro_core) to update readiness status |
| 133 | // and/or take other actions. |
Rohan Agrawal | c32d993 | 2020-06-15 11:01:47 +0000 | [diff] [blame] | 134 | func (b *Backend) EnableLivenessChannel(ctx context.Context) chan bool { |
| 135 | logger.Debug(ctx, "enable-kvstore-liveness-channel") |
Girish Kumar | cd40201 | 2020-08-18 12:17:38 +0000 | [diff] [blame] | 136 | b.livenessMutex.Lock() |
| 137 | defer b.livenessMutex.Unlock() |
divyadesai | 81bb7ba | 2020-03-11 11:45:23 +0000 | [diff] [blame] | 138 | if b.liveness == nil { |
divyadesai | 81bb7ba | 2020-03-11 11:45:23 +0000 | [diff] [blame] | 139 | b.liveness = make(chan bool, 10) |
divyadesai | 81bb7ba | 2020-03-11 11:45:23 +0000 | [diff] [blame] | 140 | b.liveness <- b.alive |
| 141 | b.lastLivenessTime = time.Now() |
| 142 | } |
| 143 | |
| 144 | return b.liveness |
| 145 | } |
| 146 | |
| 147 | // Extract Alive status of Kvstore based on type of error |
Rohan Agrawal | c32d993 | 2020-06-15 11:01:47 +0000 | [diff] [blame] | 148 | func (b *Backend) isErrorIndicatingAliveKvstore(ctx context.Context, err error) bool { |
divyadesai | 81bb7ba | 2020-03-11 11:45:23 +0000 | [diff] [blame] | 149 | // Alive unless observed an error indicating so |
| 150 | alive := true |
| 151 | |
| 152 | if err != nil { |
| 153 | |
| 154 | // timeout indicates kvstore not reachable/alive |
| 155 | if err == context.DeadlineExceeded { |
| 156 | alive = false |
| 157 | } |
| 158 | |
| 159 | // Need to analyze client-specific errors based on backend type |
| 160 | if b.StoreType == "etcd" { |
| 161 | |
| 162 | // For etcd backend, consider not-alive only for errors indicating |
| 163 | // timedout request or unavailable/corrupted cluster. For all remaining |
| 164 | // error codes listed in https://godoc.org/google.golang.org/grpc/codes#Code, |
| 165 | // we would not infer a not-alive backend because such a error may also |
| 166 | // occur due to bad client requests or sequence of operations |
| 167 | switch status.Code(err) { |
| 168 | case codes.DeadlineExceeded: |
| 169 | fallthrough |
| 170 | case codes.Unavailable: |
| 171 | fallthrough |
| 172 | case codes.DataLoss: |
| 173 | alive = false |
| 174 | } |
divyadesai | 81bb7ba | 2020-03-11 11:45:23 +0000 | [diff] [blame] | 175 | } |
| 176 | } |
| 177 | |
| 178 | return alive |
| 179 | } |
| 180 | |
| 181 | // List retrieves one or more items that match the specified key |
| 182 | func (b *Backend) List(ctx context.Context, key string) (map[string]*kvstore.KVPair, error) { |
Mahir Gunyel | 82f98dd | 2023-07-24 10:29:49 +0300 | [diff] [blame] | 183 | span, ctx := log.CreateChildSpan(ctx, "kvs-list") |
Girish Kumar | cd40201 | 2020-08-18 12:17:38 +0000 | [diff] [blame] | 184 | defer span.Finish() |
| 185 | |
Rohan Agrawal | c32d993 | 2020-06-15 11:01:47 +0000 | [diff] [blame] | 186 | formattedPath := b.makePath(ctx, key) |
| 187 | logger.Debugw(ctx, "listing-key", log.Fields{"key": key, "path": formattedPath}) |
divyadesai | 81bb7ba | 2020-03-11 11:45:23 +0000 | [diff] [blame] | 188 | |
| 189 | pair, err := b.Client.List(ctx, formattedPath) |
| 190 | |
Rohan Agrawal | c32d993 | 2020-06-15 11:01:47 +0000 | [diff] [blame] | 191 | b.updateLiveness(ctx, b.isErrorIndicatingAliveKvstore(ctx, err)) |
divyadesai | 81bb7ba | 2020-03-11 11:45:23 +0000 | [diff] [blame] | 192 | |
| 193 | return pair, err |
| 194 | } |
| 195 | |
| 196 | // Get retrieves an item that matches the specified key |
| 197 | func (b *Backend) Get(ctx context.Context, key string) (*kvstore.KVPair, error) { |
Mahir Gunyel | 82f98dd | 2023-07-24 10:29:49 +0300 | [diff] [blame] | 198 | span, ctx := log.CreateChildSpan(ctx, "kvs-get") |
Girish Kumar | cd40201 | 2020-08-18 12:17:38 +0000 | [diff] [blame] | 199 | defer span.Finish() |
| 200 | |
Rohan Agrawal | c32d993 | 2020-06-15 11:01:47 +0000 | [diff] [blame] | 201 | formattedPath := b.makePath(ctx, key) |
| 202 | logger.Debugw(ctx, "getting-key", log.Fields{"key": key, "path": formattedPath}) |
divyadesai | 81bb7ba | 2020-03-11 11:45:23 +0000 | [diff] [blame] | 203 | |
| 204 | pair, err := b.Client.Get(ctx, formattedPath) |
| 205 | |
Rohan Agrawal | c32d993 | 2020-06-15 11:01:47 +0000 | [diff] [blame] | 206 | b.updateLiveness(ctx, b.isErrorIndicatingAliveKvstore(ctx, err)) |
divyadesai | 81bb7ba | 2020-03-11 11:45:23 +0000 | [diff] [blame] | 207 | |
| 208 | return pair, err |
| 209 | } |
| 210 | |
| 211 | // Put stores an item value under the specifed key |
| 212 | func (b *Backend) Put(ctx context.Context, key string, value interface{}) error { |
Mahir Gunyel | 82f98dd | 2023-07-24 10:29:49 +0300 | [diff] [blame] | 213 | span, ctx := log.CreateChildSpan(ctx, "kvs-put") |
Girish Kumar | cd40201 | 2020-08-18 12:17:38 +0000 | [diff] [blame] | 214 | defer span.Finish() |
| 215 | |
Rohan Agrawal | c32d993 | 2020-06-15 11:01:47 +0000 | [diff] [blame] | 216 | formattedPath := b.makePath(ctx, key) |
| 217 | logger.Debugw(ctx, "putting-key", log.Fields{"key": key, "path": formattedPath}) |
divyadesai | 81bb7ba | 2020-03-11 11:45:23 +0000 | [diff] [blame] | 218 | |
| 219 | err := b.Client.Put(ctx, formattedPath, value) |
| 220 | |
Rohan Agrawal | c32d993 | 2020-06-15 11:01:47 +0000 | [diff] [blame] | 221 | b.updateLiveness(ctx, b.isErrorIndicatingAliveKvstore(ctx, err)) |
divyadesai | 81bb7ba | 2020-03-11 11:45:23 +0000 | [diff] [blame] | 222 | |
| 223 | return err |
| 224 | } |
| 225 | |
| 226 | // Delete removes an item under the specified key |
| 227 | func (b *Backend) Delete(ctx context.Context, key string) error { |
Mahir Gunyel | 82f98dd | 2023-07-24 10:29:49 +0300 | [diff] [blame] | 228 | span, ctx := log.CreateChildSpan(ctx, "kvs-delete") |
Girish Kumar | cd40201 | 2020-08-18 12:17:38 +0000 | [diff] [blame] | 229 | defer span.Finish() |
| 230 | |
Rohan Agrawal | c32d993 | 2020-06-15 11:01:47 +0000 | [diff] [blame] | 231 | formattedPath := b.makePath(ctx, key) |
| 232 | logger.Debugw(ctx, "deleting-key", log.Fields{"key": key, "path": formattedPath}) |
divyadesai | 81bb7ba | 2020-03-11 11:45:23 +0000 | [diff] [blame] | 233 | |
| 234 | err := b.Client.Delete(ctx, formattedPath) |
| 235 | |
Rohan Agrawal | c32d993 | 2020-06-15 11:01:47 +0000 | [diff] [blame] | 236 | b.updateLiveness(ctx, b.isErrorIndicatingAliveKvstore(ctx, err)) |
divyadesai | 81bb7ba | 2020-03-11 11:45:23 +0000 | [diff] [blame] | 237 | |
| 238 | return err |
| 239 | } |
| 240 | |
David K. Bainbridge | 595b670 | 2021-04-09 16:10:58 +0000 | [diff] [blame] | 241 | func (b *Backend) DeleteWithPrefix(ctx context.Context, prefixKey string) error { |
Mahir Gunyel | 82f98dd | 2023-07-24 10:29:49 +0300 | [diff] [blame] | 242 | span, ctx := log.CreateChildSpan(ctx, "kvs-delete-with-prefix") |
David K. Bainbridge | 595b670 | 2021-04-09 16:10:58 +0000 | [diff] [blame] | 243 | defer span.Finish() |
| 244 | |
| 245 | formattedPath := b.makePath(ctx, prefixKey) |
| 246 | logger.Debugw(ctx, "deleting-prefix-key", log.Fields{"key": prefixKey, "path": formattedPath}) |
| 247 | |
| 248 | err := b.Client.DeleteWithPrefix(ctx, formattedPath) |
| 249 | |
| 250 | b.updateLiveness(ctx, b.isErrorIndicatingAliveKvstore(ctx, err)) |
| 251 | |
| 252 | return err |
| 253 | } |
| 254 | |
divyadesai | 81bb7ba | 2020-03-11 11:45:23 +0000 | [diff] [blame] | 255 | // CreateWatch starts watching events for the specified key |
| 256 | func (b *Backend) CreateWatch(ctx context.Context, key string, withPrefix bool) chan *kvstore.Event { |
Mahir Gunyel | 82f98dd | 2023-07-24 10:29:49 +0300 | [diff] [blame] | 257 | span, ctx := log.CreateChildSpan(ctx, "kvs-create-watch") |
Girish Kumar | cd40201 | 2020-08-18 12:17:38 +0000 | [diff] [blame] | 258 | defer span.Finish() |
| 259 | |
Rohan Agrawal | c32d993 | 2020-06-15 11:01:47 +0000 | [diff] [blame] | 260 | formattedPath := b.makePath(ctx, key) |
| 261 | logger.Debugw(ctx, "creating-key-watch", log.Fields{"key": key, "path": formattedPath}) |
divyadesai | 81bb7ba | 2020-03-11 11:45:23 +0000 | [diff] [blame] | 262 | |
| 263 | return b.Client.Watch(ctx, formattedPath, withPrefix) |
| 264 | } |
| 265 | |
| 266 | // DeleteWatch stops watching events for the specified key |
Rohan Agrawal | c32d993 | 2020-06-15 11:01:47 +0000 | [diff] [blame] | 267 | func (b *Backend) DeleteWatch(ctx context.Context, key string, ch chan *kvstore.Event) { |
Mahir Gunyel | 82f98dd | 2023-07-24 10:29:49 +0300 | [diff] [blame] | 268 | span, ctx := log.CreateChildSpan(ctx, "kvs-delete-watch") |
Girish Kumar | cd40201 | 2020-08-18 12:17:38 +0000 | [diff] [blame] | 269 | defer span.Finish() |
| 270 | |
Rohan Agrawal | c32d993 | 2020-06-15 11:01:47 +0000 | [diff] [blame] | 271 | formattedPath := b.makePath(ctx, key) |
| 272 | logger.Debugw(ctx, "deleting-key-watch", log.Fields{"key": key, "path": formattedPath}) |
divyadesai | 81bb7ba | 2020-03-11 11:45:23 +0000 | [diff] [blame] | 273 | |
Rohan Agrawal | c32d993 | 2020-06-15 11:01:47 +0000 | [diff] [blame] | 274 | b.Client.CloseWatch(ctx, formattedPath, ch) |
divyadesai | 81bb7ba | 2020-03-11 11:45:23 +0000 | [diff] [blame] | 275 | } |