Matteo Scandolo | 2e6f1e3 | 2020-04-15 11:28:45 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2018-present Open Networking Foundation |
| 3 | |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
| 17 | package db |
| 18 | |
| 19 | import ( |
| 20 | "context" |
| 21 | "errors" |
| 22 | "fmt" |
dbainbri | 4d3a0dc | 2020-12-02 00:33:42 +0000 | [diff] [blame] | 23 | "sync" |
Matteo Scandolo | 2e6f1e3 | 2020-04-15 11:28:45 -0700 | [diff] [blame] | 24 | "time" |
| 25 | |
dbainbri | 4d3a0dc | 2020-12-02 00:33:42 +0000 | [diff] [blame] | 26 | "github.com/opencord/voltha-lib-go/v4/pkg/db/kvstore" |
| 27 | "github.com/opencord/voltha-lib-go/v4/pkg/log" |
Matteo Scandolo | 2e6f1e3 | 2020-04-15 11:28:45 -0700 | [diff] [blame] | 28 | "google.golang.org/grpc/codes" |
| 29 | "google.golang.org/grpc/status" |
| 30 | ) |
| 31 | |
| 32 | const ( |
| 33 | // Default Minimal Interval for posting alive state of backend kvstore on Liveness Channel |
| 34 | DefaultLivenessChannelInterval = time.Second * 30 |
| 35 | ) |
| 36 | |
| 37 | // Backend structure holds details for accessing the kv store |
| 38 | type Backend struct { |
Matteo Scandolo | 2e6f1e3 | 2020-04-15 11:28:45 -0700 | [diff] [blame] | 39 | Client kvstore.Client |
| 40 | StoreType string |
mpagenko | af80163 | 2020-07-03 10:00:42 +0000 | [diff] [blame] | 41 | Timeout time.Duration |
| 42 | Address string |
Matteo Scandolo | 2e6f1e3 | 2020-04-15 11:28:45 -0700 | [diff] [blame] | 43 | PathPrefix string |
dbainbri | 4d3a0dc | 2020-12-02 00:33:42 +0000 | [diff] [blame] | 44 | alive bool // Is this backend connection alive? |
| 45 | livenessMutex sync.Mutex |
Matteo Scandolo | 2e6f1e3 | 2020-04-15 11:28:45 -0700 | [diff] [blame] | 46 | liveness chan bool // channel to post alive state |
| 47 | LivenessChannelInterval time.Duration // regularly push alive state beyond this interval |
| 48 | lastLivenessTime time.Time // Instant of last alive state push |
| 49 | } |
| 50 | |
| 51 | // NewBackend creates a new instance of a Backend structure |
dbainbri | 4d3a0dc | 2020-12-02 00:33:42 +0000 | [diff] [blame] | 52 | func NewBackend(ctx context.Context, storeType string, address string, timeout time.Duration, pathPrefix string) *Backend { |
Matteo Scandolo | 2e6f1e3 | 2020-04-15 11:28:45 -0700 | [diff] [blame] | 53 | var err error |
| 54 | |
| 55 | b := &Backend{ |
| 56 | StoreType: storeType, |
mpagenko | af80163 | 2020-07-03 10:00:42 +0000 | [diff] [blame] | 57 | Address: address, |
Matteo Scandolo | 2e6f1e3 | 2020-04-15 11:28:45 -0700 | [diff] [blame] | 58 | Timeout: timeout, |
| 59 | LivenessChannelInterval: DefaultLivenessChannelInterval, |
| 60 | PathPrefix: pathPrefix, |
| 61 | alive: false, // connection considered down at start |
| 62 | } |
| 63 | |
dbainbri | 4d3a0dc | 2020-12-02 00:33:42 +0000 | [diff] [blame] | 64 | if b.Client, err = b.newClient(ctx, address, timeout); err != nil { |
| 65 | logger.Errorw(ctx, "failed-to-create-kv-client", |
Matteo Scandolo | 2e6f1e3 | 2020-04-15 11:28:45 -0700 | [diff] [blame] | 66 | log.Fields{ |
mpagenko | af80163 | 2020-07-03 10:00:42 +0000 | [diff] [blame] | 67 | "type": storeType, "address": address, |
Matteo Scandolo | 2e6f1e3 | 2020-04-15 11:28:45 -0700 | [diff] [blame] | 68 | "timeout": timeout, "prefix": pathPrefix, |
| 69 | "error": err.Error(), |
| 70 | }) |
| 71 | } |
| 72 | |
| 73 | return b |
| 74 | } |
| 75 | |
dbainbri | 4d3a0dc | 2020-12-02 00:33:42 +0000 | [diff] [blame] | 76 | func (b *Backend) newClient(ctx context.Context, address string, timeout time.Duration) (kvstore.Client, error) { |
Matteo Scandolo | 2e6f1e3 | 2020-04-15 11:28:45 -0700 | [diff] [blame] | 77 | switch b.StoreType { |
| 78 | case "consul": |
dbainbri | 4d3a0dc | 2020-12-02 00:33:42 +0000 | [diff] [blame] | 79 | return kvstore.NewConsulClient(ctx, address, timeout) |
Matteo Scandolo | 2e6f1e3 | 2020-04-15 11:28:45 -0700 | [diff] [blame] | 80 | case "etcd": |
dbainbri | 4d3a0dc | 2020-12-02 00:33:42 +0000 | [diff] [blame] | 81 | return kvstore.NewEtcdClient(ctx, address, timeout, log.WarnLevel) |
Matteo Scandolo | 2e6f1e3 | 2020-04-15 11:28:45 -0700 | [diff] [blame] | 82 | } |
| 83 | return nil, errors.New("unsupported-kv-store") |
| 84 | } |
| 85 | |
dbainbri | 4d3a0dc | 2020-12-02 00:33:42 +0000 | [diff] [blame] | 86 | func (b *Backend) makePath(ctx context.Context, key string) string { |
Matteo Scandolo | 2e6f1e3 | 2020-04-15 11:28:45 -0700 | [diff] [blame] | 87 | path := fmt.Sprintf("%s/%s", b.PathPrefix, key) |
| 88 | return path |
| 89 | } |
| 90 | |
dbainbri | 4d3a0dc | 2020-12-02 00:33:42 +0000 | [diff] [blame] | 91 | func (b *Backend) updateLiveness(ctx context.Context, alive bool) { |
Matteo Scandolo | 2e6f1e3 | 2020-04-15 11:28:45 -0700 | [diff] [blame] | 92 | // Periodically push stream of liveness data to the channel, |
| 93 | // so that in a live state, the core does not timeout and |
| 94 | // send a forced liveness message. Push alive state if the |
| 95 | // last push to channel was beyond livenessChannelInterval |
dbainbri | 4d3a0dc | 2020-12-02 00:33:42 +0000 | [diff] [blame] | 96 | b.livenessMutex.Lock() |
| 97 | defer b.livenessMutex.Unlock() |
Matteo Scandolo | 2e6f1e3 | 2020-04-15 11:28:45 -0700 | [diff] [blame] | 98 | if b.liveness != nil { |
Matteo Scandolo | 2e6f1e3 | 2020-04-15 11:28:45 -0700 | [diff] [blame] | 99 | if b.alive != alive { |
dbainbri | 4d3a0dc | 2020-12-02 00:33:42 +0000 | [diff] [blame] | 100 | logger.Debug(ctx, "update-liveness-channel-reason-change") |
Matteo Scandolo | 2e6f1e3 | 2020-04-15 11:28:45 -0700 | [diff] [blame] | 101 | b.liveness <- alive |
| 102 | b.lastLivenessTime = time.Now() |
| 103 | } else if time.Since(b.lastLivenessTime) > b.LivenessChannelInterval { |
dbainbri | 4d3a0dc | 2020-12-02 00:33:42 +0000 | [diff] [blame] | 104 | logger.Debug(ctx, "update-liveness-channel-reason-interval") |
Matteo Scandolo | 2e6f1e3 | 2020-04-15 11:28:45 -0700 | [diff] [blame] | 105 | b.liveness <- alive |
| 106 | b.lastLivenessTime = time.Now() |
| 107 | } |
| 108 | } |
| 109 | |
| 110 | // Emit log message only for alive state change |
| 111 | if b.alive != alive { |
dbainbri | 4d3a0dc | 2020-12-02 00:33:42 +0000 | [diff] [blame] | 112 | logger.Debugw(ctx, "change-kvstore-alive-status", log.Fields{"alive": alive}) |
Matteo Scandolo | 2e6f1e3 | 2020-04-15 11:28:45 -0700 | [diff] [blame] | 113 | b.alive = alive |
| 114 | } |
| 115 | } |
| 116 | |
| 117 | // Perform a dummy Key Lookup on kvstore to test Connection Liveness and |
| 118 | // post on Liveness channel |
| 119 | func (b *Backend) PerformLivenessCheck(ctx context.Context) bool { |
| 120 | alive := b.Client.IsConnectionUp(ctx) |
dbainbri | 4d3a0dc | 2020-12-02 00:33:42 +0000 | [diff] [blame] | 121 | logger.Debugw(ctx, "kvstore-liveness-check-result", log.Fields{"alive": alive}) |
Matteo Scandolo | 2e6f1e3 | 2020-04-15 11:28:45 -0700 | [diff] [blame] | 122 | |
dbainbri | 4d3a0dc | 2020-12-02 00:33:42 +0000 | [diff] [blame] | 123 | b.updateLiveness(ctx, alive) |
Matteo Scandolo | 2e6f1e3 | 2020-04-15 11:28:45 -0700 | [diff] [blame] | 124 | return alive |
| 125 | } |
| 126 | |
| 127 | // Enable the liveness monitor channel. This channel will report |
| 128 | // a "true" or "false" on every kvstore operation which indicates whether |
| 129 | // or not the connection is still Live. This channel is then picked up |
| 130 | // by the service (i.e. rw_core / ro_core) to update readiness status |
| 131 | // and/or take other actions. |
dbainbri | 4d3a0dc | 2020-12-02 00:33:42 +0000 | [diff] [blame] | 132 | func (b *Backend) EnableLivenessChannel(ctx context.Context) chan bool { |
| 133 | logger.Debug(ctx, "enable-kvstore-liveness-channel") |
| 134 | b.livenessMutex.Lock() |
| 135 | defer b.livenessMutex.Unlock() |
Matteo Scandolo | 2e6f1e3 | 2020-04-15 11:28:45 -0700 | [diff] [blame] | 136 | if b.liveness == nil { |
Matteo Scandolo | 2e6f1e3 | 2020-04-15 11:28:45 -0700 | [diff] [blame] | 137 | b.liveness = make(chan bool, 10) |
Matteo Scandolo | 2e6f1e3 | 2020-04-15 11:28:45 -0700 | [diff] [blame] | 138 | b.liveness <- b.alive |
| 139 | b.lastLivenessTime = time.Now() |
| 140 | } |
| 141 | |
| 142 | return b.liveness |
| 143 | } |
| 144 | |
| 145 | // Extract Alive status of Kvstore based on type of error |
dbainbri | 4d3a0dc | 2020-12-02 00:33:42 +0000 | [diff] [blame] | 146 | func (b *Backend) isErrorIndicatingAliveKvstore(ctx context.Context, err error) bool { |
Matteo Scandolo | 2e6f1e3 | 2020-04-15 11:28:45 -0700 | [diff] [blame] | 147 | // Alive unless observed an error indicating so |
| 148 | alive := true |
| 149 | |
| 150 | if err != nil { |
| 151 | |
| 152 | // timeout indicates kvstore not reachable/alive |
| 153 | if err == context.DeadlineExceeded { |
| 154 | alive = false |
| 155 | } |
| 156 | |
| 157 | // Need to analyze client-specific errors based on backend type |
| 158 | if b.StoreType == "etcd" { |
| 159 | |
| 160 | // For etcd backend, consider not-alive only for errors indicating |
| 161 | // timedout request or unavailable/corrupted cluster. For all remaining |
| 162 | // error codes listed in https://godoc.org/google.golang.org/grpc/codes#Code, |
| 163 | // we would not infer a not-alive backend because such a error may also |
| 164 | // occur due to bad client requests or sequence of operations |
| 165 | switch status.Code(err) { |
| 166 | case codes.DeadlineExceeded: |
| 167 | fallthrough |
| 168 | case codes.Unavailable: |
| 169 | fallthrough |
| 170 | case codes.DataLoss: |
| 171 | alive = false |
| 172 | } |
| 173 | |
| 174 | //} else { |
| 175 | // TODO: Implement for consul backend; would it be needed ever? |
| 176 | } |
| 177 | } |
| 178 | |
| 179 | return alive |
| 180 | } |
| 181 | |
| 182 | // List retrieves one or more items that match the specified key |
| 183 | func (b *Backend) List(ctx context.Context, key string) (map[string]*kvstore.KVPair, error) { |
dbainbri | 4d3a0dc | 2020-12-02 00:33:42 +0000 | [diff] [blame] | 184 | span, ctx := log.CreateChildSpan(ctx, "etcd-list") |
| 185 | defer span.Finish() |
| 186 | |
| 187 | formattedPath := b.makePath(ctx, key) |
| 188 | logger.Debugw(ctx, "listing-key", log.Fields{"key": key, "path": formattedPath}) |
Matteo Scandolo | 2e6f1e3 | 2020-04-15 11:28:45 -0700 | [diff] [blame] | 189 | |
| 190 | pair, err := b.Client.List(ctx, formattedPath) |
| 191 | |
dbainbri | 4d3a0dc | 2020-12-02 00:33:42 +0000 | [diff] [blame] | 192 | b.updateLiveness(ctx, b.isErrorIndicatingAliveKvstore(ctx, err)) |
Matteo Scandolo | 2e6f1e3 | 2020-04-15 11:28:45 -0700 | [diff] [blame] | 193 | |
| 194 | return pair, err |
| 195 | } |
| 196 | |
| 197 | // Get retrieves an item that matches the specified key |
| 198 | func (b *Backend) Get(ctx context.Context, key string) (*kvstore.KVPair, error) { |
dbainbri | 4d3a0dc | 2020-12-02 00:33:42 +0000 | [diff] [blame] | 199 | span, ctx := log.CreateChildSpan(ctx, "etcd-get") |
| 200 | defer span.Finish() |
| 201 | |
| 202 | formattedPath := b.makePath(ctx, key) |
| 203 | logger.Debugw(ctx, "getting-key", log.Fields{"key": key, "path": formattedPath}) |
Matteo Scandolo | 2e6f1e3 | 2020-04-15 11:28:45 -0700 | [diff] [blame] | 204 | |
| 205 | pair, err := b.Client.Get(ctx, formattedPath) |
| 206 | |
dbainbri | 4d3a0dc | 2020-12-02 00:33:42 +0000 | [diff] [blame] | 207 | b.updateLiveness(ctx, b.isErrorIndicatingAliveKvstore(ctx, err)) |
Matteo Scandolo | 2e6f1e3 | 2020-04-15 11:28:45 -0700 | [diff] [blame] | 208 | |
| 209 | return pair, err |
| 210 | } |
| 211 | |
| 212 | // Put stores an item value under the specifed key |
| 213 | func (b *Backend) Put(ctx context.Context, key string, value interface{}) error { |
dbainbri | 4d3a0dc | 2020-12-02 00:33:42 +0000 | [diff] [blame] | 214 | span, ctx := log.CreateChildSpan(ctx, "etcd-put") |
| 215 | defer span.Finish() |
| 216 | |
| 217 | formattedPath := b.makePath(ctx, key) |
| 218 | logger.Debugw(ctx, "putting-key", log.Fields{"key": key, "path": formattedPath}) |
Matteo Scandolo | 2e6f1e3 | 2020-04-15 11:28:45 -0700 | [diff] [blame] | 219 | |
| 220 | err := b.Client.Put(ctx, formattedPath, value) |
| 221 | |
dbainbri | 4d3a0dc | 2020-12-02 00:33:42 +0000 | [diff] [blame] | 222 | b.updateLiveness(ctx, b.isErrorIndicatingAliveKvstore(ctx, err)) |
Matteo Scandolo | 2e6f1e3 | 2020-04-15 11:28:45 -0700 | [diff] [blame] | 223 | |
| 224 | return err |
| 225 | } |
| 226 | |
| 227 | // Delete removes an item under the specified key |
| 228 | func (b *Backend) Delete(ctx context.Context, key string) error { |
dbainbri | 4d3a0dc | 2020-12-02 00:33:42 +0000 | [diff] [blame] | 229 | span, ctx := log.CreateChildSpan(ctx, "etcd-delete") |
| 230 | defer span.Finish() |
| 231 | |
| 232 | formattedPath := b.makePath(ctx, key) |
| 233 | logger.Debugw(ctx, "deleting-key", log.Fields{"key": key, "path": formattedPath}) |
Matteo Scandolo | 2e6f1e3 | 2020-04-15 11:28:45 -0700 | [diff] [blame] | 234 | |
| 235 | err := b.Client.Delete(ctx, formattedPath) |
| 236 | |
dbainbri | 4d3a0dc | 2020-12-02 00:33:42 +0000 | [diff] [blame] | 237 | b.updateLiveness(ctx, b.isErrorIndicatingAliveKvstore(ctx, err)) |
Matteo Scandolo | 2e6f1e3 | 2020-04-15 11:28:45 -0700 | [diff] [blame] | 238 | |
| 239 | return err |
| 240 | } |
| 241 | |
| 242 | // CreateWatch starts watching events for the specified key |
| 243 | func (b *Backend) CreateWatch(ctx context.Context, key string, withPrefix bool) chan *kvstore.Event { |
dbainbri | 4d3a0dc | 2020-12-02 00:33:42 +0000 | [diff] [blame] | 244 | span, ctx := log.CreateChildSpan(ctx, "etcd-create-watch") |
| 245 | defer span.Finish() |
| 246 | |
| 247 | formattedPath := b.makePath(ctx, key) |
| 248 | logger.Debugw(ctx, "creating-key-watch", log.Fields{"key": key, "path": formattedPath}) |
Matteo Scandolo | 2e6f1e3 | 2020-04-15 11:28:45 -0700 | [diff] [blame] | 249 | |
| 250 | return b.Client.Watch(ctx, formattedPath, withPrefix) |
| 251 | } |
| 252 | |
| 253 | // DeleteWatch stops watching events for the specified key |
dbainbri | 4d3a0dc | 2020-12-02 00:33:42 +0000 | [diff] [blame] | 254 | func (b *Backend) DeleteWatch(ctx context.Context, key string, ch chan *kvstore.Event) { |
| 255 | span, ctx := log.CreateChildSpan(ctx, "etcd-delete-watch") |
| 256 | defer span.Finish() |
Matteo Scandolo | 2e6f1e3 | 2020-04-15 11:28:45 -0700 | [diff] [blame] | 257 | |
dbainbri | 4d3a0dc | 2020-12-02 00:33:42 +0000 | [diff] [blame] | 258 | formattedPath := b.makePath(ctx, key) |
| 259 | logger.Debugw(ctx, "deleting-key-watch", log.Fields{"key": key, "path": formattedPath}) |
| 260 | |
| 261 | b.Client.CloseWatch(ctx, formattedPath, ch) |
Matteo Scandolo | 2e6f1e3 | 2020-04-15 11:28:45 -0700 | [diff] [blame] | 262 | } |