VOL-2191: Implement the right interpretation of instance-control attribute
from tech-profile
- When using "single-instance" type TP, make sure no other gem-port on any
other UNI port is referncing the same alloc-id before releasing the alloc-id.
- When deleting tech-profile instances for the ONU, log any errors with deleting
any TP instance, but do not break the loop (so that other TP instances can be
freed up).
- Use 2.2.17 version of voltha-lib-go
Change-Id: I13901e6e3f21b02af076c4c022d4caafc10d6491
diff --git a/vendor/github.com/opencord/voltha-lib-go/v2/pkg/db/backend.go b/vendor/github.com/opencord/voltha-lib-go/v2/pkg/db/backend.go
index c319d99..b2547c2 100644
--- a/vendor/github.com/opencord/voltha-lib-go/v2/pkg/db/backend.go
+++ b/vendor/github.com/opencord/voltha-lib-go/v2/pkg/db/backend.go
@@ -17,23 +17,36 @@
package db
import (
+ "context"
"errors"
"fmt"
"github.com/opencord/voltha-lib-go/v2/pkg/db/kvstore"
"github.com/opencord/voltha-lib-go/v2/pkg/log"
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/status"
"strconv"
"sync"
+ "time"
+)
+
+const (
+ // Default Minimal Interval for posting alive state of backend kvstore on Liveness Channel
+ DefaultLivenessChannelInterval = time.Second * 30
)
// Backend structure holds details for accessing the kv store
type Backend struct {
sync.RWMutex
- Client kvstore.Client
- StoreType string
- Host string
- Port int
- Timeout int
- PathPrefix string
+ Client kvstore.Client
+ StoreType string
+ Host string
+ Port int
+ Timeout int
+ PathPrefix string
+ alive bool // Is this backend connection alive?
+ liveness chan bool // channel to post alive state
+ LivenessChannelInterval time.Duration // regularly push alive state beyond this interval
+ lastLivenessTime time.Time // Instant of last alive state push
}
// NewBackend creates a new instance of a Backend structure
@@ -41,11 +54,13 @@
var err error
b := &Backend{
- StoreType: storeType,
- Host: host,
- Port: port,
- Timeout: timeout,
- PathPrefix: pathPrefix,
+ StoreType: storeType,
+ Host: host,
+ Port: port,
+ Timeout: timeout,
+ LivenessChannelInterval: DefaultLivenessChannelInterval,
+ PathPrefix: pathPrefix,
+ alive: false, // connection considered down at start
}
address := host + ":" + strconv.Itoa(port)
@@ -76,6 +91,100 @@
return path
}
+func (b *Backend) updateLiveness(alive bool) {
+ // Periodically push stream of liveness data to the channel,
+ // so that in a live state, the core does not timeout and
+ // send a forced liveness message. Push alive state if the
+ // last push to channel was beyond livenessChannelInterval
+ if b.liveness != nil {
+
+ if b.alive != alive {
+ log.Debug("update-liveness-channel-reason-change")
+ b.liveness <- alive
+ b.lastLivenessTime = time.Now()
+ } else if time.Now().Sub(b.lastLivenessTime) > b.LivenessChannelInterval {
+ log.Debug("update-liveness-channel-reason-interval")
+ b.liveness <- alive
+ b.lastLivenessTime = time.Now()
+ }
+ }
+
+ // Emit log message only for alive state change
+ if b.alive != alive {
+ log.Debugw("change-kvstore-alive-status", log.Fields{"alive": alive})
+ b.alive = alive
+ }
+}
+
+// Perform a dummy Key Lookup on kvstore to test Connection Liveness and
+// post on Liveness channel
+func (b *Backend) PerformLivenessCheck(timeout int) bool {
+ alive := b.Client.IsConnectionUp(timeout)
+ log.Debugw("kvstore-liveness-check-result", log.Fields{"alive": alive})
+
+ b.updateLiveness(alive)
+ return alive
+}
+
+// Enable the liveness monitor channel. This channel will report
+// a "true" or "false" on every kvstore operation which indicates whether
+// or not the connection is still Live. This channel is then picked up
+// by the service (i.e. rw_core / ro_core) to update readiness status
+// and/or take other actions.
+func (b *Backend) EnableLivenessChannel() chan bool {
+ log.Debug("enable-kvstore-liveness-channel")
+
+ if b.liveness == nil {
+ log.Debug("create-kvstore-liveness-channel")
+
+ // Channel size of 10 to avoid any possibility of blocking in Load conditions
+ b.liveness = make(chan bool, 10)
+
+ // Post initial alive state
+ b.liveness <- b.alive
+ b.lastLivenessTime = time.Now()
+ }
+
+ return b.liveness
+}
+
+// Extract Alive status of Kvstore based on type of error
+func (b *Backend) isErrorIndicatingAliveKvstore(err error) bool {
+ // Alive unless observed an error indicating so
+ alive := true
+
+ if err != nil {
+
+ // timeout indicates kvstore not reachable/alive
+ if err == context.DeadlineExceeded {
+ alive = false
+ }
+
+ // Need to analyze client-specific errors based on backend type
+ if b.StoreType == "etcd" {
+
+ // For etcd backend, consider not-alive only for errors indicating
+ // timedout request or unavailable/corrupted cluster. For all remaining
+ // error codes listed in https://godoc.org/google.golang.org/grpc/codes#Code,
+ // we would not infer a not-alive backend because such a error may also
+ // occur due to bad client requests or sequence of operations
+ switch status.Code(err) {
+ case codes.DeadlineExceeded:
+ fallthrough
+ case codes.Unavailable:
+ fallthrough
+ case codes.DataLoss:
+ alive = false
+ }
+
+ //} else {
+ // TODO: Implement for consul backend; would it be needed ever?
+ }
+ }
+
+ return alive
+}
+
// List retrieves one or more items that match the specified key
func (b *Backend) List(key string) (map[string]*kvstore.KVPair, error) {
b.Lock()
@@ -84,7 +193,11 @@
formattedPath := b.makePath(key)
log.Debugw("listing-key", log.Fields{"key": key, "path": formattedPath})
- return b.Client.List(formattedPath, b.Timeout)
+ pair, err := b.Client.List(formattedPath, b.Timeout)
+
+ b.updateLiveness(b.isErrorIndicatingAliveKvstore(err))
+
+ return pair, err
}
// Get retrieves an item that matches the specified key
@@ -95,7 +208,11 @@
formattedPath := b.makePath(key)
log.Debugw("getting-key", log.Fields{"key": key, "path": formattedPath})
- return b.Client.Get(formattedPath, b.Timeout)
+ pair, err := b.Client.Get(formattedPath, b.Timeout)
+
+ b.updateLiveness(b.isErrorIndicatingAliveKvstore(err))
+
+ return pair, err
}
// Put stores an item value under the specifed key
@@ -106,7 +223,11 @@
formattedPath := b.makePath(key)
log.Debugw("putting-key", log.Fields{"key": key, "value": string(value.([]byte)), "path": formattedPath})
- return b.Client.Put(formattedPath, value, b.Timeout)
+ err := b.Client.Put(formattedPath, value, b.Timeout)
+
+ b.updateLiveness(b.isErrorIndicatingAliveKvstore(err))
+
+ return err
}
// Delete removes an item under the specified key
@@ -117,7 +238,11 @@
formattedPath := b.makePath(key)
log.Debugw("deleting-key", log.Fields{"key": key, "path": formattedPath})
- return b.Client.Delete(formattedPath, b.Timeout)
+ err := b.Client.Delete(formattedPath, b.Timeout)
+
+ b.updateLiveness(b.isErrorIndicatingAliveKvstore(err))
+
+ return err
}
// CreateWatch starts watching events for the specified key