VOL-2009[RO Core doesn't retry KV store connection on startup]
Change-Id: I01ed30d41d968f1bf9e052014eae420973d85266
diff --git a/ro_core/core/core.go b/ro_core/core/core.go
index d022266..797c05a 100644
--- a/ro_core/core/core.go
+++ b/ro_core/core/core.go
@@ -26,6 +26,8 @@
"github.com/opencord/voltha-lib-go/v2/pkg/probe"
"github.com/opencord/voltha-protos/v2/go/voltha"
"google.golang.org/grpc"
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/status"
"time"
)
@@ -80,8 +82,48 @@
return &core
}
+// waitUntilKVStoreReachableOrMaxTries will wait until it can connect to a KV store or until maxtries has been reached
+func (core *Core) waitUntilKVStoreReachableOrMaxTries(ctx context.Context, maxRetries int, retryInterval time.Duration) error {
+ log.Infow("verifying-KV-store-connectivity", log.Fields{"host": core.config.KVStoreHost,
+ "port": core.config.KVStorePort, "retries": maxRetries, "retryInterval": retryInterval})
+
+ // Get timeout in seconds with 1 second set as minimum
+ timeout := int(core.config.CoreTimeout.Seconds())
+ if timeout < 1 {
+ timeout = 1
+ }
+ count := 0
+ for {
+ if !core.kvClient.IsConnectionUp(timeout) {
+ log.Info("KV-store-unreachable")
+ if maxRetries != -1 {
+ if count >= maxRetries {
+ return status.Error(codes.Unavailable, "kv store unreachable")
+ }
+ }
+ count += 1
+ // Take a nap before retrying
+ time.Sleep(retryInterval)
+ log.Infow("retry-KV-store-connectivity", log.Fields{"retryCount": count, "maxRetries": maxRetries, "retryInterval": retryInterval})
+
+ } else {
+ break
+ }
+ }
+ log.Info("KV-store-reachable")
+ return nil
+}
+
func (core *Core) Start(ctx context.Context) {
log.Info("starting-adaptercore", log.Fields{"coreId": core.instanceId})
+
+ // Wait until connection to KV Store is up
+ if err := core.waitUntilKVStoreReachableOrMaxTries(ctx, core.config.MaxConnectionRetries, core.config.ConnectionRetryInterval); err != nil {
+ log.Fatal("Unable-to-connect-to-KV-store")
+ }
+
+ probe.UpdateStatusFromContext(ctx, "kv-store", probe.ServiceStatusRunning)
+
core.genericMgr = newModelProxyManager(core.clusterDataProxy)
core.deviceMgr = newDeviceManager(core.clusterDataProxy, core.instanceId)
core.logicalDeviceMgr = newLogicalDeviceManager(core.deviceMgr, core.clusterDataProxy)
diff --git a/ro_core/core/core_test.go b/ro_core/core/core_test.go
index 948e63b..88bd561 100644
--- a/ro_core/core/core_test.go
+++ b/ro_core/core/core_test.go
@@ -18,14 +18,15 @@
import (
"context"
"errors"
+ "fmt"
"github.com/opencord/voltha-go/ro_core/config"
"github.com/opencord/voltha-lib-go/v2/pkg/db/kvstore"
grpcserver "github.com/opencord/voltha-lib-go/v2/pkg/grpc"
"github.com/opencord/voltha-lib-go/v2/pkg/log"
+ "github.com/opencord/voltha-lib-go/v2/pkg/mocks"
ic "github.com/opencord/voltha-protos/v2/go/inter_container"
"github.com/phayes/freeport"
"github.com/stretchr/testify/assert"
- "strconv"
"testing"
)
@@ -63,19 +64,31 @@
func MakeTestNewCore() (*config.ROCoreFlags, *roCore) {
- freePort, errP := freeport.GetFreePort()
- if errP == nil {
- freePortStr := strconv.Itoa(freePort)
+ clientPort, err := freeport.GetFreePort()
+ if err == nil {
+ peerPort, err := freeport.GetFreePort()
+ if err != nil {
+ log.Fatal(err)
+ }
+ etcdServer := mocks.StartEtcdServer(mocks.MKConfig("voltha.mock.test", clientPort, peerPort, "voltha.lib.mocks.etcd", "error"))
+ if etcdServer == nil {
+ log.Fatal("Embedded server failed to start")
+ }
+ clientAddr := fmt.Sprintf("localhost:%d", clientPort)
roCoreFlgs := config.NewROCoreFlags()
roC := newROCore(roCoreFlgs)
if (roC != nil) && (roCoreFlgs != nil) {
- addr := "127.0.0.1" + ":" + freePortStr
- cli, err := newKVClient("etcd", addr, 5)
+ cli, err := newKVClient("etcd", clientAddr, 5)
if err == nil {
roC.kvClient = cli
return roCoreFlgs, roC
}
+ if err != nil {
+ etcdServer.Stop()
+ log.Fatal("Failed to create an Etcd client")
+ }
+
}
}
return nil, nil