VOL-1723 - add readiness probe capability to rw-core
Change-Id: I1cf42e88712586f140a2dfa9d0b638b48261caac
diff --git a/common/grpc/server.go b/common/grpc/server.go
index d5685f1..e3f3d99 100644
--- a/common/grpc/server.go
+++ b/common/grpc/server.go
@@ -118,7 +118,9 @@
Stop servicing GRPC requests
*/
func (s *GrpcServer) Stop() {
- s.gs.Stop()
+ if s.gs != nil {
+ s.gs.Stop()
+ }
}
/*
diff --git a/common/probe/probe.go b/common/probe/probe.go
new file mode 100644
index 0000000..984036c
--- /dev/null
+++ b/common/probe/probe.go
@@ -0,0 +1,241 @@
+/*
+ * Copyright 2019-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package probe
+
+import (
+ "context"
+ "fmt"
+ "github.com/opencord/voltha-go/common/log"
+ "net/http"
+ "sync"
+)
+
+// ProbeContextKey used to fetch the Probe instance from a context
+type ProbeContextKeyType string
+
+// ServiceStatus typed values for service status
+type ServiceStatus int
+
+const (
+ // ServiceStatusUnknown initial state of services
+ ServiceStatusUnknown ServiceStatus = iota
+
+ // ServiceStatusPreparing to optionally be used for prep, such as connecting
+ ServiceStatusPreparing
+
+ // ServiceStatusPrepared to optionally be used when prep is complete, but before run
+ ServiceStatusPrepared
+
+ // ServiceStatusRunning service is functional
+ ServiceStatusRunning
+
+ // ServiceStatusStopped service has stopped, but not because of error
+ ServiceStatusStopped
+
+ // ServiceStatusFailed service has stopped because of an error
+ ServiceStatusFailed
+)
+
+const (
+ // ProbeContextKey value of context key to fetch probe
+ ProbeContextKey = ProbeContextKeyType("status-update-probe")
+)
+
+// String convert ServiceStatus values to strings
+func (s ServiceStatus) String() string {
+ switch s {
+ default:
+ fallthrough
+ case ServiceStatusUnknown:
+ return "Unknown"
+ case ServiceStatusPreparing:
+ return "Preparing"
+ case ServiceStatusPrepared:
+ return "Prepared"
+ case ServiceStatusRunning:
+ return "Running"
+ case ServiceStatusStopped:
+ return "Stopped"
+ case ServiceStatusFailed:
+ return "Failed"
+ }
+}
+
+// ServiceStatusUpdate status update event
+type ServiceStatusUpdate struct {
+ Name string
+ Status ServiceStatus
+}
+
+// Probe reciever on which to implement probe capabilities
+type Probe struct {
+ readyFunc func(map[string]ServiceStatus) bool
+ healthFunc func(map[string]ServiceStatus) bool
+
+ mutex sync.RWMutex
+ status map[string]ServiceStatus
+ isReady bool
+ isHealthy bool
+}
+
+// WithReadyFunc override the default ready calculation function
+func (p *Probe) WithReadyFunc(readyFunc func(map[string]ServiceStatus) bool) *Probe {
+ p.readyFunc = readyFunc
+ return p
+}
+
+// WithHealthFunc override the default health calculation function
+func (p *Probe) WithHealthFunc(healthFunc func(map[string]ServiceStatus) bool) *Probe {
+ p.healthFunc = healthFunc
+ return p
+}
+
+// RegisterService register one or more service names with the probe, status will be track against service name
+func (p *Probe) RegisterService(names ...string) {
+ p.mutex.Lock()
+ defer p.mutex.Unlock()
+ if p.status == nil {
+ p.status = make(map[string]ServiceStatus)
+ }
+ for _, name := range names {
+ if _, ok := p.status[name]; !ok {
+ p.status[name] = ServiceStatusUnknown
+ log.Debugw("probe-service-registered", log.Fields{"service-name": name})
+ }
+ }
+}
+
+// UpdateStatus utility function to send a service update to the probe
+func (p *Probe) UpdateStatus(name string, status ServiceStatus) {
+ p.mutex.Lock()
+ defer p.mutex.Unlock()
+ if p.status == nil {
+ p.status = make(map[string]ServiceStatus)
+ }
+ p.status[name] = status
+ if p.readyFunc != nil {
+ p.isReady = p.readyFunc(p.status)
+ } else {
+ p.isReady = defaultReadyFunc(p.status)
+ }
+
+ if p.healthFunc != nil {
+ p.isHealthy = p.healthFunc(p.status)
+ } else {
+ p.isHealthy = defaultHealthFunc(p.status)
+ }
+ log.Debugw("probe-service-status-updated",
+ log.Fields{
+ "service-name": name,
+ "status": status.String(),
+ "ready": p.isReady,
+ "health": p.isHealthy,
+ })
+}
+
+// UpdateStatusFromContext a convenience function to pull the Probe reference from the
+// Context, if it exists, and then calling UpdateStatus on that Probe reference. If Context
+// is nil or if a Probe reference is not associated with the ProbeContextKey then nothing
+// happens
+func UpdateStatusFromContext(ctx context.Context, name string, status ServiceStatus) {
+ if ctx != nil {
+ if value := ctx.Value(ProbeContextKey); value != nil {
+ if p, ok := value.(*Probe); ok {
+ p.UpdateStatus(name, status)
+ }
+ }
+ }
+}
+
+// pulled out to a function to help better enable unit testing
+func (p *Probe) readzFunc(w http.ResponseWriter, req *http.Request) {
+ p.mutex.RLock()
+ defer p.mutex.RUnlock()
+ if p.isReady {
+ w.WriteHeader(http.StatusOK)
+ } else {
+ w.WriteHeader(http.StatusTeapot)
+ }
+}
+func (p *Probe) healthzFunc(w http.ResponseWriter, req *http.Request) {
+ p.mutex.RLock()
+ defer p.mutex.RUnlock()
+ if p.isHealthy {
+ w.WriteHeader(http.StatusOK)
+ } else {
+ w.WriteHeader(http.StatusTeapot)
+ }
+}
+func (p *Probe) detailzFunc(w http.ResponseWriter, req *http.Request) {
+ p.mutex.RLock()
+ defer p.mutex.RUnlock()
+ w.Header().Set("Content-Type", "application/json")
+ w.Write([]byte("{"))
+ comma := ""
+ for c, s := range p.status {
+ w.Write([]byte(fmt.Sprintf("%s\"%s\": \"%s\"", comma, c, s.String())))
+ comma = ", "
+ }
+ w.Write([]byte("}"))
+ w.WriteHeader(http.StatusOK)
+
+}
+
+// ListenAndServe implements 3 HTTP endpoints on the given port for healthz, readz, and detailz. Returns only on error
+func (p *Probe) ListenAndServe(port int) {
+ mux := http.NewServeMux()
+
+ // Returns the result of the readyFunc calculation
+ mux.HandleFunc("/readz", p.readzFunc)
+
+ // Returns the result of the healthFunc calculation
+ mux.HandleFunc("/healthz", p.healthzFunc)
+
+ // Returns the details of the services and their status as JSON
+ mux.HandleFunc("/detailz", p.detailzFunc)
+ s := &http.Server{
+ Addr: fmt.Sprintf(":%d", port),
+ Handler: mux,
+ }
+ log.Fatal(s.ListenAndServe())
+}
+
+// defaultReadyFunc if all services are running then ready, else not
+func defaultReadyFunc(services map[string]ServiceStatus) bool {
+ if len(services) == 0 {
+ return false
+ }
+ for _, status := range services {
+ if status != ServiceStatusRunning {
+ return false
+ }
+ }
+ return true
+}
+
+// defaultHealthFunc if no service is stopped or failed, then healthy, else not.
+// service is start as unknown, so they are considered healthy
+func defaultHealthFunc(services map[string]ServiceStatus) bool {
+ if len(services) == 0 {
+ return false
+ }
+ for _, status := range services {
+ if status == ServiceStatusStopped || status == ServiceStatusFailed {
+ return false
+ }
+ }
+ return true
+}
diff --git a/common/probe/probe_test.go b/common/probe/probe_test.go
new file mode 100644
index 0000000..18c4835
--- /dev/null
+++ b/common/probe/probe_test.go
@@ -0,0 +1,372 @@
+/*
+ * Copyright 2019-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package probe
+
+import (
+ "context"
+ "encoding/json"
+ "github.com/opencord/voltha-go/common/log"
+ "github.com/stretchr/testify/assert"
+ "io/ioutil"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+)
+
+func init() {
+ log.AddPackage(log.JSON, log.WarnLevel, nil)
+}
+
+func TestServiceStatusString(t *testing.T) {
+ assert.Equal(t, "Unknown", ServiceStatusUnknown.String(), "ServiceStatusUnknown")
+ assert.Equal(t, "Preparing", ServiceStatusPreparing.String(), "ServiceStatusPreparing")
+ assert.Equal(t, "Prepared", ServiceStatusPrepared.String(), "ServiceStatusPrepared")
+ assert.Equal(t, "Running", ServiceStatusRunning.String(), "ServiceStatusRunning")
+ assert.Equal(t, "Stopped", ServiceStatusStopped.String(), "ServiceStatusStopped")
+ assert.Equal(t, "Failed", ServiceStatusFailed.String(), "ServiceStatusFailed")
+}
+
+func AlwaysTrue(map[string]ServiceStatus) bool {
+ return true
+}
+
+func AlwaysFalse(map[string]ServiceStatus) bool {
+ return false
+}
+
+func TestWithFuncs(t *testing.T) {
+ p := (&Probe{}).WithReadyFunc(AlwaysTrue).WithHealthFunc(AlwaysFalse)
+
+ assert.NotNil(t, p.readyFunc, "ready func not set")
+ assert.True(t, p.readyFunc(nil), "ready func not set correctly")
+ assert.NotNil(t, p.healthFunc, "health func not set")
+ assert.False(t, p.healthFunc(nil), "health func not set correctly")
+}
+
+func TestWithReadyFuncOnly(t *testing.T) {
+ p := (&Probe{}).WithReadyFunc(AlwaysTrue)
+
+ assert.NotNil(t, p.readyFunc, "ready func not set")
+ assert.True(t, p.readyFunc(nil), "ready func not set correctly")
+ assert.Nil(t, p.healthFunc, "health func set")
+}
+
+func TestWithHealthFuncOnly(t *testing.T) {
+ p := (&Probe{}).WithHealthFunc(AlwaysTrue)
+
+ assert.Nil(t, p.readyFunc, "ready func set")
+ assert.NotNil(t, p.healthFunc, "health func not set")
+ assert.True(t, p.healthFunc(nil), "health func not set correctly")
+}
+
+func TestRegisterOneService(t *testing.T) {
+ p := &Probe{}
+
+ p.RegisterService("one")
+
+ assert.Equal(t, 1, len(p.status), "wrong number of services")
+
+ _, ok := p.status["one"]
+ assert.True(t, ok, "service not found")
+}
+
+func TestRegisterMultipleServices(t *testing.T) {
+ p := &Probe{}
+
+ p.RegisterService("one", "two", "three", "four")
+
+ assert.Equal(t, 4, len(p.status), "wrong number of services")
+
+ _, ok := p.status["one"]
+ assert.True(t, ok, "service one not found")
+ _, ok = p.status["two"]
+ assert.True(t, ok, "service two not found")
+ _, ok = p.status["three"]
+ assert.True(t, ok, "service three not found")
+ _, ok = p.status["four"]
+ assert.True(t, ok, "service four not found")
+}
+
+func TestRegisterMultipleServicesIncremental(t *testing.T) {
+ p := &Probe{}
+
+ p.RegisterService("one")
+ p.RegisterService("two")
+ p.RegisterService("three", "four")
+
+ assert.Equal(t, 4, len(p.status), "wrong number of services")
+
+ _, ok := p.status["one"]
+ assert.True(t, ok, "service one not found")
+ _, ok = p.status["two"]
+ assert.True(t, ok, "service two not found")
+ _, ok = p.status["three"]
+ assert.True(t, ok, "service three not found")
+ _, ok = p.status["four"]
+ assert.True(t, ok, "service four not found")
+}
+
+func TestRegisterMultipleServicesDuplicates(t *testing.T) {
+ p := &Probe{}
+
+ p.RegisterService("one", "one", "one", "two")
+
+ assert.Equal(t, 2, len(p.status), "wrong number of services")
+
+ _, ok := p.status["one"]
+ assert.True(t, ok, "service one not found")
+ _, ok = p.status["two"]
+ assert.True(t, ok, "service two not found")
+}
+
+func TestRegisterMultipleServicesDuplicatesIncremental(t *testing.T) {
+ p := &Probe{}
+
+ p.RegisterService("one")
+ p.RegisterService("one")
+ p.RegisterService("one", "two")
+
+ assert.Equal(t, 2, len(p.status), "wrong number of services")
+
+ _, ok := p.status["one"]
+ assert.True(t, ok, "service one not found")
+ _, ok = p.status["two"]
+ assert.True(t, ok, "service two not found")
+}
+
+func TestUpdateStatus(t *testing.T) {
+ p := &Probe{}
+
+ p.RegisterService("one", "two")
+ p.UpdateStatus("one", ServiceStatusRunning)
+
+ assert.Equal(t, ServiceStatusRunning, p.status["one"], "status not set")
+ assert.Equal(t, ServiceStatusUnknown, p.status["two"], "status set")
+}
+
+func TestRegisterOverwriteStatus(t *testing.T) {
+ p := &Probe{}
+
+ p.RegisterService("one", "two")
+ p.UpdateStatus("one", ServiceStatusRunning)
+
+ assert.Equal(t, ServiceStatusRunning, p.status["one"], "status not set")
+ assert.Equal(t, ServiceStatusUnknown, p.status["two"], "status set")
+
+ p.RegisterService("one", "three")
+ assert.Equal(t, 3, len(p.status), "wrong number of services")
+ assert.Equal(t, ServiceStatusRunning, p.status["one"], "status overridden")
+ assert.Equal(t, ServiceStatusUnknown, p.status["two"], "status set")
+ assert.Equal(t, ServiceStatusUnknown, p.status["three"], "status set")
+}
+
+func TestDetailzWithServies(t *testing.T) {
+ p := (&Probe{}).WithReadyFunc(AlwaysTrue).WithHealthFunc(AlwaysTrue)
+ p.RegisterService("one", "two")
+
+ req := httptest.NewRequest("GET", "http://example.com/detailz", nil)
+ w := httptest.NewRecorder()
+ p.detailzFunc(w, req)
+ resp := w.Result()
+ body, _ := ioutil.ReadAll(resp.Body)
+
+ assert.Equal(t, http.StatusOK, resp.StatusCode, "invalid status code for no services")
+ assert.Equal(t, "application/json", resp.Header.Get("Content-Type"), "wrong content type")
+ var vals map[string]string
+ err := json.Unmarshal(body, &vals)
+ assert.Nil(t, err, "unable to unmarshal values")
+ assert.Equal(t, "Unknown", vals["one"], "wrong value")
+ assert.Equal(t, "Unknown", vals["two"], "wrong value")
+}
+
+func TestReadzNoServices(t *testing.T) {
+ p := (&Probe{}).WithReadyFunc(AlwaysTrue)
+ req := httptest.NewRequest("GET", "http://example.com/readz", nil)
+ w := httptest.NewRecorder()
+ p.readzFunc(w, req)
+ resp := w.Result()
+
+ assert.Equal(t, http.StatusTeapot, resp.StatusCode, "invalid status code for no services")
+}
+
+func TestReadzWithServices(t *testing.T) {
+ p := (&Probe{}).WithReadyFunc(AlwaysTrue).WithHealthFunc(AlwaysTrue)
+ p.RegisterService("one", "two")
+
+ req := httptest.NewRequest("GET", "http://example.com/readz", nil)
+ w := httptest.NewRecorder()
+ p.readzFunc(w, req)
+ resp := w.Result()
+ assert.Equal(t, http.StatusTeapot, resp.StatusCode, "invalid status code for no services")
+}
+
+func TestReadzNpServicesDefault(t *testing.T) {
+ p := &Probe{}
+
+ req := httptest.NewRequest("GET", "http://example.com/readz", nil)
+ w := httptest.NewRecorder()
+ p.readzFunc(w, req)
+ resp := w.Result()
+ assert.Equal(t, http.StatusTeapot, resp.StatusCode, "invalid status code")
+}
+
+func TestReadzWithServicesDefault(t *testing.T) {
+ p := &Probe{}
+ p.RegisterService("one", "two")
+ p.UpdateStatus("one", ServiceStatusRunning)
+ p.UpdateStatus("two", ServiceStatusRunning)
+
+ req := httptest.NewRequest("GET", "http://example.com/readz", nil)
+ w := httptest.NewRecorder()
+ p.readzFunc(w, req)
+ resp := w.Result()
+ assert.Equal(t, http.StatusOK, resp.StatusCode, "invalid status code")
+}
+
+func TestReadzWithServicesDefaultOne(t *testing.T) {
+ p := &Probe{}
+ p.RegisterService("one", "two")
+ p.UpdateStatus("one", ServiceStatusRunning)
+
+ req := httptest.NewRequest("GET", "http://example.com/readz", nil)
+ w := httptest.NewRecorder()
+ p.readzFunc(w, req)
+ resp := w.Result()
+ assert.Equal(t, http.StatusTeapot, resp.StatusCode, "invalid status code")
+}
+
+func TestHealthzNoServices(t *testing.T) {
+ p := (&Probe{}).WithReadyFunc(AlwaysTrue)
+ req := httptest.NewRequest("GET", "http://example.com/healthz", nil)
+ w := httptest.NewRecorder()
+ p.healthzFunc(w, req)
+ resp := w.Result()
+
+ assert.Equal(t, http.StatusTeapot, resp.StatusCode, "invalid status code for no services")
+}
+
+func TestHealthzWithServices(t *testing.T) {
+ p := (&Probe{}).WithReadyFunc(AlwaysTrue).WithHealthFunc(AlwaysTrue)
+ p.RegisterService("one", "two")
+
+ req := httptest.NewRequest("GET", "http://example.com/healthz", nil)
+ w := httptest.NewRecorder()
+ p.healthzFunc(w, req)
+ resp := w.Result()
+ assert.Equal(t, http.StatusTeapot, resp.StatusCode, "invalid status code for no services")
+}
+
+func TestHealthzNoServicesDefault(t *testing.T) {
+ p := &Probe{}
+
+ req := httptest.NewRequest("GET", "http://example.com/healthz", nil)
+ w := httptest.NewRecorder()
+ p.healthzFunc(w, req)
+ resp := w.Result()
+ assert.Equal(t, http.StatusTeapot, resp.StatusCode, "invalid status code")
+}
+
+func TestHealthzWithServicesDefault(t *testing.T) {
+ p := &Probe{}
+ p.RegisterService("one", "two")
+ p.UpdateStatus("one", ServiceStatusRunning)
+ p.UpdateStatus("two", ServiceStatusRunning)
+
+ req := httptest.NewRequest("GET", "http://example.com/healthz", nil)
+ w := httptest.NewRecorder()
+ p.healthzFunc(w, req)
+ resp := w.Result()
+ assert.Equal(t, http.StatusOK, resp.StatusCode, "invalid status code")
+}
+
+func TestHealthzWithServicesDefaultFailed(t *testing.T) {
+ p := &Probe{}
+ p.RegisterService("one", "two")
+ p.UpdateStatus("one", ServiceStatusFailed)
+
+ req := httptest.NewRequest("GET", "http://example.com/healthz", nil)
+ w := httptest.NewRecorder()
+ p.healthzFunc(w, req)
+ resp := w.Result()
+ assert.Equal(t, http.StatusTeapot, resp.StatusCode, "invalid status code")
+}
+
+func TestSetFuncsToNil(t *testing.T) {
+ p := (&Probe{}).WithReadyFunc(AlwaysTrue).WithHealthFunc(AlwaysFalse)
+ p.WithReadyFunc(nil).WithHealthFunc(nil)
+ assert.Nil(t, p.readyFunc, "ready func not reset to nil")
+ assert.Nil(t, p.healthFunc, "health func not reset to nil")
+}
+
+func TestUpdateStatusFromContext(t *testing.T) {
+ p := &Probe{}
+ p.RegisterService("one")
+ ctx := context.WithValue(context.Background(), ProbeContextKey, p)
+ UpdateStatusFromContext(ctx, "one", ServiceStatusRunning)
+
+ assert.Equal(t, 1, len(p.status), "wrong number of services")
+ _, ok := p.status["one"]
+ assert.True(t, ok, "unable to find registered service")
+ assert.Equal(t, ServiceStatusRunning, p.status["one"], "status not set correctly from context")
+
+}
+
+func TestUpdateStatusFromNilContext(t *testing.T) {
+ p := &Probe{}
+ p.RegisterService("one")
+ UpdateStatusFromContext(nil, "one", ServiceStatusRunning)
+
+ assert.Equal(t, 1, len(p.status), "wrong number of services")
+ _, ok := p.status["one"]
+ assert.True(t, ok, "unable to find registered service")
+ assert.Equal(t, ServiceStatusUnknown, p.status["one"], "status not set correctly from context")
+
+}
+
+func TestUpdateStatusFromContextWithoutProbe(t *testing.T) {
+ p := &Probe{}
+ p.RegisterService("one")
+ ctx := context.Background()
+ UpdateStatusFromContext(ctx, "one", ServiceStatusRunning)
+
+ assert.Equal(t, 1, len(p.status), "wrong number of services")
+ _, ok := p.status["one"]
+ assert.True(t, ok, "unable to find registered service")
+ assert.Equal(t, ServiceStatusUnknown, p.status["one"], "status not set correctly from context")
+
+}
+
+func TestUpdateStatusFromContextWrongType(t *testing.T) {
+ p := &Probe{}
+ p.RegisterService("one")
+ ctx := context.WithValue(context.Background(), ProbeContextKey, "Teapot")
+ UpdateStatusFromContext(ctx, "one", ServiceStatusRunning)
+
+ assert.Equal(t, 1, len(p.status), "wrong number of services")
+ _, ok := p.status["one"]
+ assert.True(t, ok, "unable to find registered service")
+ assert.Equal(t, ServiceStatusUnknown, p.status["one"], "status not set correctly from context")
+}
+
+func TestUpdateStatusNoRegistered(t *testing.T) {
+ p := (&Probe{}).WithReadyFunc(AlwaysTrue).WithHealthFunc(AlwaysFalse)
+
+ p.UpdateStatus("one", ServiceStatusRunning)
+ assert.Equal(t, 1, len(p.status), "wrong number of services")
+ _, ok := p.status["one"]
+ assert.True(t, ok, "unable to find registered service")
+ assert.Equal(t, ServiceStatusRunning, p.status["one"], "status not set correctly from context")
+}
diff --git a/rw_core/config/config.go b/rw_core/config/config.go
index 133b1a4..f9f1d3e 100644
--- a/rw_core/config/config.go
+++ b/rw_core/config/config.go
@@ -54,6 +54,7 @@
default_CorePairTopic = "rwcore_1"
default_MaxConnectionRetries = -1 // retries forever
default_ConnectionRetryInterval = 2 // in seconds
+ default_ProbePort = 8080
)
// RWCoreFlags represents the set of configurations used by the read-write core service
@@ -88,6 +89,7 @@
CorePairTopic string
MaxConnectionRetries int
ConnectionRetryInterval int
+ ProbePort int
}
func init() {
@@ -126,6 +128,7 @@
CorePairTopic: default_CorePairTopic,
MaxConnectionRetries: default_MaxConnectionRetries,
ConnectionRetryInterval: default_ConnectionRetryInterval,
+ ProbePort: default_ProbePort,
}
return &rwCoreFlag
}
@@ -213,5 +216,8 @@
help = fmt.Sprintf("The number of seconds between each connection retry attempt ")
flag.IntVar(&(cf.ConnectionRetryInterval), "connection_retry_interval", default_ConnectionRetryInterval, help)
+ help = fmt.Sprintf("The port on which to listen to answer liveness and readiness probe queries over HTTP.")
+ flag.IntVar(&(cf.ProbePort), "probe_port", default_ProbePort, help)
+
flag.Parse()
}
diff --git a/rw_core/core/adapter_manager.go b/rw_core/core/adapter_manager.go
index 0ce1828..188ae3d 100644
--- a/rw_core/core/adapter_manager.go
+++ b/rw_core/core/adapter_manager.go
@@ -21,6 +21,7 @@
"fmt"
"github.com/gogo/protobuf/proto"
"github.com/opencord/voltha-go/common/log"
+ "github.com/opencord/voltha-go/common/probe"
"github.com/opencord/voltha-go/db/model"
"github.com/opencord/voltha-protos/go/voltha"
"reflect"
@@ -127,13 +128,14 @@
// Register the callbacks
aMgr.adapterProxy.RegisterCallback(model.POST_UPDATE, aMgr.adapterUpdated)
aMgr.deviceTypeProxy.RegisterCallback(model.POST_UPDATE, aMgr.deviceTypesUpdated)
-
+ probe.UpdateStatusFromContext(ctx, "adapter-manager", probe.ServiceStatusRunning)
log.Info("adapter-manager-started")
}
func (aMgr *AdapterManager) stop(ctx context.Context) {
log.Info("stopping-device-manager")
aMgr.exitChannel <- 1
+ probe.UpdateStatusFromContext(ctx, "adapter-manager", probe.ServiceStatusStopped)
log.Info("device-manager-stopped")
}
diff --git a/rw_core/core/core.go b/rw_core/core/core.go
index 4ddea05..e74e869 100644
--- a/rw_core/core/core.go
+++ b/rw_core/core/core.go
@@ -19,6 +19,7 @@
"context"
grpcserver "github.com/opencord/voltha-go/common/grpc"
"github.com/opencord/voltha-go/common/log"
+ "github.com/opencord/voltha-go/common/probe"
"github.com/opencord/voltha-go/db/kvstore"
"github.com/opencord/voltha-go/db/model"
"github.com/opencord/voltha-go/kafka"
@@ -77,16 +78,39 @@
}
func (core *Core) Start(ctx context.Context) {
+
+ // If the context has a probe then fetch it and register our services
+ var p *probe.Probe
+ if value := ctx.Value(probe.ProbeContextKey); value != nil {
+ if _, ok := value.(*probe.Probe); ok {
+ p = value.(*probe.Probe)
+ p.RegisterService(
+ "message-bus",
+ "kv-store",
+ "device-manager",
+ "logical-device-manager",
+ "adapter-manager",
+ "grpc-service",
+ )
+ }
+ }
+
log.Info("starting-core-services", log.Fields{"coreId": core.instanceId})
// Wait until connection to KV Store is up
if err := core.waitUntilKVStoreReachableOrMaxTries(ctx, core.config.MaxConnectionRetries, core.config.ConnectionRetryInterval); err != nil {
log.Fatal("Unable-to-connect-to-KV-store")
}
+ if p != nil {
+ p.UpdateStatus("kv-store", probe.ServiceStatusRunning)
+ }
if err := core.waitUntilKafkaMessagingProxyIsUpOrMaxTries(ctx, core.config.MaxConnectionRetries, core.config.ConnectionRetryInterval); err != nil {
log.Fatal("Failure-starting-kafkaMessagingProxy")
}
+ if p != nil {
+ p.UpdateStatus("message-bus", probe.ServiceStatusRunning)
+ }
log.Debugw("values", log.Fields{"kmp": core.kmp})
core.deviceMgr = newDeviceManager(core)
@@ -141,9 +165,21 @@
core.grpcServer.AddService(f)
log.Info("grpc-service-added")
- // Start the server
- core.grpcServer.Start(context.Background())
+ /*
+ * Start the GRPC server
+ *
+ * This is a bit sub-optimal here as the grpcServer.Start call does not return (blocks)
+ * until something fails, but we want to send a "start" status update. As written this
+ * means that we are actually sending the "start" status update before the server is
+ * started, which means it is possible that the status is "running" before it actually is.
+ *
+ * This means that there is a small window in which the core could return its status as
+ * ready, when it really isn't.
+ */
+ probe.UpdateStatusFromContext(ctx, "grpc-service", probe.ServiceStatusRunning)
log.Info("grpc-server-started")
+ core.grpcServer.Start(context.Background())
+ probe.UpdateStatusFromContext(ctx, "grpc-service", probe.ServiceStatusStopped)
}
func (core *Core) waitUntilKafkaMessagingProxyIsUpOrMaxTries(ctx context.Context, maxRetries int, retryInterval int) error {
diff --git a/rw_core/core/device_manager.go b/rw_core/core/device_manager.go
index f5b017c..91c359a 100755
--- a/rw_core/core/device_manager.go
+++ b/rw_core/core/device_manager.go
@@ -19,6 +19,7 @@
"context"
"errors"
"github.com/opencord/voltha-go/common/log"
+ "github.com/opencord/voltha-go/common/probe"
"github.com/opencord/voltha-go/db/model"
"github.com/opencord/voltha-go/kafka"
"github.com/opencord/voltha-go/rw_core/utils"
@@ -71,12 +72,14 @@
log.Info("starting-device-manager")
dMgr.logicalDeviceMgr = logicalDeviceMgr
dMgr.stateTransitions = NewTransitionMap(dMgr)
+ probe.UpdateStatusFromContext(ctx, "device-manager", probe.ServiceStatusRunning)
log.Info("device-manager-started")
}
func (dMgr *DeviceManager) stop(ctx context.Context) {
log.Info("stopping-device-manager")
dMgr.exitChannel <- 1
+ probe.UpdateStatusFromContext(ctx, "device-manager", probe.ServiceStatusStopped)
log.Info("device-manager-stopped")
}
diff --git a/rw_core/core/logical_device_manager.go b/rw_core/core/logical_device_manager.go
index 235aca5..fa9713f 100644
--- a/rw_core/core/logical_device_manager.go
+++ b/rw_core/core/logical_device_manager.go
@@ -19,6 +19,7 @@
"context"
"errors"
"github.com/opencord/voltha-go/common/log"
+ "github.com/opencord/voltha-go/common/probe"
"github.com/opencord/voltha-go/db/model"
"github.com/opencord/voltha-go/kafka"
"github.com/opencord/voltha-protos/go/openflow_13"
@@ -62,12 +63,14 @@
func (ldMgr *LogicalDeviceManager) start(ctx context.Context) {
log.Info("starting-logical-device-manager")
+ probe.UpdateStatusFromContext(ctx, "logical-device-manager", probe.ServiceStatusRunning)
log.Info("logical-device-manager-started")
}
func (ldMgr *LogicalDeviceManager) stop(ctx context.Context) {
log.Info("stopping-logical-device-manager")
ldMgr.exitChannel <- 1
+ probe.UpdateStatusFromContext(ctx, "logical-device-manager", probe.ServiceStatusStopped)
log.Info("logical-device-manager-stopped")
}
diff --git a/rw_core/main.go b/rw_core/main.go
index 9ace2fb..6f06576 100644
--- a/rw_core/main.go
+++ b/rw_core/main.go
@@ -21,6 +21,7 @@
"fmt"
grpcserver "github.com/opencord/voltha-go/common/grpc"
"github.com/opencord/voltha-go/common/log"
+ "github.com/opencord/voltha-go/common/probe"
"github.com/opencord/voltha-go/common/version"
"github.com/opencord/voltha-go/db/kvstore"
"github.com/opencord/voltha-go/kafka"
@@ -149,7 +150,7 @@
rw.core.Start(ctx)
}
-func (rw *rwCore) stop() {
+func (rw *rwCore) stop(ctx context.Context) {
// Stop leadership tracking
rw.halted = true
@@ -166,7 +167,7 @@
rw.kvClient.Close()
}
- rw.core.Stop(nil)
+ rw.core.Stop(ctx)
//if rw.kafkaClient != nil {
// rw.kafkaClient.Stop()
@@ -262,17 +263,32 @@
log.Infow("rw-core-config", log.Fields{"config": *cf})
+ // Create the core
+ rw := newRWCore(cf)
+
+ // Create a context adding the status update channel
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
- rw := newRWCore(cf)
- go rw.start(ctx, instanceId)
+ /*
+ * Create and start the liveness and readiness container management probes. This
+ * is done in the main function so just in case the main starts multiple other
+ * objects there can be a single probe end point for the process.
+ */
+ p := &probe.Probe{}
+ go p.ListenAndServe(rw.config.ProbePort)
+
+ // Add the probe to the context to pass to all the services started
+ probeCtx := context.WithValue(ctx, probe.ProbeContextKey, p)
+
+ // Start the core
+ go rw.start(probeCtx, instanceId)
code := waitForExit()
log.Infow("received-a-closing-signal", log.Fields{"code": code})
// Cleanup before leaving
- rw.stop()
+ rw.stop(probeCtx)
elapsed := time.Since(start)
log.Infow("rw-core-run-time", log.Fields{"core": instanceId, "time": elapsed / time.Second})