[VOL-1743] Create Kubernetes Probes for Open OLT Adapter
Change-Id: I93da6287f51cecab8b7145bd1786e9447617216b
diff --git a/config/config.go b/config/config.go
index ad81762..ae5c0c6 100644
--- a/config/config.go
+++ b/config/config.go
@@ -43,6 +43,8 @@
defaultCoretopic = "rwcore"
defaultEventtopic = "voltha.events"
defaultOnunumber = 1
+ defaultProbeHost = ""
+ defaultProbePort = 8080
)
// AdapterFlags represents the set of configurations used by the read-write adaptercore service
@@ -64,6 +66,8 @@
OnuNumber int
Banner bool
DisplayVersionOnly bool
+ ProbeHost string
+ ProbePort int
}
func init() {
@@ -89,6 +93,8 @@
OnuNumber: defaultOnunumber,
Banner: defaultBanner,
DisplayVersionOnly: defaultDisplayVersionOnly,
+ ProbeHost: defaultProbeHost,
+ ProbePort: defaultProbePort,
}
return &adapterFlags
}
@@ -141,6 +147,12 @@
help = fmt.Sprintf("Show version information and exit")
flag.BoolVar(&(so.DisplayVersionOnly), "version", defaultDisplayVersionOnly, help)
+ help = fmt.Sprintf("The address on which to listen to answer liveness and readiness probe queries over HTTP.")
+ flag.StringVar(&(so.ProbeHost), "probe_host", defaultProbeHost, help)
+
+ help = fmt.Sprintf("The port on which to listen to answer liveness and readiness probe queries over HTTP.")
+ flag.IntVar(&(so.ProbePort), "probe_port", defaultProbePort, help)
+
flag.Parse()
containerName := getContainerInfo()
diff --git a/go.mod b/go.mod
index a80c8b3..91d0a3b 100644
--- a/go.mod
+++ b/go.mod
@@ -5,11 +5,13 @@
require (
github.com/gogo/protobuf v1.3.0
github.com/golang/protobuf v1.3.2
+ github.com/jstemmer/go-junit-report v0.9.1 // indirect
github.com/onsi/ginkgo v1.8.0 // indirect
github.com/onsi/gomega v1.5.0 // indirect
github.com/opencord/voltha-go v0.0.0-20191018170815-cb7c88a83125
github.com/opencord/voltha-lib-go v0.0.0-20191017201200-e73f91e306e9
github.com/opencord/voltha-protos v1.0.3
+ github.com/t-yuki/gocover-cobertura v0.0.0-20180217150009-aaee18c8195c // indirect
go.etcd.io/etcd v0.0.0-20190930204107-236ac2a90522
google.golang.org/grpc v1.24.0
)
diff --git a/go.sum b/go.sum
index b8092ca..c02d9a7 100644
--- a/go.sum
+++ b/go.sum
@@ -147,6 +147,8 @@
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo=
github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o=
+github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
@@ -246,6 +248,8 @@
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/t-yuki/gocover-cobertura v0.0.0-20180217150009-aaee18c8195c h1:+aPplBwWcHBo6q9xrfWdMrT9o4kltkmmvpemgIjep/8=
+github.com/t-yuki/gocover-cobertura v0.0.0-20180217150009-aaee18c8195c/go.mod h1:SbErYREK7xXdsRiigaQiQkI9McGRzYMvlKYaP3Nimdk=
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8 h1:ndzgwNDnKIqyCvHTXaCqh9KlOWKvBry6nuXMJmonVsE=
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
diff --git a/main.go b/main.go
index 760fbe4..5adb596 100644
--- a/main.go
+++ b/main.go
@@ -34,6 +34,7 @@
"github.com/opencord/voltha-lib-go/pkg/db/kvstore"
"github.com/opencord/voltha-lib-go/pkg/kafka"
"github.com/opencord/voltha-lib-go/pkg/log"
+ "github.com/opencord/voltha-lib-go/pkg/probe"
ac "github.com/opencord/voltha-openolt-adapter/adaptercore"
"github.com/opencord/voltha-openolt-adapter/config"
"github.com/opencord/voltha-openolt-adapter/config/version"
@@ -74,19 +75,41 @@
log.Info("Starting Core Adapter components")
var err error
+ var p *probe.Probe
+ if value := ctx.Value(probe.ProbeContextKey); value != nil {
+ if _, ok := value.(*probe.Probe); ok {
+ p = value.(*probe.Probe)
+ p.RegisterService(
+ "message-bus",
+ "kv-store",
+ "container-proxy",
+ "core-request-handler",
+ "register-with-core",
+ )
+ }
+ }
+
// Setup KV Client
log.Debugw("create-kv-client", log.Fields{"kvstore": a.config.KVStoreType})
if err = a.setKVClient(); err != nil {
log.Fatal("error-setting-kv-client")
}
+ if p != nil {
+ p.UpdateStatus("kv-store", probe.ServiceStatusRunning)
+ }
+
// Setup Kafka Client
if a.kafkaClient, err = newKafkaClient("sarama", a.config.KafkaAdapterHost, a.config.KafkaAdapterPort); err != nil {
log.Fatal("Unsupported-common-client")
}
+ if p != nil {
+ p.UpdateStatus("message-bus", probe.ServiceStatusRunning)
+ }
+
// Start the common InterContainer Proxy - retries indefinitely
- if a.kip, err = a.startInterContainerProxy(-1); err != nil {
+ if a.kip, err = a.startInterContainerProxy(ctx, -1); err != nil {
log.Fatal("error-starting-inter-container-proxy")
}
@@ -107,12 +130,12 @@
}
// Register the core request handler
- if err = a.setupRequestHandler(a.instanceID, a.iAdapter); err != nil {
+ if err = a.setupRequestHandler(ctx, a.instanceID, a.iAdapter); err != nil {
log.Fatal("error-setting-core-request-handler")
}
// Register this adapter to the Core - retries indefinitely
- if err = a.registerWithCore(-1); err != nil {
+ if err = a.registerWithCore(ctx, -1); err != nil {
log.Fatal("error-registering-with-core")
}
}
@@ -179,7 +202,7 @@
return nil
}
-func (a *adapter) startInterContainerProxy(retries int) (*kafka.InterContainerProxy, error) {
+func (a *adapter) startInterContainerProxy(ctx context.Context, retries int) (*kafka.InterContainerProxy, error) {
log.Infow("starting-intercontainer-messaging-proxy", log.Fields{"host": a.config.KafkaAdapterHost,
"port": a.config.KafkaAdapterPort, "topic": a.config.Topic})
var err error
@@ -206,7 +229,7 @@
break
}
}
-
+ probe.UpdateStatusFromContext(ctx, "container-proxy", probe.ServiceStatusRunning)
log.Info("common-messaging-proxy-created")
return kip, nil
}
@@ -227,7 +250,7 @@
return sOLT, nil
}
-func (a *adapter) setupRequestHandler(coreInstanceID string, iadapter adapters.IAdapter) error {
+func (a *adapter) setupRequestHandler(ctx context.Context, coreInstanceID string, iadapter adapters.IAdapter) error {
log.Info("setting-request-handler")
requestProxy := com.NewRequestHandlerProxy(coreInstanceID, iadapter, a.coreProxy)
if err := a.kip.SubscribeWithRequestHandlerInterface(kafka.Topic{Name: a.config.Topic}, requestProxy); err != nil {
@@ -235,11 +258,12 @@
return err
}
+ probe.UpdateStatusFromContext(ctx, "core-request-handler", probe.ServiceStatusRunning)
log.Info("request-handler-setup-done")
return nil
}
-func (a *adapter) registerWithCore(retries int) error {
+func (a *adapter) registerWithCore(ctx context.Context, retries int) error {
log.Info("registering-with-core")
adapterDescription := &voltha.Adapter{Id: "openolt", // Unique name for the device type
Vendor: "VOLTHA OpenOLT",
@@ -263,6 +287,7 @@
break
}
}
+ probe.UpdateStatusFromContext(ctx, "register-with-core", probe.ServiceStatusRunning)
log.Info("registered-with-core")
return nil
}
@@ -352,7 +377,13 @@
defer cancel()
ad := newAdapter(cf)
- go ad.start(ctx)
+
+ p := &probe.Probe{}
+ go p.ListenAndServe(fmt.Sprintf("%s:%d", ad.config.ProbeHost, ad.config.ProbePort))
+
+ probeCtx := context.WithValue(ctx, probe.ProbeContextKey, p)
+
+ go ad.start(probeCtx)
code := waitForExit()
log.Infow("received-a-closing-signal", log.Fields{"code": code})
diff --git a/main_test.go b/main_test.go
index 8e74cc8..05ee582 100644
--- a/main_test.go
+++ b/main_test.go
@@ -83,7 +83,8 @@
func Test_registerWithCore(t *testing.T) {
ad := newMockAdapter()
- err := ad.registerWithCore(1)
+ ctx := context.TODO()
+ err := ad.registerWithCore(ctx, 1)
if err != nil {
t.Errorf("Expected error:nil, got error: %v", err)
}
@@ -92,7 +93,8 @@
ad := newMockAdapter()
kc := &mockKafkaClient{}
ad.kafkaClient = kc
- icp, err := ad.startInterContainerProxy(1)
+ ctx := context.TODO()
+ icp, err := ad.startInterContainerProxy(ctx, 1)
if icp != nil {
t.Log("Intercontainer proxy ", icp)
}
@@ -167,7 +169,8 @@
ad.config.KVStorePort, ad.config.KVStoreType)
printBanner()
printVersion()
- if err := ad.setupRequestHandler(ad.config.InstanceID, oolt); err != nil {
+ ctx := context.TODO()
+ if err := ad.setupRequestHandler(ctx, ad.config.InstanceID, oolt); err != nil {
t.Logf("adapter.setupRequestHandler() error = %v", err)
}
diff --git a/vendor/github.com/opencord/voltha-lib-go/pkg/probe/probe.go b/vendor/github.com/opencord/voltha-lib-go/pkg/probe/probe.go
new file mode 100644
index 0000000..192c7ca
--- /dev/null
+++ b/vendor/github.com/opencord/voltha-lib-go/pkg/probe/probe.go
@@ -0,0 +1,253 @@
+/*
+ * Copyright 2019-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package probe
+
+import (
+ "context"
+ "fmt"
+ "github.com/opencord/voltha-lib-go/pkg/log"
+ "net/http"
+ "sync"
+)
+
+// ProbeContextKey used to fetch the Probe instance from a context
+type ProbeContextKeyType string
+
+// ServiceStatus typed values for service status
+type ServiceStatus int
+
+const (
+ // ServiceStatusUnknown initial state of services
+ ServiceStatusUnknown ServiceStatus = iota
+
+ // ServiceStatusPreparing to optionally be used for prep, such as connecting
+ ServiceStatusPreparing
+
+ // ServiceStatusPrepared to optionally be used when prep is complete, but before run
+ ServiceStatusPrepared
+
+ // ServiceStatusRunning service is functional
+ ServiceStatusRunning
+
+ // ServiceStatusStopped service has stopped, but not because of error
+ ServiceStatusStopped
+
+ // ServiceStatusFailed service has stopped because of an error
+ ServiceStatusFailed
+)
+
+const (
+ // ProbeContextKey value of context key to fetch probe
+ ProbeContextKey = ProbeContextKeyType("status-update-probe")
+)
+
+// String convert ServiceStatus values to strings
+func (s ServiceStatus) String() string {
+ switch s {
+ default:
+ fallthrough
+ case ServiceStatusUnknown:
+ return "Unknown"
+ case ServiceStatusPreparing:
+ return "Preparing"
+ case ServiceStatusPrepared:
+ return "Prepared"
+ case ServiceStatusRunning:
+ return "Running"
+ case ServiceStatusStopped:
+ return "Stopped"
+ case ServiceStatusFailed:
+ return "Failed"
+ }
+}
+
+// ServiceStatusUpdate status update event
+type ServiceStatusUpdate struct {
+ Name string
+ Status ServiceStatus
+}
+
+// Probe reciever on which to implement probe capabilities
+type Probe struct {
+ readyFunc func(map[string]ServiceStatus) bool
+ healthFunc func(map[string]ServiceStatus) bool
+
+ mutex sync.RWMutex
+ status map[string]ServiceStatus
+ isReady bool
+ isHealthy bool
+}
+
+// WithReadyFunc override the default ready calculation function
+func (p *Probe) WithReadyFunc(readyFunc func(map[string]ServiceStatus) bool) *Probe {
+ p.readyFunc = readyFunc
+ return p
+}
+
+// WithHealthFunc override the default health calculation function
+func (p *Probe) WithHealthFunc(healthFunc func(map[string]ServiceStatus) bool) *Probe {
+ p.healthFunc = healthFunc
+ return p
+}
+
+// RegisterService register one or more service names with the probe, status will be track against service name
+func (p *Probe) RegisterService(names ...string) {
+ p.mutex.Lock()
+ defer p.mutex.Unlock()
+ if p.status == nil {
+ p.status = make(map[string]ServiceStatus)
+ }
+ for _, name := range names {
+ if _, ok := p.status[name]; !ok {
+ p.status[name] = ServiceStatusUnknown
+ log.Debugw("probe-service-registered", log.Fields{"service-name": name})
+ }
+ }
+
+ if p.readyFunc != nil {
+ p.isReady = p.readyFunc(p.status)
+ } else {
+ p.isReady = defaultReadyFunc(p.status)
+ }
+
+ if p.healthFunc != nil {
+ p.isHealthy = p.healthFunc(p.status)
+ } else {
+ p.isHealthy = defaultHealthFunc(p.status)
+ }
+}
+
+// UpdateStatus utility function to send a service update to the probe
+func (p *Probe) UpdateStatus(name string, status ServiceStatus) {
+ p.mutex.Lock()
+ defer p.mutex.Unlock()
+ if p.status == nil {
+ p.status = make(map[string]ServiceStatus)
+ }
+ p.status[name] = status
+ if p.readyFunc != nil {
+ p.isReady = p.readyFunc(p.status)
+ } else {
+ p.isReady = defaultReadyFunc(p.status)
+ }
+
+ if p.healthFunc != nil {
+ p.isHealthy = p.healthFunc(p.status)
+ } else {
+ p.isHealthy = defaultHealthFunc(p.status)
+ }
+ log.Debugw("probe-service-status-updated",
+ log.Fields{
+ "service-name": name,
+ "status": status.String(),
+ "ready": p.isReady,
+ "health": p.isHealthy,
+ })
+}
+
+// UpdateStatusFromContext a convenience function to pull the Probe reference from the
+// Context, if it exists, and then calling UpdateStatus on that Probe reference. If Context
+// is nil or if a Probe reference is not associated with the ProbeContextKey then nothing
+// happens
+func UpdateStatusFromContext(ctx context.Context, name string, status ServiceStatus) {
+ if ctx != nil {
+ if value := ctx.Value(ProbeContextKey); value != nil {
+ if p, ok := value.(*Probe); ok {
+ p.UpdateStatus(name, status)
+ }
+ }
+ }
+}
+
+// pulled out to a function to help better enable unit testing
+func (p *Probe) readzFunc(w http.ResponseWriter, req *http.Request) {
+ p.mutex.RLock()
+ defer p.mutex.RUnlock()
+ if p.isReady {
+ w.WriteHeader(http.StatusOK)
+ } else {
+ w.WriteHeader(http.StatusTeapot)
+ }
+}
+func (p *Probe) healthzFunc(w http.ResponseWriter, req *http.Request) {
+ p.mutex.RLock()
+ defer p.mutex.RUnlock()
+ if p.isHealthy {
+ w.WriteHeader(http.StatusOK)
+ } else {
+ w.WriteHeader(http.StatusTeapot)
+ }
+}
+func (p *Probe) detailzFunc(w http.ResponseWriter, req *http.Request) {
+ p.mutex.RLock()
+ defer p.mutex.RUnlock()
+ w.Header().Set("Content-Type", "application/json")
+ w.Write([]byte("{"))
+ comma := ""
+ for c, s := range p.status {
+ w.Write([]byte(fmt.Sprintf("%s\"%s\": \"%s\"", comma, c, s.String())))
+ comma = ", "
+ }
+ w.Write([]byte("}"))
+ w.WriteHeader(http.StatusOK)
+
+}
+
+// ListenAndServe implements 3 HTTP endpoints on the given port for healthz, readz, and detailz. Returns only on error
+func (p *Probe) ListenAndServe(address string) {
+ mux := http.NewServeMux()
+
+ // Returns the result of the readyFunc calculation
+ mux.HandleFunc("/readz", p.readzFunc)
+
+ // Returns the result of the healthFunc calculation
+ mux.HandleFunc("/healthz", p.healthzFunc)
+
+ // Returns the details of the services and their status as JSON
+ mux.HandleFunc("/detailz", p.detailzFunc)
+ s := &http.Server{
+ Addr: address,
+ Handler: mux,
+ }
+ log.Fatal(s.ListenAndServe())
+}
+
+// defaultReadyFunc if all services are running then ready, else not
+func defaultReadyFunc(services map[string]ServiceStatus) bool {
+ if len(services) == 0 {
+ return false
+ }
+ for _, status := range services {
+ if status != ServiceStatusRunning {
+ return false
+ }
+ }
+ return true
+}
+
+// defaultHealthFunc if no service is stopped or failed, then healthy, else not.
+// service is start as unknown, so they are considered healthy
+func defaultHealthFunc(services map[string]ServiceStatus) bool {
+ if len(services) == 0 {
+ return false
+ }
+ for _, status := range services {
+ if status == ServiceStatusStopped || status == ServiceStatusFailed {
+ return false
+ }
+ }
+ return true
+}
diff --git a/vendor/modules.txt b/vendor/modules.txt
index 50dc809..2ceee28 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -69,6 +69,7 @@
github.com/opencord/voltha-lib-go/pkg/db/kvstore
github.com/opencord/voltha-lib-go/pkg/kafka
github.com/opencord/voltha-lib-go/pkg/log
+github.com/opencord/voltha-lib-go/pkg/probe
github.com/opencord/voltha-lib-go/pkg/techprofile
github.com/opencord/voltha-lib-go/pkg/db/model
github.com/opencord/voltha-lib-go/pkg/ponresourcemanager