Create Kubernetes Probes for API Server[VOL-1731]
Change-Id: Ie289ffce77af284f2b8f62603f8570d80a598e50
diff --git a/arouter.json b/arouter.json
index 57e7715..26c3536 100644
--- a/arouter.json
+++ b/arouter.json
@@ -287,6 +287,8 @@
"api": {
"_comment": "If this isn't defined then no api is available for dynamic configuration and queries",
"address": "localhost",
- "port": 55554
+ "port": 55554,
+ "probeHost": "localhost",
+ "probePort": 8080
}
}
diff --git a/cmd/arouter/arouter.go b/cmd/arouter/arouter.go
index 6fd9dbf..73f8782 100644
--- a/cmd/arouter/arouter.go
+++ b/cmd/arouter/arouter.go
@@ -20,6 +20,7 @@
"fmt"
"github.com/opencord/voltha-api-server/internal/pkg/afrouter"
"github.com/opencord/voltha-lib-go/v2/pkg/log"
+ "github.com/opencord/voltha-lib-go/v2/pkg/probe"
"github.com/opencord/voltha-lib-go/v2/pkg/version"
_ "github.com/opencord/voltha-protos/v2"
"google.golang.org/grpc/grpclog"
@@ -70,6 +71,18 @@
grpclog.SetLoggerV2(grpclog.NewLoggerV2(os.Stderr, ioutil.Discard, ioutil.Discard))
}
+ /*
+ * Create and start the liveness and readiness container management probes. This
+ * is done in the main function so just in case the main starts multiple other
+ * objects there can be a single probe end point for the process.
+ */
+ p := &probe.Probe{}
+ go p.ListenAndServe(fmt.Sprintf("%s:%d", conf.Api.ProbeHost, conf.Api.ProbePort))
+
+ p.RegisterService(
+ "affinity-router-proxy",
+ )
+
// Install the signal and error handlers.
err = afrouter.InitExitHandler()
if err != nil {
@@ -78,7 +91,7 @@
}
// Create the affinity router proxy...
- if ap, err := afrouter.NewArouterProxy(conf); err != nil {
+ if ap, err := afrouter.NewArouterProxy(conf, p); err != nil {
log.Errorf("Failed to create the arouter proxy, exiting:%v", err)
return 1
// and start it.
diff --git a/cmd/arouterd/arouterd.go b/cmd/arouterd/arouterd.go
index eedfe5c..5452b97 100644
--- a/cmd/arouterd/arouterd.go
+++ b/cmd/arouterd/arouterd.go
@@ -17,21 +17,25 @@
package main
import (
+ "context"
"flag"
"fmt"
"github.com/opencord/voltha-api-server/internal/pkg/afrouterd"
+ "github.com/opencord/voltha-lib-go/v2/pkg/probe"
"github.com/opencord/voltha-lib-go/v2/pkg/version"
+ "math"
"os"
"path"
"github.com/opencord/voltha-lib-go/v2/pkg/log"
pb "github.com/opencord/voltha-protos/v2/go/afrouter"
- "golang.org/x/net/context"
)
var (
instanceID = afrouterd.GetStrEnv("HOSTNAME", "arouterd001")
afrouterApiAddress = afrouterd.GetStrEnv("AFROUTER_API_ADDRESS", "localhost:55554")
+ probeHost = afrouterd.GetStrEnv("PROBE_HOST", "")
+ probePort = afrouterd.GetIntEnv("PROBE_PORT", 0, math.MaxUint16, 8081)
)
type Configuration struct {
@@ -62,16 +66,26 @@
// Set up kubernetes api
clientset := afrouterd.K8sClientSet()
+ p := &probe.Probe{}
+ go p.ListenAndServe(fmt.Sprintf("%s:%d", probeHost, probePort))
+
+ p.RegisterService(
+ "affinity-router",
+ "message-bus",
+ )
+
for {
// Connect to the affinity router
conn, err := afrouterd.Connect(context.Background(), afrouterApiAddress) // This is a sidecar container so communicating over localhost
if err != nil {
panic(err)
}
+ p.UpdateStatus("affinity-router", probe.ServiceStatusRunning)
// monitor the connection status, end context if connection is lost
- ctx := afrouterd.ConnectionActiveContext(conn)
+ ctx := afrouterd.ConnectionActiveContext(conn, p)
+ probeCtx := context.WithValue(ctx, probe.ProbeContextKey, p)
// set up the client
client := pb.NewConfigurationClient(conn)
@@ -79,10 +93,10 @@
// these two processes do the majority of the work
log.Info("Starting discovery monitoring")
- doneCh, _ := afrouterd.StartDiscoveryMonitor(ctx, client)
+ doneCh, _ := afrouterd.StartDiscoveryMonitor(probeCtx, client)
log.Info("Starting core monitoring")
- afrouterd.CoreMonitor(ctx, client, clientset)
+ afrouterd.CoreMonitor(probeCtx, client, clientset)
//ensure the discovery monitor to quit
<-doneCh
diff --git a/go.sum b/go.sum
index 7780350..da303b0 100644
--- a/go.sum
+++ b/go.sum
@@ -142,6 +142,8 @@
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo=
github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o=
+github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
@@ -233,6 +235,8 @@
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/t-yuki/gocover-cobertura v0.0.0-20180217150009-aaee18c8195c h1:+aPplBwWcHBo6q9xrfWdMrT9o4kltkmmvpemgIjep/8=
+github.com/t-yuki/gocover-cobertura v0.0.0-20180217150009-aaee18c8195c/go.mod h1:SbErYREK7xXdsRiigaQiQkI9McGRzYMvlKYaP3Nimdk=
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
diff --git a/internal/pkg/afrouter/api_test.go b/internal/pkg/afrouter/api_test.go
index 0bcb1d4..fc00a1b 100644
--- a/internal/pkg/afrouter/api_test.go
+++ b/internal/pkg/afrouter/api_test.go
@@ -18,6 +18,7 @@
import (
"fmt"
+ "github.com/opencord/voltha-lib-go/v2/pkg/probe"
"github.com/phayes/freeport"
"github.com/stretchr/testify/assert"
"testing"
@@ -98,9 +99,9 @@
}
func makeProxy(numBackends int, numConnections int) (*ArouterProxy, error) {
-
+ p := &probe.Probe{}
conf := makeConfig(3, 2)
- arouter, err := NewArouterProxy(&conf)
+ arouter, err := NewArouterProxy(&conf, p)
return arouter, err
}
diff --git a/internal/pkg/afrouter/arproxy.go b/internal/pkg/afrouter/arproxy.go
index 80c8611..6d1c747 100644
--- a/internal/pkg/afrouter/arproxy.go
+++ b/internal/pkg/afrouter/arproxy.go
@@ -22,6 +22,7 @@
import (
"github.com/opencord/voltha-lib-go/v2/pkg/log"
+ "github.com/opencord/voltha-lib-go/v2/pkg/probe"
)
// String names for display in error messages.
@@ -33,7 +34,7 @@
}
// Create the routing proxy
-func NewArouterProxy(conf *Configuration) (*ArouterProxy, error) {
+func NewArouterProxy(conf *Configuration, p *probe.Probe) (*ArouterProxy, error) {
arProxy = &ArouterProxy{servers: make(map[string]*server)}
// Create all the servers listed in the configuration
for _, s := range conf.Servers {
@@ -53,6 +54,7 @@
arProxy.api = api
}
+ p.UpdateStatus("affinity-router-proxy", probe.ServiceStatusRunning)
return arProxy, nil
}
diff --git a/internal/pkg/afrouter/config.go b/internal/pkg/afrouter/config.go
index 8fdfd41..3a61e63 100644
--- a/internal/pkg/afrouter/config.go
+++ b/internal/pkg/afrouter/config.go
@@ -44,11 +44,8 @@
err := cmdParse.Parse(os.Args[1:])
if err != nil {
- //return err
return nil, errors.New("Error parsing the command line")
}
- //if(!cmdParse.Parsed()) {
- //}
if val, have := os.LookupEnv("HOSTNAME"); have {
config.InstanceID = val
@@ -179,8 +176,10 @@
// Api configuration
type ApiConfig struct {
- Addr string `json:"address"`
- Port uint `json:"port"`
+ Addr string `json:"address"`
+ Port uint `json:"port"`
+ ProbeHost string `json:"probeHost"`
+ ProbePort int `json:"probePort"`
}
func (conf *Configuration) LoadConfig() error {
diff --git a/internal/pkg/afrouterd/discoveryMonitor.go b/internal/pkg/afrouterd/discoveryMonitor.go
index 679d3f7..2e3e831 100644
--- a/internal/pkg/afrouterd/discoveryMonitor.go
+++ b/internal/pkg/afrouterd/discoveryMonitor.go
@@ -21,6 +21,7 @@
"github.com/golang/protobuf/ptypes"
"github.com/opencord/voltha-lib-go/v2/pkg/kafka"
"github.com/opencord/voltha-lib-go/v2/pkg/log"
+ "github.com/opencord/voltha-lib-go/v2/pkg/probe"
pb "github.com/opencord/voltha-protos/v2/go/afrouter"
ic "github.com/opencord/voltha-protos/v2/go/inter_container"
"golang.org/x/net/context"
@@ -52,6 +53,7 @@
func monitorDiscovery(kc kafka.Client, ctx context.Context, client pb.ConfigurationClient, ch <-chan *ic.InterContainerMessage, doneCh chan<- struct{}) {
defer close(doneCh)
defer kc.Stop()
+ defer probe.UpdateStatusFromContext(ctx, "message-bus", probe.ServiceStatusStopped)
monitorLoop:
for {
@@ -85,15 +87,18 @@
panic(err)
}
+ probe.UpdateStatusFromContext(ctx, "message-bus", probe.ServiceStatusPreparing)
for {
if err := kc.Start(); err != nil {
log.Error("Could not connect to kafka")
} else {
+ probe.UpdateStatusFromContext(ctx, "message-bus", probe.ServiceStatusRunning)
break
}
select {
case <-ctx.Done():
close(doneCh)
+ probe.UpdateStatusFromContext(ctx, "message-bus", probe.ServiceStatusStopped)
return doneCh, errors.New("GRPC context done")
case <-time.After(5 * time.Second):
@@ -104,6 +109,7 @@
log.Errorf("Could not subscribe to the '%s' channel, discovery disabled", kafkaTopic)
close(doneCh)
kc.Stop()
+ probe.UpdateStatusFromContext(ctx, "message-bus", probe.ServiceStatusStopped)
return doneCh, err
}
diff --git a/internal/pkg/afrouterd/misc.go b/internal/pkg/afrouterd/misc.go
index 7946205..1f0b379 100644
--- a/internal/pkg/afrouterd/misc.go
+++ b/internal/pkg/afrouterd/misc.go
@@ -19,6 +19,7 @@
import (
"fmt"
"github.com/opencord/voltha-lib-go/v2/pkg/log"
+ "github.com/opencord/voltha-lib-go/v2/pkg/probe"
pb "github.com/opencord/voltha-protos/v2/go/afrouter"
"golang.org/x/net/context"
"google.golang.org/grpc"
@@ -110,7 +111,7 @@
}
// endOnClose cancels the context when the connection closes
-func ConnectionActiveContext(conn *grpc.ClientConn) context.Context {
+func ConnectionActiveContext(conn *grpc.ClientConn, p *probe.Probe) context.Context {
ctx, disconnected := context.WithCancel(context.Background())
go func() {
for state := conn.GetState(); state != connectivity.TransientFailure && state != connectivity.Shutdown; state = conn.GetState() {
@@ -119,6 +120,7 @@
}
}
log.Infof("Connection to afrouter lost")
+ p.UpdateStatus("affinity-router", probe.ServiceStatusStopped)
disconnected()
}()
return ctx
diff --git a/vendor/github.com/opencord/voltha-lib-go/v2/pkg/probe/probe.go b/vendor/github.com/opencord/voltha-lib-go/v2/pkg/probe/probe.go
new file mode 100644
index 0000000..9823566
--- /dev/null
+++ b/vendor/github.com/opencord/voltha-lib-go/v2/pkg/probe/probe.go
@@ -0,0 +1,253 @@
+/*
+ * Copyright 2019-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package probe
+
+import (
+ "context"
+ "fmt"
+ "github.com/opencord/voltha-lib-go/v2/pkg/log"
+ "net/http"
+ "sync"
+)
+
+// ProbeContextKey used to fetch the Probe instance from a context
+type ProbeContextKeyType string
+
+// ServiceStatus typed values for service status
+type ServiceStatus int
+
+const (
+ // ServiceStatusUnknown initial state of services
+ ServiceStatusUnknown ServiceStatus = iota
+
+ // ServiceStatusPreparing to optionally be used for prep, such as connecting
+ ServiceStatusPreparing
+
+ // ServiceStatusPrepared to optionally be used when prep is complete, but before run
+ ServiceStatusPrepared
+
+ // ServiceStatusRunning service is functional
+ ServiceStatusRunning
+
+ // ServiceStatusStopped service has stopped, but not because of error
+ ServiceStatusStopped
+
+ // ServiceStatusFailed service has stopped because of an error
+ ServiceStatusFailed
+)
+
+const (
+ // ProbeContextKey value of context key to fetch probe
+ ProbeContextKey = ProbeContextKeyType("status-update-probe")
+)
+
+// String convert ServiceStatus values to strings
+func (s ServiceStatus) String() string {
+ switch s {
+ default:
+ fallthrough
+ case ServiceStatusUnknown:
+ return "Unknown"
+ case ServiceStatusPreparing:
+ return "Preparing"
+ case ServiceStatusPrepared:
+ return "Prepared"
+ case ServiceStatusRunning:
+ return "Running"
+ case ServiceStatusStopped:
+ return "Stopped"
+ case ServiceStatusFailed:
+ return "Failed"
+ }
+}
+
+// ServiceStatusUpdate status update event
+type ServiceStatusUpdate struct {
+ Name string
+ Status ServiceStatus
+}
+
+// Probe reciever on which to implement probe capabilities
+type Probe struct {
+ readyFunc func(map[string]ServiceStatus) bool
+ healthFunc func(map[string]ServiceStatus) bool
+
+ mutex sync.RWMutex
+ status map[string]ServiceStatus
+ isReady bool
+ isHealthy bool
+}
+
+// WithReadyFunc override the default ready calculation function
+func (p *Probe) WithReadyFunc(readyFunc func(map[string]ServiceStatus) bool) *Probe {
+ p.readyFunc = readyFunc
+ return p
+}
+
+// WithHealthFunc override the default health calculation function
+func (p *Probe) WithHealthFunc(healthFunc func(map[string]ServiceStatus) bool) *Probe {
+ p.healthFunc = healthFunc
+ return p
+}
+
+// RegisterService register one or more service names with the probe, status will be track against service name
+func (p *Probe) RegisterService(names ...string) {
+ p.mutex.Lock()
+ defer p.mutex.Unlock()
+ if p.status == nil {
+ p.status = make(map[string]ServiceStatus)
+ }
+ for _, name := range names {
+ if _, ok := p.status[name]; !ok {
+ p.status[name] = ServiceStatusUnknown
+ log.Debugw("probe-service-registered", log.Fields{"service-name": name})
+ }
+ }
+
+ if p.readyFunc != nil {
+ p.isReady = p.readyFunc(p.status)
+ } else {
+ p.isReady = defaultReadyFunc(p.status)
+ }
+
+ if p.healthFunc != nil {
+ p.isHealthy = p.healthFunc(p.status)
+ } else {
+ p.isHealthy = defaultHealthFunc(p.status)
+ }
+}
+
+// UpdateStatus utility function to send a service update to the probe
+func (p *Probe) UpdateStatus(name string, status ServiceStatus) {
+ p.mutex.Lock()
+ defer p.mutex.Unlock()
+ if p.status == nil {
+ p.status = make(map[string]ServiceStatus)
+ }
+ p.status[name] = status
+ if p.readyFunc != nil {
+ p.isReady = p.readyFunc(p.status)
+ } else {
+ p.isReady = defaultReadyFunc(p.status)
+ }
+
+ if p.healthFunc != nil {
+ p.isHealthy = p.healthFunc(p.status)
+ } else {
+ p.isHealthy = defaultHealthFunc(p.status)
+ }
+ log.Debugw("probe-service-status-updated",
+ log.Fields{
+ "service-name": name,
+ "status": status.String(),
+ "ready": p.isReady,
+ "health": p.isHealthy,
+ })
+}
+
+// UpdateStatusFromContext a convenience function to pull the Probe reference from the
+// Context, if it exists, and then calling UpdateStatus on that Probe reference. If Context
+// is nil or if a Probe reference is not associated with the ProbeContextKey then nothing
+// happens
+func UpdateStatusFromContext(ctx context.Context, name string, status ServiceStatus) {
+ if ctx != nil {
+ if value := ctx.Value(ProbeContextKey); value != nil {
+ if p, ok := value.(*Probe); ok {
+ p.UpdateStatus(name, status)
+ }
+ }
+ }
+}
+
+// pulled out to a function to help better enable unit testing
+func (p *Probe) readzFunc(w http.ResponseWriter, req *http.Request) {
+ p.mutex.RLock()
+ defer p.mutex.RUnlock()
+ if p.isReady {
+ w.WriteHeader(http.StatusOK)
+ } else {
+ w.WriteHeader(http.StatusTeapot)
+ }
+}
+func (p *Probe) healthzFunc(w http.ResponseWriter, req *http.Request) {
+ p.mutex.RLock()
+ defer p.mutex.RUnlock()
+ if p.isHealthy {
+ w.WriteHeader(http.StatusOK)
+ } else {
+ w.WriteHeader(http.StatusTeapot)
+ }
+}
+func (p *Probe) detailzFunc(w http.ResponseWriter, req *http.Request) {
+ p.mutex.RLock()
+ defer p.mutex.RUnlock()
+ w.Header().Set("Content-Type", "application/json")
+ w.Write([]byte("{"))
+ comma := ""
+ for c, s := range p.status {
+ w.Write([]byte(fmt.Sprintf("%s\"%s\": \"%s\"", comma, c, s.String())))
+ comma = ", "
+ }
+ w.Write([]byte("}"))
+ w.WriteHeader(http.StatusOK)
+
+}
+
+// ListenAndServe implements 3 HTTP endpoints on the given port for healthz, readz, and detailz. Returns only on error
+func (p *Probe) ListenAndServe(address string) {
+ mux := http.NewServeMux()
+
+ // Returns the result of the readyFunc calculation
+ mux.HandleFunc("/readz", p.readzFunc)
+
+ // Returns the result of the healthFunc calculation
+ mux.HandleFunc("/healthz", p.healthzFunc)
+
+ // Returns the details of the services and their status as JSON
+ mux.HandleFunc("/detailz", p.detailzFunc)
+ s := &http.Server{
+ Addr: address,
+ Handler: mux,
+ }
+ log.Fatal(s.ListenAndServe())
+}
+
+// defaultReadyFunc if all services are running then ready, else not
+func defaultReadyFunc(services map[string]ServiceStatus) bool {
+ if len(services) == 0 {
+ return false
+ }
+ for _, status := range services {
+ if status != ServiceStatusRunning {
+ return false
+ }
+ }
+ return true
+}
+
+// defaultHealthFunc if no service is stopped or failed, then healthy, else not.
+// service is start as unknown, so they are considered healthy
+func defaultHealthFunc(services map[string]ServiceStatus) bool {
+ if len(services) == 0 {
+ return false
+ }
+ for _, status := range services {
+ if status == ServiceStatusStopped || status == ServiceStatusFailed {
+ return false
+ }
+ }
+ return true
+}
diff --git a/vendor/modules.txt b/vendor/modules.txt
index 42a8181..a3641ed 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -48,6 +48,7 @@
github.com/modern-go/reflect2
# github.com/opencord/voltha-lib-go/v2 v2.2.9
github.com/opencord/voltha-lib-go/v2/pkg/log
+github.com/opencord/voltha-lib-go/v2/pkg/probe
github.com/opencord/voltha-lib-go/v2/pkg/version
github.com/opencord/voltha-lib-go/v2/pkg/kafka
# github.com/opencord/voltha-protos/v2 v2.0.1