[VOL-4442] grpc streaming connection monitoring
Change-Id: I6b26a29c74be8833e7262eb59d266e6cce66f0c3
diff --git a/rw_core/mocks/adapter.go b/rw_core/mocks/adapter.go
index b99ca52..5a9130e 100644
--- a/rw_core/mocks/adapter.go
+++ b/rw_core/mocks/adapter.go
@@ -169,12 +169,11 @@
ta.Probe.UpdateStatus(ctx, serviceName, probe.ServiceStatusStopped)
}
-func setAndTestCoreServiceHandler(ctx context.Context, conn *grpc.ClientConn, clientConn *common.Connection) interface{} {
- svc := core_service.NewCoreServiceClient(conn)
- if h, err := svc.GetHealthStatus(ctx, clientConn); err != nil || h.State != health.HealthStatus_HEALTHY {
+func setCoreServiceHandler(ctx context.Context, conn *grpc.ClientConn) interface{} {
+ if conn == nil {
return nil
}
- return svc
+ return core_service.NewCoreServiceClient(conn)
}
// gRPC service
diff --git a/rw_core/mocks/adapter_olt.go b/rw_core/mocks/adapter_olt.go
index 96dfb33..0e8c58e 100644
--- a/rw_core/mocks/adapter_olt.go
+++ b/rw_core/mocks/adapter_olt.go
@@ -22,11 +22,14 @@
"fmt"
"strconv"
"strings"
+ "time"
"github.com/golang/protobuf/ptypes/empty"
"github.com/opencord/voltha-lib-go/v7/pkg/probe"
+ "github.com/opencord/voltha-protos/v5/go/adapter_service"
"github.com/opencord/voltha-protos/v5/go/common"
"github.com/opencord/voltha-protos/v5/go/extension"
+ "github.com/opencord/voltha-protos/v5/go/health"
"github.com/opencord/voltha-protos/v5/go/omci"
"github.com/phayes/freeport"
@@ -92,13 +95,14 @@
// Establish grpc connection to Core
if oltA.coreClient, err = vgrpc.NewClient(
- "olt-endpoint",
+ "mock-olt-endpoint",
oltA.coreEnpoint,
+ "core_service.CoreService",
oltA.oltRestarted); err != nil {
logger.Fatal(ctx, "grpc-client-not-created")
}
- go oltA.coreClient.Start(probeCtx, setAndTestCoreServiceHandler)
+ go oltA.coreClient.Start(probeCtx, setCoreServiceHandler)
logger.Debugw(ctx, "OLTAdapter-started", log.Fields{"grpc-address": oltA.serviceEndpoint})
@@ -431,3 +435,40 @@
}
}
+
+func (oltA *OLTAdapter) GetHealthStatus(stream adapter_service.AdapterService_GetHealthStatusServer) error {
+ ctx := context.Background()
+ logger.Debugw(ctx, "receive-stream-connection", log.Fields{"stream": stream})
+
+ if stream == nil {
+ return fmt.Errorf("conn-is-nil %v", stream)
+ }
+ initialRequestTime := time.Now()
+ var remoteClient *common.Connection
+ var tempClient *common.Connection
+ var err error
+loop:
+ for {
+ tempClient, err = stream.Recv()
+ if err != nil {
+ logger.Warnw(ctx, "received-stream-error", log.Fields{"remote-client": remoteClient, "error": err})
+ break loop
+ }
+ err = stream.Send(&health.HealthStatus{State: health.HealthStatus_HEALTHY})
+ if err != nil {
+ logger.Warnw(ctx, "sending-stream-error", log.Fields{"remote-client": remoteClient, "error": err})
+ break loop
+ }
+ remoteClient = tempClient
+ logger.Debugw(ctx, "received-keep-alive", log.Fields{"remote-client": remoteClient})
+
+ select {
+ case <-stream.Context().Done():
+ logger.Infow(ctx, "stream-keep-alive-context-done", log.Fields{"remote-client": remoteClient, "error": stream.Context().Err()})
+ break loop
+ default:
+ }
+ }
+ logger.Errorw(ctx, "connection-down", log.Fields{"remote-client": remoteClient, "error": err, "initial-conn-time": initialRequestTime})
+ return err
+}
diff --git a/rw_core/mocks/adapter_onu.go b/rw_core/mocks/adapter_onu.go
index d5669c3..dcfc60c 100644
--- a/rw_core/mocks/adapter_onu.go
+++ b/rw_core/mocks/adapter_onu.go
@@ -21,13 +21,16 @@
"fmt"
"strconv"
"strings"
+ "time"
"github.com/golang/protobuf/ptypes/empty"
vgrpc "github.com/opencord/voltha-lib-go/v7/pkg/grpc"
"github.com/opencord/voltha-lib-go/v7/pkg/probe"
+ "github.com/opencord/voltha-protos/v5/go/adapter_service"
"github.com/opencord/voltha-protos/v5/go/common"
ca "github.com/opencord/voltha-protos/v5/go/core_adapter"
"github.com/opencord/voltha-protos/v5/go/extension"
+ "github.com/opencord/voltha-protos/v5/go/health"
"github.com/phayes/freeport"
"github.com/gogo/protobuf/proto"
@@ -87,12 +90,13 @@
// Establish grpc connection to Core
if onuA.coreClient, err = vgrpc.NewClient(
- "onu-endpoint",
+ "mock-onu-endpoint",
onuA.coreEnpoint,
+ "core_service.CoreService",
onuA.onuRestarted); err != nil {
logger.Fatal(ctx, "grpc-client-not-created")
}
- go onuA.coreClient.Start(probeCtx, setAndTestCoreServiceHandler)
+ go onuA.coreClient.Start(probeCtx, setCoreServiceHandler)
logger.Debugw(ctx, "ONUAdapter-started", log.Fields{"grpc-address": onuA.serviceEndpoint})
}
@@ -297,3 +301,40 @@
func (onuA *ONUAdapter) StartOmciTest(ctx context.Context, _ *ca.OMCITest) (*omci.TestResponse, error) { // nolint
return &omci.TestResponse{Result: omci.TestResponse_SUCCESS}, nil
}
+
+func (onuA *ONUAdapter) GetHealthStatus(stream adapter_service.AdapterService_GetHealthStatusServer) error {
+ ctx := context.Background()
+ logger.Debugw(ctx, "receive-stream-connection", log.Fields{"stream": stream})
+
+ if stream == nil {
+ return fmt.Errorf("conn-is-nil %v", stream)
+ }
+ initialRequestTime := time.Now()
+ var remoteClient *common.Connection
+ var tempClient *common.Connection
+ var err error
+loop:
+ for {
+ tempClient, err = stream.Recv()
+ if err != nil {
+ logger.Warnw(ctx, "received-stream-error", log.Fields{"remote-client": remoteClient, "error": err})
+ break loop
+ }
+ err = stream.Send(&health.HealthStatus{State: health.HealthStatus_HEALTHY})
+ if err != nil {
+ logger.Warnw(ctx, "sending-stream-error", log.Fields{"remote-client": remoteClient, "error": err})
+ break loop
+ }
+ remoteClient = tempClient
+ logger.Debugw(ctx, "received-keep-alive", log.Fields{"remote-client": remoteClient})
+
+ select {
+ case <-stream.Context().Done():
+ logger.Infow(ctx, "stream-keep-alive-context-done", log.Fields{"remote-client": remoteClient, "error": stream.Context().Err()})
+ break loop
+ default:
+ }
+ }
+ logger.Errorw(ctx, "connection-down", log.Fields{"remote-client": remoteClient, "error": err, "initial-conn-time": initialRequestTime})
+ return err
+}