VOL-3734 optimise rpc events to be send to queue and then to kafka from the queue
Change-Id: I5e068722412b6d9526760900d9173aaf51e00946
diff --git a/vendor/github.com/opencord/voltha-lib-go/v4/pkg/events/events_proxy.go b/vendor/github.com/opencord/voltha-lib-go/v4/pkg/events/events_proxy.go
index 2301f43..910fec3 100644
--- a/vendor/github.com/opencord/voltha-lib-go/v4/pkg/events/events_proxy.go
+++ b/vendor/github.com/opencord/voltha-lib-go/v4/pkg/events/events_proxy.go
@@ -17,11 +17,13 @@
package events
import (
+ "container/ring"
"context"
"errors"
"fmt"
"strconv"
"strings"
+ "sync"
"time"
"github.com/golang/protobuf/ptypes"
@@ -31,9 +33,17 @@
"github.com/opencord/voltha-protos/v4/go/voltha"
)
+// TODO: Make configurable through helm chart
+const EVENT_THRESHOLD = 1000
+
+type lastEvent struct{}
+
type EventProxy struct {
- kafkaClient kafka.Client
- eventTopic kafka.Topic
+ kafkaClient kafka.Client
+ eventTopic kafka.Topic
+ eventQueue *EventQueue
+ queueCtx context.Context
+ queueCancelCtx context.CancelFunc
}
func NewEventProxy(opts ...EventProxyOption) *EventProxy {
@@ -41,6 +51,8 @@
for _, option := range opts {
option(&proxy)
}
+ proxy.eventQueue = newEventQueue()
+ proxy.queueCtx, proxy.queueCancelCtx = context.WithCancel(context.Background())
return &proxy
}
@@ -84,8 +96,8 @@
header.Type = eventType
header.TypeVersion = eventif.EventTypeVersion
- // raisedTs is in nanoseconds
- timestamp, err := ptypes.TimestampProto(time.Unix(0, raisedTs))
+ // raisedTs is in seconds
+ timestamp, err := ptypes.TimestampProto(time.Unix(raisedTs, 0))
if err != nil {
return nil, err
}
@@ -112,15 +124,7 @@
return err
}
event.EventType = &voltha.Event_RpcEvent{RpcEvent: rpcEvent}
- if err := ep.sendEvent(ctx, &event); err != nil {
- logger.Errorw(ctx, "Failed to send rpc event to KAFKA bus", log.Fields{"rpc-event": rpcEvent})
- return err
- }
- logger.Debugw(ctx, "Successfully sent RPC event to KAFKA bus", log.Fields{"Id": event.Header.Id, "Category": event.Header.Category,
- "SubCategory": event.Header.SubCategory, "Type": event.Header.Type, "TypeVersion": event.Header.TypeVersion,
- "ReportedTs": event.Header.ReportedTs, "ResourceId": rpcEvent.ResourceId, "Context": rpcEvent.Context,
- "RPCEventName": id})
-
+ ep.eventQueue.push(&event)
return nil
}
@@ -195,3 +199,155 @@
func (ep *EventProxy) SendLiveness(ctx context.Context) error {
return ep.kafkaClient.SendLiveness(ctx)
}
+
+// Start the event proxy
+func (ep *EventProxy) Start() {
+ eq := ep.eventQueue
+ go eq.start(ep.queueCtx)
+ logger.Debugw(context.Background(), "event-proxy-starting...", log.Fields{"events-threashold": EVENT_THRESHOLD})
+ for {
+ // Notify the queue I am ready
+ eq.readyToSendToKafkaCh <- struct{}{}
+ // Wait for an event
+ elem, ok := <-eq.eventChannel
+ if !ok {
+ logger.Debug(context.Background(), "event-channel-closed-exiting")
+ break
+ }
+ // Check for last event
+ if _, ok := elem.(*lastEvent); ok {
+ // close the queuing loop
+ logger.Info(context.Background(), "received-last-event")
+ ep.queueCancelCtx()
+ break
+ }
+ ctx := context.Background()
+ event, ok := elem.(*voltha.Event)
+ if !ok {
+ logger.Warnw(ctx, "invalid-event", log.Fields{"element": elem})
+ continue
+ }
+ if err := ep.sendEvent(ctx, event); err != nil {
+ logger.Errorw(ctx, "failed-to-send-event-to-kafka-bus", log.Fields{"event": event})
+ } else {
+ logger.Debugw(ctx, "successfully-sent-rpc-event-to-kafka-bus", log.Fields{"id": event.Header.Id, "category": event.Header.Category,
+ "sub-category": event.Header.SubCategory, "type": event.Header.Type, "type-version": event.Header.TypeVersion,
+ "reported-ts": event.Header.ReportedTs, "event-type": event.EventType})
+ }
+ }
+}
+
+func (ep *EventProxy) Stop() {
+ ep.eventQueue.stop()
+}
+
+type EventQueue struct {
+ mutex sync.RWMutex
+ eventChannel chan interface{}
+ insertPosition *ring.Ring
+ popPosition *ring.Ring
+ dataToSendAvailable chan struct{}
+ readyToSendToKafkaCh chan struct{}
+ eventQueueStopped chan struct{}
+}
+
+func newEventQueue() *EventQueue {
+ ev := &EventQueue{
+ eventChannel: make(chan interface{}),
+ insertPosition: ring.New(EVENT_THRESHOLD),
+ dataToSendAvailable: make(chan struct{}),
+ readyToSendToKafkaCh: make(chan struct{}),
+ eventQueueStopped: make(chan struct{}),
+ }
+ ev.popPosition = ev.insertPosition
+ return ev
+}
+
+// push is invoked to push an event at the back of a queue
+func (eq *EventQueue) push(event interface{}) {
+ eq.mutex.Lock()
+
+ if eq.insertPosition != nil {
+ // Handle Queue is full.
+ // TODO: Current default is to overwrite old data if queue is full. Is there a need to
+ // block caller if max threshold is reached?
+ if eq.insertPosition.Value != nil && eq.insertPosition == eq.popPosition {
+ eq.popPosition = eq.popPosition.Next()
+ }
+
+ // Insert data and move pointer to next empty position
+ eq.insertPosition.Value = event
+ eq.insertPosition = eq.insertPosition.Next()
+
+ // Check for last event
+ if _, ok := event.(*lastEvent); ok {
+ eq.insertPosition = nil
+ }
+ eq.mutex.Unlock()
+ // Notify waiting thread of data availability
+ eq.dataToSendAvailable <- struct{}{}
+
+ } else {
+ logger.Debug(context.Background(), "event-queue-is-closed-as-insert-position-is-cleared")
+ eq.mutex.Unlock()
+ }
+}
+
+// start starts the routine that extracts an element from the event queue and
+// send it to the kafka sending routine to process.
+func (eq *EventQueue) start(ctx context.Context) {
+ logger.Info(ctx, "starting-event-queue")
+loop:
+ for {
+ select {
+ case <-eq.dataToSendAvailable:
+ // Do nothing - use to prevent caller pushing data to block
+ case <-eq.readyToSendToKafkaCh:
+ {
+ // Kafka sending routine is ready to process an event
+ eq.mutex.Lock()
+ element := eq.popPosition.Value
+ if element == nil {
+ // No events to send. Wait
+ eq.mutex.Unlock()
+ select {
+ case _, ok := <-eq.dataToSendAvailable:
+ if !ok {
+ // channel closed
+ eq.eventQueueStopped <- struct{}{}
+ return
+ }
+ case <-ctx.Done():
+ logger.Info(ctx, "event-queue-context-done")
+ eq.eventQueueStopped <- struct{}{}
+ return
+ }
+ eq.mutex.Lock()
+ element = eq.popPosition.Value
+ }
+ eq.popPosition.Value = nil
+ eq.popPosition = eq.popPosition.Next()
+ eq.mutex.Unlock()
+ eq.eventChannel <- element
+ }
+ case <-ctx.Done():
+ logger.Info(ctx, "event-queue-context-done")
+ eq.eventQueueStopped <- struct{}{}
+ break loop
+ }
+ }
+ logger.Info(ctx, "event-queue-stopped")
+
+}
+
+func (eq *EventQueue) stop() {
+ // Flush all
+ eq.push(&lastEvent{})
+ <-eq.eventQueueStopped
+ eq.mutex.Lock()
+ close(eq.readyToSendToKafkaCh)
+ close(eq.dataToSendAvailable)
+ close(eq.eventChannel)
+ eq.mutex.Unlock()
+
+}