[VOL-4663] create voltha event topic (voltha.events) with conifgurable no of partitions and replication factor
Change-Id: I9949f9087c455c9976dad4b1e63be740bb58715f
diff --git a/go.mod b/go.mod
index 97f00b6..d6f2212 100644
--- a/go.mod
+++ b/go.mod
@@ -16,7 +16,7 @@
github.com/google/gopacket v1.1.17
github.com/looplab/fsm v0.2.0
github.com/opencord/omci-lib-go/v2 v2.2.0
- github.com/opencord/voltha-lib-go/v7 v7.1.7
+ github.com/opencord/voltha-lib-go/v7 v7.1.8
github.com/opencord/voltha-protos/v5 v5.2.2
github.com/stretchr/testify v1.7.0
google.golang.org/grpc v1.44.0
diff --git a/go.sum b/go.sum
index 61044c9..fb49fc2 100644
--- a/go.sum
+++ b/go.sum
@@ -200,8 +200,8 @@
github.com/onsi/gomega v1.14.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0=
github.com/opencord/omci-lib-go/v2 v2.2.0 h1:9eawwD1+oWHTzEv9Q/JeSN12N/3YSGKoj1dr2Pm434g=
github.com/opencord/omci-lib-go/v2 v2.2.0/go.mod h1:o1S/jhDLHNikFU7uG2TR5UOM5KmKlqwLlVncXi0FBYQ=
-github.com/opencord/voltha-lib-go/v7 v7.1.7 h1:EMdvxOa9MKiNn5giXqansl7EKc9uPCstU/1kplmqNxI=
-github.com/opencord/voltha-lib-go/v7 v7.1.7/go.mod h1:lnwlFfhDVMBg2siCv1CajB1fvfAU9Cs8VbB64LQ8zVg=
+github.com/opencord/voltha-lib-go/v7 v7.1.8 h1:5k+1Ul+T+gmvM7GONbK1/+YrX4tizAc3REgHoFvug0I=
+github.com/opencord/voltha-lib-go/v7 v7.1.8/go.mod h1:lnwlFfhDVMBg2siCv1CajB1fvfAU9Cs8VbB64LQ8zVg=
github.com/opencord/voltha-protos/v5 v5.2.2 h1:1Bcgl+Fmp00ZxlDrHZdcbjpMgOwX6TnZmOTrYm9SbR8=
github.com/opencord/voltha-protos/v5 v5.2.2/go.mod h1:ZGcyW79kQKIo7AySo1LRu613E6uiozixrCF0yNB/4x8=
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
diff --git a/internal/pkg/core/device_handler.go b/internal/pkg/core/device_handler.go
index b05cd10..fbf2ea7 100755
--- a/internal/pkg/core/device_handler.go
+++ b/internal/pkg/core/device_handler.go
@@ -2859,7 +2859,7 @@
cEventObjectType, cOnuActivatedEvent, "Cleared")
}
/* Send event to KAFKA */
- if err := dh.EventProxy.SendDeviceEvent(ctx, &de, equipment, pon, raisedTs); err != nil {
+ if err := dh.EventProxy.SendDeviceEventWithKey(ctx, &de, equipment, pon, raisedTs, aDeviceID); err != nil {
logger.Warnw(ctx, "could not send ONU_ACTIVATED event",
log.Fields{"device-id": aDeviceID, "error": err})
}
diff --git a/vendor/github.com/opencord/voltha-lib-go/v7/pkg/events/eventif/events_proxy_if.go b/vendor/github.com/opencord/voltha-lib-go/v7/pkg/events/eventif/events_proxy_if.go
index e4ebc36..625cd0b 100644
--- a/vendor/github.com/opencord/voltha-lib-go/v7/pkg/events/eventif/events_proxy_if.go
+++ b/vendor/github.com/opencord/voltha-lib-go/v7/pkg/events/eventif/events_proxy_if.go
@@ -26,6 +26,8 @@
type EventProxy interface {
SendDeviceEvent(ctx context.Context, deviceEvent *voltha.DeviceEvent, category EventCategory,
subCategory EventSubCategory, raisedTs int64) error
+ SendDeviceEventWithKey(ctx context.Context, deviceEvent *voltha.DeviceEvent, category EventCategory,
+ subCategory EventSubCategory, raisedTs int64, key string) error
SendKpiEvent(ctx context.Context, id string, deviceEvent *voltha.KpiEvent2, category EventCategory,
subCategory EventSubCategory, raisedTs int64) error
SendRPCEvent(ctx context.Context, id string, deviceEvent *voltha.RPCEvent, category EventCategory,
diff --git a/vendor/github.com/opencord/voltha-lib-go/v7/pkg/events/events_proxy.go b/vendor/github.com/opencord/voltha-lib-go/v7/pkg/events/events_proxy.go
index e4493f9..4b46854 100644
--- a/vendor/github.com/opencord/voltha-lib-go/v7/pkg/events/events_proxy.go
+++ b/vendor/github.com/opencord/voltha-lib-go/v7/pkg/events/events_proxy.go
@@ -122,6 +122,11 @@
/* Send out device events*/
func (ep *EventProxy) SendDeviceEvent(ctx context.Context, deviceEvent *voltha.DeviceEvent, category eventif.EventCategory, subCategory eventif.EventSubCategory, raisedTs int64) error {
+ return ep.SendDeviceEventWithKey(ctx, deviceEvent, category, subCategory, raisedTs, "")
+}
+
+/* Send out device events with key*/
+func (ep *EventProxy) SendDeviceEventWithKey(ctx context.Context, deviceEvent *voltha.DeviceEvent, category eventif.EventCategory, subCategory eventif.EventSubCategory, raisedTs int64, key string) error {
if deviceEvent == nil {
logger.Error(ctx, "Recieved empty device event")
return errors.New("Device event nil")
@@ -134,11 +139,12 @@
return err
}
event.EventType = &de
- if err := ep.sendEvent(ctx, &event); err != nil {
+
+ if err := ep.sendEvent(ctx, &event, key); err != nil {
logger.Errorw(ctx, "Failed to send device event to KAFKA bus", log.Fields{"device-event": deviceEvent})
return err
}
- logger.Infow(ctx, "Successfully sent device event KAFKA", log.Fields{"Id": event.Header.Id, "Category": event.Header.Category,
+ logger.Infow(ctx, "Successfully sent device event KAFKA", log.Fields{"key": key, "Id": event.Header.Id, "Category": event.Header.Category,
"SubCategory": event.Header.SubCategory, "Type": event.Header.Type, "TypeVersion": event.Header.TypeVersion,
"ReportedTs": event.Header.ReportedTs, "ResourceId": deviceEvent.ResourceId, "Context": deviceEvent.Context,
"DeviceEventName": deviceEvent.DeviceEventName})
@@ -161,7 +167,8 @@
return err
}
event.EventType = &de
- if err := ep.sendEvent(ctx, &event); err != nil {
+
+ if err := ep.sendEvent(ctx, &event, strconv.FormatInt(raisedTs, 10)); err != nil {
logger.Errorw(ctx, "Failed to send kpi event to KAFKA bus", log.Fields{"device-event": kpiEvent})
return err
}
@@ -173,9 +180,9 @@
}
-func (ep *EventProxy) sendEvent(ctx context.Context, event *voltha.Event) error {
+func (ep *EventProxy) sendEvent(ctx context.Context, event *voltha.Event, key string) error {
logger.Debugw(ctx, "Send event to kafka", log.Fields{"event": event})
- if err := ep.kafkaClient.Send(ctx, event, &ep.eventTopic); err != nil {
+ if err := ep.kafkaClient.Send(ctx, event, &ep.eventTopic, key); err != nil {
return err
}
logger.Debugw(ctx, "Sent event to kafka", log.Fields{"event": event})
@@ -193,7 +200,13 @@
// Start the event proxy
func (ep *EventProxy) Start() error {
+ if !ep.eventTopicExits(context.Background()) {
+ logger.Errorw(context.Background(), "event-topic-doesn't-exist-in-kafka", log.Fields{"element": ep.eventTopic.Name})
+ return fmt.Errorf("event topic doesn't exist in kafka")
+ }
+
eq := ep.eventQueue
+
go eq.start(ep.queueCtx)
logger.Debugw(context.Background(), "event-proxy-starting...", log.Fields{"events-threashold": EVENT_THRESHOLD})
for {
@@ -218,8 +231,8 @@
logger.Warnw(ctx, "invalid-event", log.Fields{"element": elem})
continue
}
- if err := ep.sendEvent(ctx, event); err != nil {
- logger.Errorw(ctx, "failed-to-send-event-to-kafka-bus", log.Fields{"event": event})
+ if err := ep.sendEvent(ctx, event, ""); err != nil {
+ logger.Warnw(ctx, "failed-to-send-event-to-kafka-bus", log.Fields{"event": event})
} else {
logger.Debugw(ctx, "successfully-sent-rpc-event-to-kafka-bus", log.Fields{"id": event.Header.Id, "category": event.Header.Category,
"sub-category": event.Header.SubCategory, "type": event.Header.Type, "type-version": event.Header.TypeVersion,
@@ -345,3 +358,21 @@
eq.mutex.Unlock()
}
+
+func (ep *EventProxy) eventTopicExits(ctx context.Context) bool {
+
+ // check if voltha.events topic exists
+ topics, err := ep.kafkaClient.ListTopics(ctx)
+ if err != nil {
+ logger.Errorw(ctx, "fail-to-get-topics", log.Fields{"topic": ep.eventTopic.Name, "error": err})
+ return false
+ }
+
+ logger.Debugw(ctx, "topics in kafka", log.Fields{"topics": topics, "event-topic": ep.eventTopic.Name})
+ for _, topic := range topics {
+ if topic == ep.eventTopic.Name {
+ return true
+ }
+ }
+ return false
+}
diff --git a/vendor/github.com/opencord/voltha-lib-go/v7/pkg/kafka/client.go b/vendor/github.com/opencord/voltha-lib-go/v7/pkg/kafka/client.go
index fdc05bc..afc2955 100644
--- a/vendor/github.com/opencord/voltha-lib-go/v7/pkg/kafka/client.go
+++ b/vendor/github.com/opencord/voltha-lib-go/v7/pkg/kafka/client.go
@@ -77,4 +77,5 @@
SendLiveness(ctx context.Context) error
EnableLivenessChannel(ctx context.Context, enable bool) chan bool
EnableHealthinessChannel(ctx context.Context, enable bool) chan bool
+ ListTopics(ctx context.Context) ([]string, error)
}
diff --git a/vendor/github.com/opencord/voltha-lib-go/v7/pkg/kafka/sarama_client.go b/vendor/github.com/opencord/voltha-lib-go/v7/pkg/kafka/sarama_client.go
index 185f6ec..680aa67 100644
--- a/vendor/github.com/opencord/voltha-lib-go/v7/pkg/kafka/sarama_client.go
+++ b/vendor/github.com/opencord/voltha-lib-go/v7/pkg/kafka/sarama_client.go
@@ -327,13 +327,16 @@
topicDetails[topic.Name] = topicDetail
if err := sc.cAdmin.CreateTopic(topic.Name, topicDetail, false); err != nil {
- if err == sarama.ErrTopicAlreadyExists {
- // Not an error
- logger.Debugw(ctx, "topic-already-exist", log.Fields{"topic": topic.Name})
- return nil
+ switch typedErr := err.(type) {
+ case *sarama.TopicError:
+ if typedErr.Err == sarama.ErrTopicAlreadyExists {
+ err = nil
+ }
}
- logger.Errorw(ctx, "create-topic-failure", log.Fields{"error": err})
- return err
+ if err != nil {
+ logger.Errorw(ctx, "create-topic-failure", log.Fields{"error": err})
+ return err
+ }
}
// TODO: Wait until the topic has been created. No API is available in the Sarama clusterAdmin to
// do so.
@@ -832,7 +835,7 @@
// This Creates the publisher
config := sarama.NewConfig()
config.Version = sarama.V1_0_0_0
- config.Producer.Partitioner = sarama.NewRandomPartitioner
+ config.Producer.Partitioner = sarama.NewHashPartitioner
config.Producer.Flush.Frequency = time.Duration(sc.producerFlushFrequency)
config.Producer.Flush.Messages = sc.producerFlushMessages
config.Producer.Flush.MaxMessages = sc.producerFlushMaxmessages
@@ -1152,3 +1155,21 @@
}
return nil
}
+
+func (sc *SaramaClient) ListTopics(ctx context.Context) ([]string, error) {
+
+ config := sarama.NewConfig()
+ client, err := sarama.NewClient([]string{sc.KafkaAddress}, config)
+ if err != nil {
+ logger.Debugw(ctx, "list-topics-failure", log.Fields{"error": err, "broker-address": sc.KafkaAddress})
+ return nil, err
+ }
+
+ topics, err := client.Topics()
+ if err != nil {
+ logger.Debugw(ctx, "list-topics-failure", log.Fields{"error": err, "broker-address": sc.KafkaAddress})
+ return nil, err
+ }
+
+ return topics, nil
+}
diff --git a/vendor/modules.txt b/vendor/modules.txt
index 94a4d19..c2c7606 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -138,7 +138,7 @@
github.com/opencord/omci-lib-go/v2
github.com/opencord/omci-lib-go/v2/generated
github.com/opencord/omci-lib-go/v2/meframe
-# github.com/opencord/voltha-lib-go/v7 v7.1.7
+# github.com/opencord/voltha-lib-go/v7 v7.1.8
## explicit
github.com/opencord/voltha-lib-go/v7/pkg/config
github.com/opencord/voltha-lib-go/v7/pkg/db