blob: 3381fdcd0c1dd7b21eed34628a147e4b755a5ead [file] [log] [blame]
khenaidoobf6e7bb2018-08-14 22:27:29 -04001/*
2 * Copyright 2018-present Open Networking Foundation
3
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7
8 * http://www.apache.org/licenses/LICENSE-2.0
9
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
khenaidooabad44c2018-08-03 16:58:35 -040016package kafka
17
18import (
19 "context"
20 "errors"
21 "fmt"
khenaidooabad44c2018-08-03 16:58:35 -040022 "github.com/golang/protobuf/proto"
23 "github.com/golang/protobuf/ptypes"
24 "github.com/golang/protobuf/ptypes/any"
25 "github.com/google/uuid"
26 "github.com/opencord/voltha-go/common/log"
khenaidoo79232702018-12-04 11:00:41 -050027 ic "github.com/opencord/voltha-go/protos/inter_container"
khenaidooabad44c2018-08-03 16:58:35 -040028 "reflect"
khenaidoo19374072018-12-11 11:05:15 -050029 "strings"
khenaidooabad44c2018-08-03 16:58:35 -040030 "sync"
31 "time"
32)
33
34// Initialize the logger - gets the default until the main function setup the logger
35func init() {
khenaidoob9203542018-09-17 22:56:37 -040036 log.AddPackage(log.JSON, log.WarnLevel, nil)
khenaidooabad44c2018-08-03 16:58:35 -040037}
38
39const (
khenaidoo43c82122018-11-22 18:38:28 -050040 DefaultMaxRetries = 3
41 DefaultRequestTimeout = 500 // 500 milliseconds - to handle a wider latency range
khenaidooabad44c2018-08-03 16:58:35 -040042)
43
khenaidoo43c82122018-11-22 18:38:28 -050044// requestHandlerChannel represents an interface associated with a channel. Whenever, an event is
45// obtained from that channel, this interface is invoked. This is used to handle
46// async requests into the Core via the kafka messaging bus
47type requestHandlerChannel struct {
48 requesthandlerInterface interface{}
khenaidoo79232702018-12-04 11:00:41 -050049 ch <-chan *ic.InterContainerMessage
khenaidooabad44c2018-08-03 16:58:35 -040050}
51
khenaidoo43c82122018-11-22 18:38:28 -050052// transactionChannel represents a combination of a topic and a channel onto which a response received
53// on the kafka bus will be sent to
54type transactionChannel struct {
55 topic *Topic
khenaidoo79232702018-12-04 11:00:41 -050056 ch chan *ic.InterContainerMessage
khenaidoo43c82122018-11-22 18:38:28 -050057}
58
59// InterContainerProxy represents the messaging proxy
60type InterContainerProxy struct {
61 kafkaHost string
62 kafkaPort int
63 DefaultTopic *Topic
64 defaultRequestHandlerInterface interface{}
khenaidoo79232702018-12-04 11:00:41 -050065 deviceDiscoveryTopic *Topic
khenaidoo43c82122018-11-22 18:38:28 -050066 kafkaClient Client
67 doneCh chan int
68
69 // This map is used to map a topic to an interface and channel. When a request is received
70 // on that channel (registered to the topic) then that interface is invoked.
71 topicToRequestHandlerChannelMap map[string]*requestHandlerChannel
72 lockTopicRequestHandlerChannelMap sync.RWMutex
73
74 // This map is used to map a channel to a response topic. This channel handles all responses on that
khenaidoo4c1a5bf2018-11-29 15:53:42 -050075 // channel for that topic and forward them to the appropriate consumers channel, using the
khenaidoo43c82122018-11-22 18:38:28 -050076 // transactionIdToChannelMap.
khenaidoo79232702018-12-04 11:00:41 -050077 topicToResponseChannelMap map[string]<-chan *ic.InterContainerMessage
khenaidoo43c82122018-11-22 18:38:28 -050078 lockTopicResponseChannelMap sync.RWMutex
79
khenaidoo4c1a5bf2018-11-29 15:53:42 -050080 // This map is used to map a transaction to a consumers channel. This is used whenever a request has been
khenaidoo43c82122018-11-22 18:38:28 -050081 // sent out and we are waiting for a response.
82 transactionIdToChannelMap map[string]*transactionChannel
khenaidooabad44c2018-08-03 16:58:35 -040083 lockTransactionIdToChannelMap sync.RWMutex
84}
85
khenaidoo43c82122018-11-22 18:38:28 -050086type InterContainerProxyOption func(*InterContainerProxy)
khenaidooabad44c2018-08-03 16:58:35 -040087
khenaidoo43c82122018-11-22 18:38:28 -050088func InterContainerHost(host string) InterContainerProxyOption {
89 return func(args *InterContainerProxy) {
90 args.kafkaHost = host
khenaidooabad44c2018-08-03 16:58:35 -040091 }
92}
93
khenaidoo43c82122018-11-22 18:38:28 -050094func InterContainerPort(port int) InterContainerProxyOption {
95 return func(args *InterContainerProxy) {
96 args.kafkaPort = port
khenaidooabad44c2018-08-03 16:58:35 -040097 }
98}
99
khenaidoo43c82122018-11-22 18:38:28 -0500100func DefaultTopic(topic *Topic) InterContainerProxyOption {
101 return func(args *InterContainerProxy) {
khenaidooabad44c2018-08-03 16:58:35 -0400102 args.DefaultTopic = topic
103 }
104}
105
khenaidoo79232702018-12-04 11:00:41 -0500106func DeviceDiscoveryTopic(topic *Topic) InterContainerProxyOption {
107 return func(args *InterContainerProxy) {
108 args.deviceDiscoveryTopic = topic
109 }
110}
111
khenaidoo43c82122018-11-22 18:38:28 -0500112func RequestHandlerInterface(handler interface{}) InterContainerProxyOption {
113 return func(args *InterContainerProxy) {
114 args.defaultRequestHandlerInterface = handler
khenaidooabad44c2018-08-03 16:58:35 -0400115 }
116}
117
khenaidoo43c82122018-11-22 18:38:28 -0500118func MsgClient(client Client) InterContainerProxyOption {
119 return func(args *InterContainerProxy) {
120 args.kafkaClient = client
121 }
122}
123
124func NewInterContainerProxy(opts ...InterContainerProxyOption) (*InterContainerProxy, error) {
125 proxy := &InterContainerProxy{
126 kafkaHost: DefaultKafkaHost,
127 kafkaPort: DefaultKafkaPort,
khenaidooabad44c2018-08-03 16:58:35 -0400128 }
129
130 for _, option := range opts {
131 option(proxy)
132 }
133
134 // Create the locks for all the maps
khenaidoo43c82122018-11-22 18:38:28 -0500135 proxy.lockTopicRequestHandlerChannelMap = sync.RWMutex{}
khenaidooabad44c2018-08-03 16:58:35 -0400136 proxy.lockTransactionIdToChannelMap = sync.RWMutex{}
khenaidoo43c82122018-11-22 18:38:28 -0500137 proxy.lockTopicResponseChannelMap = sync.RWMutex{}
khenaidooabad44c2018-08-03 16:58:35 -0400138
139 return proxy, nil
140}
141
khenaidoo43c82122018-11-22 18:38:28 -0500142func (kp *InterContainerProxy) Start() error {
khenaidooabad44c2018-08-03 16:58:35 -0400143 log.Info("Starting-Proxy")
144
khenaidoo43c82122018-11-22 18:38:28 -0500145 // Kafka MsgClient should already have been created. If not, output fatal error
146 if kp.kafkaClient == nil {
147 log.Fatal("kafka-client-not-set")
148 }
149
khenaidooabad44c2018-08-03 16:58:35 -0400150 // Create the Done channel
151 kp.doneCh = make(chan int, 1)
152
khenaidoo43c82122018-11-22 18:38:28 -0500153 // Start the kafka client
khenaidoo4c1a5bf2018-11-29 15:53:42 -0500154 if err := kp.kafkaClient.Start(); err != nil {
khenaidoo43c82122018-11-22 18:38:28 -0500155 log.Errorw("Cannot-create-kafka-proxy", log.Fields{"error": err})
khenaidooabad44c2018-08-03 16:58:35 -0400156 return err
157 }
158
khenaidoo43c82122018-11-22 18:38:28 -0500159 // Create the topic to response channel map
khenaidoo79232702018-12-04 11:00:41 -0500160 kp.topicToResponseChannelMap = make(map[string]<-chan *ic.InterContainerMessage)
khenaidoo43c82122018-11-22 18:38:28 -0500161 //
khenaidooabad44c2018-08-03 16:58:35 -0400162 // Create the transactionId to Channel Map
khenaidoo43c82122018-11-22 18:38:28 -0500163 kp.transactionIdToChannelMap = make(map[string]*transactionChannel)
164
165 // Create the topic to request channel map
166 kp.topicToRequestHandlerChannelMap = make(map[string]*requestHandlerChannel)
khenaidooabad44c2018-08-03 16:58:35 -0400167
168 return nil
169}
170
khenaidoo43c82122018-11-22 18:38:28 -0500171func (kp *InterContainerProxy) Stop() {
172 log.Info("stopping-intercontainer-proxy")
173 kp.doneCh <- 1
174 // TODO : Perform cleanup
175 //kp.kafkaClient.Stop()
176 //kp.deleteAllTopicRequestHandlerChannelMap()
177 //kp.deleteAllTopicResponseChannelMap()
178 //kp.deleteAllTransactionIdToChannelMap()
khenaidooabad44c2018-08-03 16:58:35 -0400179}
180
khenaidoo79232702018-12-04 11:00:41 -0500181// DeviceDiscovered publish the discovered device onto the kafka messaging bus
khenaidoo19374072018-12-11 11:05:15 -0500182func (kp *InterContainerProxy) DeviceDiscovered(deviceId string, deviceType string, parentId string, publisher string) error {
khenaidoo79232702018-12-04 11:00:41 -0500183 log.Debugw("sending-device-discovery-msg", log.Fields{"deviceId": deviceId})
184 // Simple validation
185 if deviceId == "" || deviceType == "" {
186 log.Errorw("invalid-parameters", log.Fields{"id": deviceId, "type": deviceType})
187 return errors.New("invalid-parameters")
188 }
189 // Create the device discovery message
190 header := &ic.Header{
191 Id: uuid.New().String(),
192 Type: ic.MessageType_DEVICE_DISCOVERED,
193 FromTopic: kp.DefaultTopic.Name,
194 ToTopic: kp.deviceDiscoveryTopic.Name,
195 Timestamp: time.Now().UnixNano(),
196 }
197 body := &ic.DeviceDiscovered{
198 Id: deviceId,
199 DeviceType: deviceType,
200 ParentId: parentId,
khenaidood2b6df92018-12-13 16:37:20 -0500201 Publisher: publisher,
khenaidoo79232702018-12-04 11:00:41 -0500202 }
203
204 var marshalledData *any.Any
205 var err error
206 if marshalledData, err = ptypes.MarshalAny(body); err != nil {
207 log.Errorw("cannot-marshal-request", log.Fields{"error": err})
208 return err
209 }
210 msg := &ic.InterContainerMessage{
211 Header: header,
212 Body: marshalledData,
213 }
214
215 // Send the message
216 if err := kp.kafkaClient.Send(msg, kp.deviceDiscoveryTopic); err != nil {
217 log.Errorw("cannot-send-device-discovery-message", log.Fields{"error": err})
218 return err
219 }
220 return nil
221}
222
khenaidoo43c82122018-11-22 18:38:28 -0500223// InvokeRPC is used to send a request to a given topic
224func (kp *InterContainerProxy) InvokeRPC(ctx context.Context, rpc string, toTopic *Topic, replyToTopic *Topic,
225 waitForResponse bool, kvArgs ...*KVArg) (bool, *any.Any) {
226
227 // If a replyToTopic is provided then we use it, otherwise just use the default toTopic. The replyToTopic is
228 // typically the device ID.
229 responseTopic := replyToTopic
230 if responseTopic == nil {
231 responseTopic = kp.DefaultTopic
232 }
233
khenaidooabad44c2018-08-03 16:58:35 -0400234 // Encode the request
khenaidoo43c82122018-11-22 18:38:28 -0500235 protoRequest, err := encodeRequest(rpc, toTopic, responseTopic, kvArgs...)
khenaidooabad44c2018-08-03 16:58:35 -0400236 if err != nil {
237 log.Warnw("cannot-format-request", log.Fields{"rpc": rpc, "error": err})
238 return false, nil
239 }
240
241 // Subscribe for response, if needed, before sending request
khenaidoo79232702018-12-04 11:00:41 -0500242 var ch <-chan *ic.InterContainerMessage
khenaidooabad44c2018-08-03 16:58:35 -0400243 if waitForResponse {
244 var err error
khenaidoo43c82122018-11-22 18:38:28 -0500245 if ch, err = kp.subscribeForResponse(*responseTopic, protoRequest.Header.Id); err != nil {
246 log.Errorw("failed-to-subscribe-for-response", log.Fields{"error": err, "toTopic": toTopic.Name})
khenaidooabad44c2018-08-03 16:58:35 -0400247 }
248 }
249
khenaidoo43c82122018-11-22 18:38:28 -0500250 // Send request - if the topic is formatted with a device Id then we will send the request using a
251 // specific key, hence ensuring a single partition is used to publish the request. This ensures that the
252 // subscriber on that topic will receive the request in the order it was sent. The key used is the deviceId.
253 key := GetDeviceIdFromTopic(*toTopic)
254 log.Debugw("sending-msg", log.Fields{"rpc": rpc, "toTopic": toTopic, "replyTopic": responseTopic, "key": key})
255 go kp.kafkaClient.Send(protoRequest, toTopic, key)
khenaidooabad44c2018-08-03 16:58:35 -0400256
257 if waitForResponse {
khenaidoob9203542018-09-17 22:56:37 -0400258 // Create a child context based on the parent context, if any
khenaidooabad44c2018-08-03 16:58:35 -0400259 var cancel context.CancelFunc
khenaidoob9203542018-09-17 22:56:37 -0400260 childCtx := context.Background()
khenaidooabad44c2018-08-03 16:58:35 -0400261 if ctx == nil {
262 ctx, cancel = context.WithTimeout(context.Background(), DefaultRequestTimeout*time.Millisecond)
khenaidoob9203542018-09-17 22:56:37 -0400263 } else {
264 childCtx, cancel = context.WithTimeout(ctx, DefaultRequestTimeout*time.Millisecond)
khenaidooabad44c2018-08-03 16:58:35 -0400265 }
khenaidoob9203542018-09-17 22:56:37 -0400266 defer cancel()
khenaidooabad44c2018-08-03 16:58:35 -0400267
268 // Wait for response as well as timeout or cancellation
269 // Remove the subscription for a response on return
270 defer kp.unSubscribeForResponse(protoRequest.Header.Id)
271 select {
272 case msg := <-ch:
khenaidoo43c82122018-11-22 18:38:28 -0500273 log.Debugw("received-response", log.Fields{"rpc": rpc, "msgHeader": msg.Header})
khenaidoo79232702018-12-04 11:00:41 -0500274 var responseBody *ic.InterContainerResponseBody
khenaidooabad44c2018-08-03 16:58:35 -0400275 var err error
276 if responseBody, err = decodeResponse(msg); err != nil {
277 log.Errorw("decode-response-error", log.Fields{"error": err})
278 }
279 return responseBody.Success, responseBody.Result
280 case <-ctx.Done():
281 log.Debugw("context-cancelled", log.Fields{"rpc": rpc, "ctx": ctx.Err()})
282 // pack the error as proto any type
khenaidoo79232702018-12-04 11:00:41 -0500283 protoError := &ic.Error{Reason: ctx.Err().Error()}
khenaidooabad44c2018-08-03 16:58:35 -0400284 var marshalledArg *any.Any
285 if marshalledArg, err = ptypes.MarshalAny(protoError); err != nil {
286 return false, nil // Should never happen
287 }
288 return false, marshalledArg
khenaidoob9203542018-09-17 22:56:37 -0400289 case <-childCtx.Done():
290 log.Debugw("context-cancelled", log.Fields{"rpc": rpc, "ctx": childCtx.Err()})
291 // pack the error as proto any type
khenaidoo79232702018-12-04 11:00:41 -0500292 protoError := &ic.Error{Reason: childCtx.Err().Error()}
khenaidoob9203542018-09-17 22:56:37 -0400293 var marshalledArg *any.Any
294 if marshalledArg, err = ptypes.MarshalAny(protoError); err != nil {
295 return false, nil // Should never happen
296 }
297 return false, marshalledArg
khenaidooabad44c2018-08-03 16:58:35 -0400298 case <-kp.doneCh:
khenaidoo43c82122018-11-22 18:38:28 -0500299 log.Infow("received-exit-signal", log.Fields{"toTopic": toTopic.Name, "rpc": rpc})
khenaidooabad44c2018-08-03 16:58:35 -0400300 return true, nil
301 }
302 }
303 return true, nil
304}
305
khenaidoo43c82122018-11-22 18:38:28 -0500306// SubscribeWithRequestHandlerInterface allows a caller to assign a target object to be invoked automatically
khenaidooabad44c2018-08-03 16:58:35 -0400307// when a message is received on a given topic
khenaidoo43c82122018-11-22 18:38:28 -0500308func (kp *InterContainerProxy) SubscribeWithRequestHandlerInterface(topic Topic, handler interface{}) error {
khenaidooabad44c2018-08-03 16:58:35 -0400309
310 // Subscribe to receive messages for that topic
khenaidoo79232702018-12-04 11:00:41 -0500311 var ch <-chan *ic.InterContainerMessage
khenaidooabad44c2018-08-03 16:58:35 -0400312 var err error
khenaidoo4c1a5bf2018-11-29 15:53:42 -0500313 if ch, err = kp.kafkaClient.Subscribe(&topic); err != nil {
khenaidoo43c82122018-11-22 18:38:28 -0500314 //if ch, err = kp.Subscribe(topic); err != nil {
khenaidooabad44c2018-08-03 16:58:35 -0400315 log.Errorw("failed-to-subscribe", log.Fields{"error": err, "topic": topic.Name})
316 }
khenaidoo43c82122018-11-22 18:38:28 -0500317
318 kp.defaultRequestHandlerInterface = handler
319 kp.addToTopicRequestHandlerChannelMap(topic.Name, &requestHandlerChannel{requesthandlerInterface: handler, ch: ch})
khenaidooabad44c2018-08-03 16:58:35 -0400320 // Launch a go routine to receive and process kafka messages
khenaidoo43c82122018-11-22 18:38:28 -0500321 go kp.waitForRequest(ch, topic, handler)
khenaidooabad44c2018-08-03 16:58:35 -0400322
323 return nil
324}
325
khenaidoo43c82122018-11-22 18:38:28 -0500326// SubscribeWithDefaultRequestHandler allows a caller to add a topic to an existing target object to be invoked automatically
327// when a message is received on a given topic. So far there is only 1 target registered per microservice
328func (kp *InterContainerProxy) SubscribeWithDefaultRequestHandler(topic Topic) error {
329 // Subscribe to receive messages for that topic
khenaidoo79232702018-12-04 11:00:41 -0500330 var ch <-chan *ic.InterContainerMessage
khenaidoo43c82122018-11-22 18:38:28 -0500331 var err error
khenaidoo4c1a5bf2018-11-29 15:53:42 -0500332 if ch, err = kp.kafkaClient.Subscribe(&topic); err != nil {
khenaidoo43c82122018-11-22 18:38:28 -0500333 log.Errorw("failed-to-subscribe", log.Fields{"error": err, "topic": topic.Name})
334 }
335 kp.addToTopicRequestHandlerChannelMap(topic.Name, &requestHandlerChannel{requesthandlerInterface: kp.defaultRequestHandlerInterface, ch: ch})
336
337 // Launch a go routine to receive and process kafka messages
338 go kp.waitForRequest(ch, topic, kp.defaultRequestHandlerInterface)
339
khenaidooabad44c2018-08-03 16:58:35 -0400340 return nil
341}
342
khenaidoo43c82122018-11-22 18:38:28 -0500343func (kp *InterContainerProxy) UnSubscribeFromRequestHandler(topic Topic) error {
344 return kp.deleteFromTopicRequestHandlerChannelMap(topic.Name)
345}
346
khenaidoo4c1a5bf2018-11-29 15:53:42 -0500347// setupTopicResponseChannelMap sets up single consumers channel that will act as a broadcast channel for all
khenaidoo43c82122018-11-22 18:38:28 -0500348// responses from that topic.
khenaidoo79232702018-12-04 11:00:41 -0500349func (kp *InterContainerProxy) setupTopicResponseChannelMap(topic string, arg <-chan *ic.InterContainerMessage) {
khenaidoo43c82122018-11-22 18:38:28 -0500350 kp.lockTopicResponseChannelMap.Lock()
351 defer kp.lockTopicResponseChannelMap.Unlock()
352 if _, exist := kp.topicToResponseChannelMap[topic]; !exist {
353 kp.topicToResponseChannelMap[topic] = arg
khenaidooabad44c2018-08-03 16:58:35 -0400354 }
355}
356
khenaidoo43c82122018-11-22 18:38:28 -0500357func (kp *InterContainerProxy) isTopicSubscribedForResponse(topic string) bool {
358 kp.lockTopicResponseChannelMap.Lock()
359 defer kp.lockTopicResponseChannelMap.Unlock()
360 _, exist := kp.topicToResponseChannelMap[topic]
361 return exist
362}
363
364func (kp *InterContainerProxy) deleteFromTopicResponseChannelMap(topic string) error {
365 kp.lockTopicResponseChannelMap.Lock()
366 defer kp.lockTopicResponseChannelMap.Unlock()
367 if _, exist := kp.topicToResponseChannelMap[topic]; exist {
368 // Unsubscribe to this topic first - this will close the subscribed channel
369 var err error
370 if err = kp.kafkaClient.UnSubscribe(&Topic{Name: topic}, kp.topicToResponseChannelMap[topic]); err != nil {
371 log.Errorw("unsubscribing-error", log.Fields{"topic": topic})
372 }
373 delete(kp.topicToResponseChannelMap, topic)
374 return err
375 } else {
376 return errors.New(fmt.Sprintf("%s-Topic-not-found", topic))
khenaidooabad44c2018-08-03 16:58:35 -0400377 }
378}
379
khenaidoo43c82122018-11-22 18:38:28 -0500380func (kp *InterContainerProxy) deleteAllTopicResponseChannelMap() error {
381 kp.lockTopicResponseChannelMap.Lock()
382 defer kp.lockTopicResponseChannelMap.Unlock()
383 var err error
384 for topic, _ := range kp.topicToResponseChannelMap {
385 // Unsubscribe to this topic first - this will close the subscribed channel
386 if err = kp.kafkaClient.UnSubscribe(&Topic{Name: topic}, kp.topicToResponseChannelMap[topic]); err != nil {
387 log.Errorw("unsubscribing-error", log.Fields{"topic": topic, "error": err})
388 }
389 delete(kp.topicToResponseChannelMap, topic)
khenaidooabad44c2018-08-03 16:58:35 -0400390 }
khenaidoo43c82122018-11-22 18:38:28 -0500391 return err
khenaidooabad44c2018-08-03 16:58:35 -0400392}
393
khenaidoo43c82122018-11-22 18:38:28 -0500394func (kp *InterContainerProxy) addToTopicRequestHandlerChannelMap(topic string, arg *requestHandlerChannel) {
395 kp.lockTopicRequestHandlerChannelMap.Lock()
396 defer kp.lockTopicRequestHandlerChannelMap.Unlock()
397 if _, exist := kp.topicToRequestHandlerChannelMap[topic]; !exist {
398 kp.topicToRequestHandlerChannelMap[topic] = arg
khenaidooabad44c2018-08-03 16:58:35 -0400399 }
khenaidooabad44c2018-08-03 16:58:35 -0400400}
401
khenaidoo43c82122018-11-22 18:38:28 -0500402func (kp *InterContainerProxy) deleteFromTopicRequestHandlerChannelMap(topic string) error {
403 kp.lockTopicRequestHandlerChannelMap.Lock()
404 defer kp.lockTopicRequestHandlerChannelMap.Unlock()
405 if _, exist := kp.topicToRequestHandlerChannelMap[topic]; exist {
406 // Close the kafka client client first by unsubscribing to this topic
407 kp.kafkaClient.UnSubscribe(&Topic{Name: topic}, kp.topicToRequestHandlerChannelMap[topic].ch)
408 delete(kp.topicToRequestHandlerChannelMap, topic)
khenaidooabad44c2018-08-03 16:58:35 -0400409 return nil
khenaidoo43c82122018-11-22 18:38:28 -0500410 } else {
411 return errors.New(fmt.Sprintf("%s-Topic-not-found", topic))
khenaidooabad44c2018-08-03 16:58:35 -0400412 }
khenaidooabad44c2018-08-03 16:58:35 -0400413}
414
khenaidoo43c82122018-11-22 18:38:28 -0500415func (kp *InterContainerProxy) deleteAllTopicRequestHandlerChannelMap() error {
416 kp.lockTopicRequestHandlerChannelMap.Lock()
417 defer kp.lockTopicRequestHandlerChannelMap.Unlock()
418 var err error
419 for topic, _ := range kp.topicToRequestHandlerChannelMap {
420 // Close the kafka client client first by unsubscribing to this topic
421 if err = kp.kafkaClient.UnSubscribe(&Topic{Name: topic}, kp.topicToRequestHandlerChannelMap[topic].ch); err != nil {
422 log.Errorw("unsubscribing-error", log.Fields{"topic": topic, "error": err})
423 }
424 delete(kp.topicToRequestHandlerChannelMap, topic)
425 }
426 return err
427}
428
khenaidoo79232702018-12-04 11:00:41 -0500429func (kp *InterContainerProxy) addToTransactionIdToChannelMap(id string, topic *Topic, arg chan *ic.InterContainerMessage) {
khenaidooabad44c2018-08-03 16:58:35 -0400430 kp.lockTransactionIdToChannelMap.Lock()
431 defer kp.lockTransactionIdToChannelMap.Unlock()
432 if _, exist := kp.transactionIdToChannelMap[id]; !exist {
khenaidoo43c82122018-11-22 18:38:28 -0500433 kp.transactionIdToChannelMap[id] = &transactionChannel{topic: topic, ch: arg}
khenaidooabad44c2018-08-03 16:58:35 -0400434 }
435}
436
khenaidoo43c82122018-11-22 18:38:28 -0500437func (kp *InterContainerProxy) deleteFromTransactionIdToChannelMap(id string) {
khenaidooabad44c2018-08-03 16:58:35 -0400438 kp.lockTransactionIdToChannelMap.Lock()
439 defer kp.lockTransactionIdToChannelMap.Unlock()
khenaidoo43c82122018-11-22 18:38:28 -0500440 if transChannel, exist := kp.transactionIdToChannelMap[id]; exist {
441 // Close the channel first
442 close(transChannel.ch)
khenaidooabad44c2018-08-03 16:58:35 -0400443 delete(kp.transactionIdToChannelMap, id)
444 }
445}
446
khenaidoo43c82122018-11-22 18:38:28 -0500447func (kp *InterContainerProxy) deleteTopicTransactionIdToChannelMap(id string) {
448 kp.lockTransactionIdToChannelMap.Lock()
449 defer kp.lockTransactionIdToChannelMap.Unlock()
450 for key, value := range kp.transactionIdToChannelMap {
451 if value.topic.Name == id {
452 close(value.ch)
453 delete(kp.transactionIdToChannelMap, key)
khenaidooabad44c2018-08-03 16:58:35 -0400454 }
455 }
khenaidooabad44c2018-08-03 16:58:35 -0400456}
457
khenaidoo43c82122018-11-22 18:38:28 -0500458func (kp *InterContainerProxy) deleteAllTransactionIdToChannelMap() {
459 kp.lockTransactionIdToChannelMap.Lock()
460 defer kp.lockTransactionIdToChannelMap.Unlock()
461 for key, value := range kp.transactionIdToChannelMap {
462 close(value.ch)
463 delete(kp.transactionIdToChannelMap, key)
khenaidooabad44c2018-08-03 16:58:35 -0400464 }
khenaidooabad44c2018-08-03 16:58:35 -0400465}
466
khenaidoo43c82122018-11-22 18:38:28 -0500467func (kp *InterContainerProxy) DeleteTopic(topic Topic) error {
468 // If we have any consumers on that topic we need to close them
469 kp.deleteFromTopicResponseChannelMap(topic.Name)
470 kp.deleteFromTopicRequestHandlerChannelMap(topic.Name)
471 kp.deleteTopicTransactionIdToChannelMap(topic.Name)
472 return kp.kafkaClient.DeleteTopic(&topic)
473}
474
475func encodeReturnedValue(returnedVal interface{}) (*any.Any, error) {
khenaidooabad44c2018-08-03 16:58:35 -0400476 // Encode the response argument - needs to be a proto message
477 if returnedVal == nil {
478 return nil, nil
479 }
480 protoValue, ok := returnedVal.(proto.Message)
481 if !ok {
482 log.Warnw("response-value-not-proto-message", log.Fields{"error": ok, "returnVal": returnedVal})
483 err := errors.New("response-value-not-proto-message")
484 return nil, err
485 }
486
487 // Marshal the returned value, if any
488 var marshalledReturnedVal *any.Any
489 var err error
490 if marshalledReturnedVal, err = ptypes.MarshalAny(protoValue); err != nil {
491 log.Warnw("cannot-marshal-returned-val", log.Fields{"error": err})
492 return nil, err
493 }
494 return marshalledReturnedVal, nil
495}
496
khenaidoo79232702018-12-04 11:00:41 -0500497func encodeDefaultFailedResponse(request *ic.InterContainerMessage) *ic.InterContainerMessage {
498 responseHeader := &ic.Header{
khenaidooabad44c2018-08-03 16:58:35 -0400499 Id: request.Header.Id,
khenaidoo79232702018-12-04 11:00:41 -0500500 Type: ic.MessageType_RESPONSE,
khenaidooabad44c2018-08-03 16:58:35 -0400501 FromTopic: request.Header.ToTopic,
502 ToTopic: request.Header.FromTopic,
503 Timestamp: time.Now().Unix(),
504 }
khenaidoo79232702018-12-04 11:00:41 -0500505 responseBody := &ic.InterContainerResponseBody{
khenaidooabad44c2018-08-03 16:58:35 -0400506 Success: false,
507 Result: nil,
508 }
509 var marshalledResponseBody *any.Any
510 var err error
511 // Error should never happen here
512 if marshalledResponseBody, err = ptypes.MarshalAny(responseBody); err != nil {
513 log.Warnw("cannot-marshal-failed-response-body", log.Fields{"error": err})
514 }
515
khenaidoo79232702018-12-04 11:00:41 -0500516 return &ic.InterContainerMessage{
khenaidooabad44c2018-08-03 16:58:35 -0400517 Header: responseHeader,
518 Body: marshalledResponseBody,
519 }
520
521}
522
523//formatRequest formats a request to send over kafka and returns an InterContainerMessage message on success
524//or an error on failure
khenaidoo79232702018-12-04 11:00:41 -0500525func encodeResponse(request *ic.InterContainerMessage, success bool, returnedValues ...interface{}) (*ic.InterContainerMessage, error) {
khenaidoo43c82122018-11-22 18:38:28 -0500526 //log.Debugw("encodeResponse", log.Fields{"success": success, "returnedValues": returnedValues})
khenaidoo79232702018-12-04 11:00:41 -0500527 responseHeader := &ic.Header{
khenaidooabad44c2018-08-03 16:58:35 -0400528 Id: request.Header.Id,
khenaidoo79232702018-12-04 11:00:41 -0500529 Type: ic.MessageType_RESPONSE,
khenaidooabad44c2018-08-03 16:58:35 -0400530 FromTopic: request.Header.ToTopic,
531 ToTopic: request.Header.FromTopic,
532 Timestamp: time.Now().Unix(),
533 }
534
535 // Go over all returned values
536 var marshalledReturnedVal *any.Any
537 var err error
538 for _, returnVal := range returnedValues {
khenaidoo43c82122018-11-22 18:38:28 -0500539 if marshalledReturnedVal, err = encodeReturnedValue(returnVal); err != nil {
khenaidooabad44c2018-08-03 16:58:35 -0400540 log.Warnw("cannot-marshal-response-body", log.Fields{"error": err})
541 }
542 break // for now we support only 1 returned value - (excluding the error)
543 }
544
khenaidoo79232702018-12-04 11:00:41 -0500545 responseBody := &ic.InterContainerResponseBody{
khenaidooabad44c2018-08-03 16:58:35 -0400546 Success: success,
547 Result: marshalledReturnedVal,
548 }
549
550 // Marshal the response body
551 var marshalledResponseBody *any.Any
552 if marshalledResponseBody, err = ptypes.MarshalAny(responseBody); err != nil {
553 log.Warnw("cannot-marshal-response-body", log.Fields{"error": err})
554 return nil, err
555 }
556
khenaidoo79232702018-12-04 11:00:41 -0500557 return &ic.InterContainerMessage{
khenaidooabad44c2018-08-03 16:58:35 -0400558 Header: responseHeader,
559 Body: marshalledResponseBody,
560 }, nil
561}
562
563func CallFuncByName(myClass interface{}, funcName string, params ...interface{}) (out []reflect.Value, err error) {
564 myClassValue := reflect.ValueOf(myClass)
khenaidoo19374072018-12-11 11:05:15 -0500565 // Capitalize the first letter in the funcName to workaround the first capital letters required to
566 // invoke a function from a different package
567 funcName = strings.Title(funcName)
khenaidooabad44c2018-08-03 16:58:35 -0400568 m := myClassValue.MethodByName(funcName)
569 if !m.IsValid() {
khenaidoo43c82122018-11-22 18:38:28 -0500570 return make([]reflect.Value, 0), fmt.Errorf("method-not-found \"%s\"", funcName)
khenaidooabad44c2018-08-03 16:58:35 -0400571 }
572 in := make([]reflect.Value, len(params))
573 for i, param := range params {
574 in[i] = reflect.ValueOf(param)
575 }
576 out = m.Call(in)
577 return
578}
579
khenaidoo79232702018-12-04 11:00:41 -0500580func (kp *InterContainerProxy) handleRequest(msg *ic.InterContainerMessage, targetInterface interface{}) {
khenaidooabad44c2018-08-03 16:58:35 -0400581
khenaidoo43c82122018-11-22 18:38:28 -0500582 // First extract the header to know whether this is a request - responses are handled by a different handler
khenaidoo79232702018-12-04 11:00:41 -0500583 if msg.Header.Type == ic.MessageType_REQUEST {
khenaidooabad44c2018-08-03 16:58:35 -0400584
585 var out []reflect.Value
586 var err error
587
588 // Get the request body
khenaidoo79232702018-12-04 11:00:41 -0500589 requestBody := &ic.InterContainerRequestBody{}
khenaidooabad44c2018-08-03 16:58:35 -0400590 if err = ptypes.UnmarshalAny(msg.Body, requestBody); err != nil {
591 log.Warnw("cannot-unmarshal-request", log.Fields{"error": err})
592 } else {
khenaidoo43c82122018-11-22 18:38:28 -0500593 log.Debugw("received-request", log.Fields{"rpc": requestBody.Rpc, "header": msg.Header})
khenaidooabad44c2018-08-03 16:58:35 -0400594 // let the callee unpack the arguments as its the only one that knows the real proto type
595 out, err = CallFuncByName(targetInterface, requestBody.Rpc, requestBody.Args)
596 if err != nil {
597 log.Warn(err)
598 }
599 }
600 // Response required?
601 if requestBody.ResponseRequired {
602 // If we already have an error before then just return that
khenaidoo79232702018-12-04 11:00:41 -0500603 var returnError *ic.Error
khenaidooabad44c2018-08-03 16:58:35 -0400604 var returnedValues []interface{}
605 var success bool
606 if err != nil {
khenaidoo79232702018-12-04 11:00:41 -0500607 returnError = &ic.Error{Reason: err.Error()}
khenaidooabad44c2018-08-03 16:58:35 -0400608 returnedValues = make([]interface{}, 1)
609 returnedValues[0] = returnError
610 } else {
khenaidoo43c82122018-11-22 18:38:28 -0500611 //log.Debugw("returned-api-response", log.Fields{"len": len(out), "err": err})
khenaidoob9203542018-09-17 22:56:37 -0400612 returnedValues = make([]interface{}, 0)
613 // Check for errors first
614 lastIndex := len(out) - 1
615 if out[lastIndex].Interface() != nil { // Error
616 if goError, ok := out[lastIndex].Interface().(error); ok {
khenaidoo79232702018-12-04 11:00:41 -0500617 returnError = &ic.Error{Reason: goError.Error()}
khenaidoob9203542018-09-17 22:56:37 -0400618 returnedValues = append(returnedValues, returnError)
619 } else { // Should never happen
khenaidoo79232702018-12-04 11:00:41 -0500620 returnError = &ic.Error{Reason: "incorrect-error-returns"}
khenaidoob9203542018-09-17 22:56:37 -0400621 returnedValues = append(returnedValues, returnError)
622 }
623 } else { // Non-error case
624 success = true
625 for idx, val := range out {
khenaidoo43c82122018-11-22 18:38:28 -0500626 //log.Debugw("returned-api-response-loop", log.Fields{"idx": idx, "val": val.Interface()})
khenaidoob9203542018-09-17 22:56:37 -0400627 if idx != lastIndex {
628 returnedValues = append(returnedValues, val.Interface())
khenaidooabad44c2018-08-03 16:58:35 -0400629 }
khenaidooabad44c2018-08-03 16:58:35 -0400630 }
631 }
632 }
633
khenaidoo79232702018-12-04 11:00:41 -0500634 var icm *ic.InterContainerMessage
khenaidooabad44c2018-08-03 16:58:35 -0400635 if icm, err = encodeResponse(msg, success, returnedValues...); err != nil {
khenaidoob9203542018-09-17 22:56:37 -0400636 log.Warnw("error-encoding-response-returning-failure-result", log.Fields{"error": err})
khenaidooabad44c2018-08-03 16:58:35 -0400637 icm = encodeDefaultFailedResponse(msg)
638 }
khenaidoo43c82122018-11-22 18:38:28 -0500639 // To preserve ordering of messages, all messages to a given topic are sent to the same partition
640 // by providing a message key. The key is encoded in the topic name. If the deviceId is not
641 // present then the key will be empty, hence all messages for a given topic will be sent to all
642 // partitions.
643 replyTopic := &Topic{Name: msg.Header.FromTopic}
644 key := GetDeviceIdFromTopic(*replyTopic)
khenaidoo90847922018-12-03 14:47:51 -0500645 log.Debugw("sending-response-to-kafka", log.Fields{"rpc": requestBody.Rpc, "header": icm.Header, "key": key})
khenaidoo43c82122018-11-22 18:38:28 -0500646 // TODO: handle error response.
647 kp.kafkaClient.Send(icm, replyTopic, key)
khenaidooabad44c2018-08-03 16:58:35 -0400648 }
649
khenaidooabad44c2018-08-03 16:58:35 -0400650 }
651}
652
khenaidoo79232702018-12-04 11:00:41 -0500653func (kp *InterContainerProxy) waitForRequest(ch <-chan *ic.InterContainerMessage, topic Topic, targetInterface interface{}) {
khenaidooabad44c2018-08-03 16:58:35 -0400654 // Wait for messages
655 for msg := range ch {
khenaidoo43c82122018-11-22 18:38:28 -0500656 //log.Debugw("request-received", log.Fields{"msg": msg, "topic": topic.Name, "target": targetInterface})
khenaidooabad44c2018-08-03 16:58:35 -0400657 go kp.handleRequest(msg, targetInterface)
658 }
659}
660
khenaidoo79232702018-12-04 11:00:41 -0500661func (kp *InterContainerProxy) dispatchResponse(msg *ic.InterContainerMessage) {
khenaidooabad44c2018-08-03 16:58:35 -0400662 kp.lockTransactionIdToChannelMap.Lock()
663 defer kp.lockTransactionIdToChannelMap.Unlock()
664 if _, exist := kp.transactionIdToChannelMap[msg.Header.Id]; !exist {
665 log.Debugw("no-waiting-channel", log.Fields{"transaction": msg.Header.Id})
666 return
667 }
khenaidoo43c82122018-11-22 18:38:28 -0500668 kp.transactionIdToChannelMap[msg.Header.Id].ch <- msg
khenaidooabad44c2018-08-03 16:58:35 -0400669}
670
khenaidoo43c82122018-11-22 18:38:28 -0500671// waitForResponse listens for messages on the subscribedCh, ensure we get a response with the transaction ID,
khenaidoo4c1a5bf2018-11-29 15:53:42 -0500672// and then dispatches to the consumers
khenaidoo79232702018-12-04 11:00:41 -0500673func (kp *InterContainerProxy) waitForResponseLoop(subscribedCh <-chan *ic.InterContainerMessage, topic *Topic) {
khenaidoo43c82122018-11-22 18:38:28 -0500674 log.Debugw("starting-response-loop-for-topic", log.Fields{"topic": topic.Name})
khenaidooabad44c2018-08-03 16:58:35 -0400675startloop:
676 for {
677 select {
khenaidoo43c82122018-11-22 18:38:28 -0500678 case msg := <-subscribedCh:
679 //log.Debugw("message-received", log.Fields{"msg": msg, "fromTopic": msg.Header.FromTopic})
khenaidoo79232702018-12-04 11:00:41 -0500680 if msg.Header.Type == ic.MessageType_RESPONSE {
khenaidoo43c82122018-11-22 18:38:28 -0500681 go kp.dispatchResponse(msg)
682 }
khenaidooabad44c2018-08-03 16:58:35 -0400683 case <-kp.doneCh:
684 log.Infow("received-exit-signal", log.Fields{"topic": topic.Name})
685 break startloop
686 }
687 }
khenaidoo43c82122018-11-22 18:38:28 -0500688 //log.Infow("received-exit-signal-out-of-for-loop", log.Fields{"topic": topic.Name})
689 // We got an exit signal. Unsubscribe to the channel
690 //kp.kafkaClient.UnSubscribe(topic, subscribedCh)
khenaidooabad44c2018-08-03 16:58:35 -0400691}
692
693// subscribeForResponse allows a caller to subscribe to a given topic when waiting for a response.
694// This method is built to prevent all subscribers to receive all messages as is the case of the Subscribe
695// API. There is one response channel waiting for kafka messages before dispatching the message to the
696// corresponding waiting channel
khenaidoo79232702018-12-04 11:00:41 -0500697func (kp *InterContainerProxy) subscribeForResponse(topic Topic, trnsId string) (chan *ic.InterContainerMessage, error) {
khenaidoob9203542018-09-17 22:56:37 -0400698 log.Debugw("subscribeForResponse", log.Fields{"topic": topic.Name, "trnsid": trnsId})
khenaidooabad44c2018-08-03 16:58:35 -0400699
khenaidoo43c82122018-11-22 18:38:28 -0500700 // First check whether we already have a channel listening for response on that topic. If there is
701 // already one then it will be reused. If not, it will be created.
702 if !kp.isTopicSubscribedForResponse(topic.Name) {
khenaidoo79232702018-12-04 11:00:41 -0500703 var subscribedCh <-chan *ic.InterContainerMessage
khenaidooabad44c2018-08-03 16:58:35 -0400704 var err error
khenaidoo4c1a5bf2018-11-29 15:53:42 -0500705 if subscribedCh, err = kp.kafkaClient.Subscribe(&topic); err != nil {
khenaidoo43c82122018-11-22 18:38:28 -0500706 log.Debugw("subscribe-failure", log.Fields{"topic": topic.Name})
khenaidooabad44c2018-08-03 16:58:35 -0400707 return nil, err
708 }
khenaidoo43c82122018-11-22 18:38:28 -0500709 kp.setupTopicResponseChannelMap(topic.Name, subscribedCh)
710 go kp.waitForResponseLoop(subscribedCh, &topic)
khenaidooabad44c2018-08-03 16:58:35 -0400711 }
712
khenaidoo4c1a5bf2018-11-29 15:53:42 -0500713 // Create a specific channel for this consumers. We cannot use the channel from the kafkaclient as it will
khenaidoo43c82122018-11-22 18:38:28 -0500714 // broadcast any message for this topic to all channels waiting on it.
khenaidoo79232702018-12-04 11:00:41 -0500715 ch := make(chan *ic.InterContainerMessage)
khenaidoo43c82122018-11-22 18:38:28 -0500716 kp.addToTransactionIdToChannelMap(trnsId, &topic, ch)
khenaidooabad44c2018-08-03 16:58:35 -0400717
718 return ch, nil
719}
720
khenaidoo43c82122018-11-22 18:38:28 -0500721func (kp *InterContainerProxy) unSubscribeForResponse(trnsId string) error {
khenaidooabad44c2018-08-03 16:58:35 -0400722 log.Debugw("unsubscribe-for-response", log.Fields{"trnsId": trnsId})
khenaidoo43c82122018-11-22 18:38:28 -0500723 if _, exist := kp.transactionIdToChannelMap[trnsId]; exist {
724 // The delete operation will close the channel
725 kp.deleteFromTransactionIdToChannelMap(trnsId)
726 }
khenaidooabad44c2018-08-03 16:58:35 -0400727 return nil
728}
729
730//formatRequest formats a request to send over kafka and returns an InterContainerMessage message on success
731//or an error on failure
khenaidoo79232702018-12-04 11:00:41 -0500732func encodeRequest(rpc string, toTopic *Topic, replyTopic *Topic, kvArgs ...*KVArg) (*ic.InterContainerMessage, error) {
733 requestHeader := &ic.Header{
khenaidooabad44c2018-08-03 16:58:35 -0400734 Id: uuid.New().String(),
khenaidoo79232702018-12-04 11:00:41 -0500735 Type: ic.MessageType_REQUEST,
khenaidooabad44c2018-08-03 16:58:35 -0400736 FromTopic: replyTopic.Name,
737 ToTopic: toTopic.Name,
738 Timestamp: time.Now().Unix(),
739 }
khenaidoo79232702018-12-04 11:00:41 -0500740 requestBody := &ic.InterContainerRequestBody{
khenaidooabad44c2018-08-03 16:58:35 -0400741 Rpc: rpc,
742 ResponseRequired: true,
743 ReplyToTopic: replyTopic.Name,
744 }
745
746 for _, arg := range kvArgs {
khenaidoo2c6f1672018-09-20 23:14:41 -0400747 if arg == nil {
748 // In case the caller sends an array with empty args
749 continue
750 }
khenaidooabad44c2018-08-03 16:58:35 -0400751 var marshalledArg *any.Any
752 var err error
753 // ascertain the value interface type is a proto.Message
754 protoValue, ok := arg.Value.(proto.Message)
755 if !ok {
756 log.Warnw("argument-value-not-proto-message", log.Fields{"error": ok, "Value": arg.Value})
757 err := errors.New("argument-value-not-proto-message")
758 return nil, err
759 }
760 if marshalledArg, err = ptypes.MarshalAny(protoValue); err != nil {
761 log.Warnw("cannot-marshal-request", log.Fields{"error": err})
762 return nil, err
763 }
khenaidoo79232702018-12-04 11:00:41 -0500764 protoArg := &ic.Argument{
khenaidooabad44c2018-08-03 16:58:35 -0400765 Key: arg.Key,
766 Value: marshalledArg,
767 }
768 requestBody.Args = append(requestBody.Args, protoArg)
769 }
770
771 var marshalledData *any.Any
772 var err error
773 if marshalledData, err = ptypes.MarshalAny(requestBody); err != nil {
774 log.Warnw("cannot-marshal-request", log.Fields{"error": err})
775 return nil, err
776 }
khenaidoo79232702018-12-04 11:00:41 -0500777 request := &ic.InterContainerMessage{
khenaidooabad44c2018-08-03 16:58:35 -0400778 Header: requestHeader,
779 Body: marshalledData,
780 }
781 return request, nil
782}
783
khenaidoo79232702018-12-04 11:00:41 -0500784func decodeResponse(response *ic.InterContainerMessage) (*ic.InterContainerResponseBody, error) {
khenaidooabad44c2018-08-03 16:58:35 -0400785 // Extract the message body
khenaidoo79232702018-12-04 11:00:41 -0500786 responseBody := ic.InterContainerResponseBody{}
khenaidooabad44c2018-08-03 16:58:35 -0400787 if err := ptypes.UnmarshalAny(response.Body, &responseBody); err != nil {
788 log.Warnw("cannot-unmarshal-response", log.Fields{"error": err})
789 return nil, err
790 }
khenaidoo43c82122018-11-22 18:38:28 -0500791 //log.Debugw("response-decoded-successfully", log.Fields{"response-status": &responseBody.Success})
khenaidooabad44c2018-08-03 16:58:35 -0400792
793 return &responseBody, nil
794
795}