blob: 443a55eaf5821178e9ba0603f005336db269bbc4 [file] [log] [blame]
sslobodr392ebd52019-01-18 12:41:49 -05001/*
2 * Copyright 2018-present Open Networking Foundation
3
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7
8 * http://www.apache.org/licenses/LICENSE-2.0
9
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16// gRPC affinity router with active/active backends
17
18package afrouter
19
20import (
21 "fmt"
22 "errors"
23 "strconv"
24 "io/ioutil"
25 "google.golang.org/grpc"
26 "github.com/golang/protobuf/proto"
27 "github.com/opencord/voltha-go/common/log"
28 pb "github.com/golang/protobuf/protoc-gen-go/descriptor"
29)
30
31type AffinityRouter struct {
32 name string
33 routerType int // TODO: This is probably not needed
34 association int
35 routingField string
36 grpcService string
37 protoDescriptor *pb.FileDescriptorSet
38 methodMap map[string]byte
39 nbBindingMthdMap map[string]byte
40 bkndClstr *backendCluster
41 affinity map[string]*backend
42 curBknd **backend
43}
44
sslobodrcd37bc52019-01-24 11:47:16 -050045func newAffinityRouter(rconf *RouterConfig, config *RouteConfig) (Router,error) {
sslobodr392ebd52019-01-18 12:41:49 -050046 var err error = nil
47 var rtrn_err bool = false
48 // Validate the configuration
49
50 // A name must exist
51 if config.Name == "" {
52 log.Error("A router 'name' must be specified")
53 rtrn_err = true
54 }
55
56 if rconf.ProtoPackage == "" {
57 log.Error("A 'package' must be specified")
58 rtrn_err = true
59 }
60
61 if rconf.ProtoService == "" {
62 log.Error("A 'service' must be specified")
63 rtrn_err = true
64 }
65
66 //if config.RouteField == "" {
67 // log.Error("A 'routing_field' must be specified")
68 // rtrn_err = true
69 //}
70
71 // TODO The overrieds section is currently not being used
72 // so the router will route all methods based on the
73 // routing_field. This needs to be added so that methods
74 // can have different routing fields.
75 var bptr *backend
76 bptr = nil
77 dr := AffinityRouter{
78 name:config.Name,
79 grpcService:rconf.ProtoService,
80 affinity:make(map[string]*backend),
81 methodMap:make(map[string]byte),
82 nbBindingMthdMap:make(map[string]byte),
83 curBknd:&bptr,
84 //serialNo:0,
85 }
86 // An association must exist
87 dr.association = strIndex(rAssnNames, config.Association)
88 if dr.association == 0 {
89 if config.Association == "" {
90 log.Error("An association must be specified")
91 } else {
92 log.Errorf("The association '%s' is not valid", config.Association)
93 }
94 rtrn_err = true
95 }
96
97
98 // This has already been validated bfore this function
99 // is called so just use it.
sslobodr5f0b5a32019-01-24 07:45:19 -0500100 for idx := range rTypeNames {
sslobodr392ebd52019-01-18 12:41:49 -0500101 if config.Type == rTypeNames[idx] {
102 dr.routerType = idx
103 break
104 }
105 }
106
107 // Load the protobuf descriptor file
108 dr.protoDescriptor = &pb.FileDescriptorSet{}
109 fb, err := ioutil.ReadFile(config.ProtoFile);
110 if err != nil {
111 log.Errorf("Could not open proto file '%s'",config.ProtoFile)
112 rtrn_err = true
113 }
114 err = proto.Unmarshal(fb, dr.protoDescriptor)
115 if err != nil {
116 log.Errorf("Could not unmarshal %s, %v", "proto.pb", err)
117 rtrn_err = true
118 }
119
120
121 // Build the routing structure based on the loaded protobuf
122 // descriptor file and the config information.
123 type key struct {
124 mthd string
125 field string
126 }
127 var msgs map[key]byte = make(map[key]byte)
sslobodr5f0b5a32019-01-24 07:45:19 -0500128 for _,f := range dr.protoDescriptor.File {
sslobodr392ebd52019-01-18 12:41:49 -0500129 // Build a temporary map of message types by name.
sslobodr5f0b5a32019-01-24 07:45:19 -0500130 for _,m := range f.MessageType {
131 for _,fld := range m.Field {
sslobodr392ebd52019-01-18 12:41:49 -0500132 log.Debugf("Processing message '%s', field '%s'", *m.Name, *fld.Name)
133 msgs[key{*m.Name, *fld.Name}] = byte(*fld.Number)
134 }
135 }
136 }
137 log.Debugf("The map contains: %v", msgs)
sslobodr5f0b5a32019-01-24 07:45:19 -0500138 for _,f := range dr.protoDescriptor.File {
sslobodr392ebd52019-01-18 12:41:49 -0500139 if *f.Package == rconf.ProtoPackage {
sslobodr5f0b5a32019-01-24 07:45:19 -0500140 for _, s:= range f.Service {
sslobodr392ebd52019-01-18 12:41:49 -0500141 if *s.Name == rconf.ProtoService {
142 log.Debugf("Loading package data '%s' for service '%s' for router '%s'", *f.Package, *s.Name, dr.name)
143 // Now create a map keyed by method name with the value being the
144 // field number of the route selector.
145 var ok bool
sslobodr5f0b5a32019-01-24 07:45:19 -0500146 for _,m := range s.Method {
sslobodr392ebd52019-01-18 12:41:49 -0500147 // Find the input type in the messages and extract the
148 // field number and save it for future reference.
149 log.Debugf("Processing method '%s'",*m.Name)
150 // Determine if this is a method we're supposed to be processing.
151 if needMethod(*m.Name, config) == true {
152 log.Debugf("Enabling method '%s'",*m.Name)
153 // The input type has the package name prepended to it. Remove it.
154 in := (*m.InputType)[len(rconf.ProtoPackage)+2:]
155 dr.methodMap[*m.Name], ok = msgs[key{in, config.RouteField}]
156 if ok == false {
157 log.Errorf("Method '%s' has no field named '%s' in it's parameter message '%s'",
158 *m.Name, config.RouteField, in)
159 rtrn_err = true
160 }
161 }
162 // The sb method is always included in the methods so we can check it here too.
163 if needSbMethod(*m.Name, config) == true {
164 log.Debugf("Enabling southbound method '%s'",*m.Name)
165 // The output type has the package name prepended to it. Remove it.
166 out := (*m.OutputType)[len(rconf.ProtoPackage)+2:]
167 dr.nbBindingMthdMap[*m.Name], ok = msgs[key{out, config.RouteField}]
168 if ok == false {
169 log.Errorf("Method '%s' has no field named '%s' in it's parameter message '%s'",
170 *m.Name, config.RouteField, out)
171 rtrn_err = true
172 }
173 }
174 }
175 }
176 }
177 }
178 }
179
180
181 // Create the backend cluster or link to an existing one
182 ok := true
183 if dr.bkndClstr, ok = bClusters[config.backendCluster.Name]; ok == false {
sslobodrcd37bc52019-01-24 11:47:16 -0500184 if dr.bkndClstr, err = newBackendCluster(config.backendCluster); err != nil {
sslobodr392ebd52019-01-18 12:41:49 -0500185 log.Errorf("Could not create a backend for router %s", config.Name)
186 rtrn_err = true
187 }
188 }
189
190 if rtrn_err {
191 return dr,errors.New(fmt.Sprintf("Failed to create a new router '%s'",dr.name))
192 }
193
194 return dr,nil
195}
196
197func needSbMethod(mthd string, conf *RouteConfig) bool {
198 for _,m := range conf.NbBindingMethods {
199 if mthd == m {
200 return true
201 }
202 }
203 return false
204}
205
206func needMethod(mthd string, conf *RouteConfig) bool {
207 for _,m := range conf.Methods {
208 if mthd == m {
209 return true
210 }
211 }
212 return false
213}
214
215func (r AffinityRouter) Service() (string) {
216 return r.grpcService
217}
218
219func (r AffinityRouter) Name() (string) {
220 return r.name
221}
222
223func (r AffinityRouter) skipField(data *[]byte, idx *int) (error) {
224 switch (*data)[*idx]&3 {
225 case 0: // Varint
226 (*idx)++
227 for (*data)[*idx] >= 128 { (*idx)++}
228 case 1: // 64 bit
229 (*idx)+= 9
230 case 2: // Length delimited
231 (*idx)++
232 b := proto.NewBuffer((*data)[*idx:])
233 t , _ := b.DecodeVarint()
234 (*idx) += int(t)+1
235 case 3: // Deprecated
236 case 4: // Deprecated
237 case 5: // 32 bit
238 (*idx)+= 5
239 }
240 return nil
241}
242
243func (r AffinityRouter) decodeProtoField(payload []byte, fieldId byte) (string, error) {
244 idx :=0
245 b := proto.NewBuffer([]byte{})
246 b.DebugPrint("The Buffer", payload)
247 for { // Find the route selector field
248 log.Debugf("Decoding afinity value attributeNumber: %d from %v at index %d", fieldId, payload, idx)
249 log.Debugf("Attempting match with payload: %d, methodTable: %d", payload[idx], fieldId)
250 if payload[idx]>>3 == fieldId {
251 log.Debugf("Method match with payload: %d, methodTable: %d", payload[idx], fieldId)
252 // TODO: Consider supporting other selector types.... Way, way in the future
253 // ok, the future is now, support strings as well... ugh.
254 var selector string
255 switch payload[idx]&3 {
256 case 0: // Integer
257 b.SetBuf(payload[idx+1:])
258 v,e := b.DecodeVarint()
259 if e == nil {
260 log.Debugf("Decoded the ing field: %v", v)
261 selector = strconv.Itoa(int(v))
262 } else {
263 log.Errorf("Failed to decode varint %v", e)
264 return "", e
265 }
266 case 2: // Length delimited AKA string
267 b.SetBuf(payload[idx+1:])
268 v,e := b.DecodeStringBytes()
269 if e == nil {
270 log.Debugf("Decoded the string field: %v", v)
271 selector = v
272 } else {
273 log.Errorf("Failed to decode string %v", e)
274 return "", e
275 }
276 default:
277 err := errors.New(fmt.Sprintf("Only integer and string route selectors are permitted"))
278 log.Error(err)
279 return "", err
280 }
281 return selector, nil
282 } else if err := r.skipField(&payload, &idx); err != nil {
283 log.Errorf("Parsing message failed %v", err)
284 return "", err
285 }
286 }
287}
288
289func (r AffinityRouter) Route(sel interface{}) *backend {
290 switch sl := sel.(type) {
291 case *nbFrame:
292 log.Debugf("Route called for nbFrame with method %s", sl.mthdSlice[REQ_METHOD]);
293 // Check if this method should be affinity bound from the
294 // reply rather than the request.
295 if _,ok := r.nbBindingMthdMap[sl.mthdSlice[REQ_METHOD]]; ok == true {
296 var err error
297 log.Debugf("Method '%s' affinity binds on reply", sl.mthdSlice[REQ_METHOD])
298 // Just round robin route the southbound request
299 if *r.curBknd, err = r.bkndClstr.nextBackend(*r.curBknd,BE_SEQ_RR); err == nil {
300 return *r.curBknd
301 } else {
302 sl.err = err
303 return nil
304 }
305 }
306 // Not a south affinity binding method, proceed with north affinity binding.
307 if selector,err := r.decodeProtoField(sl.payload, r.methodMap[sl.mthdSlice[REQ_METHOD]]); err == nil {
308 if rtrn,ok := r.affinity[selector]; ok {
309 return rtrn
310 } else {
311 // The selector isn't in the map, create a new affinity mapping
312 log.Debugf("MUST CREATE A NEW AFFINITY MAP ENTRY!!")
313 var err error
314 if *r.curBknd, err = r.bkndClstr.nextBackend(*r.curBknd,BE_SEQ_RR); err == nil {
315 r.setAffinity(selector, *r.curBknd)
316 //r.affinity[selector] = *r.curBknd
317 //log.Debugf("New affinity set to backend %s",(*r.curBknd).name)
318 return *r.curBknd
319 } else {
320 sl.err = err
321 return nil
322 }
323 }
324 }
325 default:
326 log.Errorf("Internal: invalid data type in Route call %v", sel);
327 return nil
328 }
329 log.Errorf("Bad lookup in affinity map %v",r.affinity);
330 return nil
331}
332
333func (ar AffinityRouter) GetMetaKeyVal(serverStream grpc.ServerStream) (string,string,error) {
334 return "","",nil
335}
336
337func (ar AffinityRouter) BackendCluster(mthd string, metaKey string) (*backendCluster,error) {
338 return ar.bkndClstr, nil
339}
340
341func (ar AffinityRouter) FindBackendCluster(beName string) *backendCluster {
342 if beName == ar.bkndClstr.name {
343 return ar.bkndClstr
344 }
345 return nil
346}
347
348func (r AffinityRouter) ReplyHandler(sel interface{}) error {
349 switch sl := sel.(type) {
350 case *sbFrame:
351 sl.lck.Lock()
352 defer sl.lck.Unlock()
353 log.Debugf("Reply handler called for sbFrame with method %s", sl.method);
354 // Determine if reply action is required.
355 if fld, ok := r.nbBindingMthdMap[sl.method]; ok == true && len(sl.payload) > 0 {
356 // Extract the field value from the frame and
357 // and set affinity accordingly
358 if selector,err := r.decodeProtoField(sl.payload, fld); err == nil {
359 log.Debug("Settign affinity on reply")
360 if r.setAffinity(selector, sl.be) != nil {
361 log.Error("Setting affinity on reply failed")
362 }
363 return nil
364 } else {
365 err := errors.New(fmt.Sprintf("Failed to decode reply field %d for method %s", fld, sl.method))
366 log.Error(err)
367 return err
368 }
369 }
370 return nil
371 default:
372 err := errors.New(fmt.Sprintf("Internal: invalid data type in ReplyHander call %v", sl))
373 log.Error(err)
374 return err
375 }
376}
377
378func (ar AffinityRouter) setAffinity(key string, be *backend) error {
379 if be2,ok := ar.affinity[key]; ok == false {
380 ar.affinity[key] = be
381 log.Debugf("New affinity set to backend %s for key %s",be.name, key)
382 } else if be2 != be {
383 err := errors.New(fmt.Sprintf("Attempting multiple sets of affinity for key %s to backend %s from %s on router %s",
384 key, be.name, ar.affinity[key].name, ar.name))
385 log.Error(err)
386 return err
387 }
388 return nil
389}