sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2018-present Open Networking Foundation |
| 3 | |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 16 | |
| 17 | package afrouter |
| 18 | |
| 19 | // Backend manager handles redundant connections per backend |
| 20 | |
| 21 | import ( |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 22 | "errors" |
Kent Hagerman | 0ab4cb2 | 2019-04-24 13:13:35 -0400 | [diff] [blame] | 23 | "fmt" |
| 24 | "github.com/opencord/voltha-go/common/log" |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 25 | "golang.org/x/net/context" |
| 26 | "google.golang.org/grpc" |
| 27 | "google.golang.org/grpc/codes" |
Kent Hagerman | 0ab4cb2 | 2019-04-24 13:13:35 -0400 | [diff] [blame] | 28 | "google.golang.org/grpc/metadata" |
Kent Hagerman | fcfb16b | 2019-06-20 11:40:03 -0400 | [diff] [blame] | 29 | "net/url" |
Kent Hagerman | 0ab4cb2 | 2019-04-24 13:13:35 -0400 | [diff] [blame] | 30 | "strconv" |
| 31 | "strings" |
| 32 | "sync" |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 33 | ) |
| 34 | |
Kent Hagerman | 1e9061e | 2019-05-21 16:01:21 -0400 | [diff] [blame] | 35 | // backend represents a collection of backends in a HA configuration |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 36 | type backend struct { |
Kent Hagerman | 1e9061e | 2019-05-21 16:01:21 -0400 | [diff] [blame] | 37 | mutex sync.Mutex |
| 38 | name string |
| 39 | beType backendType |
| 40 | activeAssociation association |
| 41 | connFailCallback func(string, *backend) bool |
| 42 | connections map[string]*connection |
Kent Hagerman | 03b5899 | 2019-08-29 17:21:03 -0400 | [diff] [blame^] | 43 | openConns map[*connection]*grpc.ClientConn |
| 44 | activeRequests map[*request]struct{} |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 45 | } |
| 46 | |
Kent Hagerman | 1e9061e | 2019-05-21 16:01:21 -0400 | [diff] [blame] | 47 | type association struct { |
| 48 | strategy associationStrategy |
| 49 | location associationLocation |
Kent Hagerman | 0ab4cb2 | 2019-04-24 13:13:35 -0400 | [diff] [blame] | 50 | field string // Used only if location is protobuf |
| 51 | key string |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 52 | } |
| 53 | |
Kent Hagerman | 03b5899 | 2019-08-29 17:21:03 -0400 | [diff] [blame^] | 54 | // splitActiveStreamsUnsafe expects the caller to have already locked the backend mutex |
| 55 | func (be *backend) splitActiveStreamsUnsafe(cn *connection, conn *grpc.ClientConn) { |
| 56 | if len(be.activeRequests) != 0 { |
| 57 | log.Debugf("Creating new streams for %d existing requests", len(be.activeRequests)) |
| 58 | } |
| 59 | for r := range be.activeRequests { |
| 60 | r.mutex.Lock() |
| 61 | if _, have := r.streams[cn.name]; !have { |
| 62 | log.Debugf("Opening southbound stream for existing request '%s'", r.methodInfo.method) |
| 63 | if stream, err := grpc.NewClientStream(r.ctx, clientStreamDescForProxying, conn, r.methodInfo.all); err != nil { |
| 64 | log.Debugf("Failed to create a client stream '%s', %v", cn.name, err) |
| 65 | } else { |
| 66 | go r.catchupRequestStreamThenForwardResponseStream(cn.name, stream) |
| 67 | // new thread will unlock the request mutex |
| 68 | continue |
| 69 | } |
| 70 | } |
| 71 | r.mutex.Unlock() |
| 72 | } |
| 73 | } |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 74 | |
Kent Hagerman | 03b5899 | 2019-08-29 17:21:03 -0400 | [diff] [blame^] | 75 | // openSouthboundStreams sets up a connection to each southbound frame |
| 76 | func (be *backend) openSouthboundStreams(srv interface{}, serverStream grpc.ServerStream, nf *requestFrame, sf *responseFrame) (*request, error) { |
| 77 | be.mutex.Lock() |
| 78 | defer be.mutex.Unlock() |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 79 | |
Kent Hagerman | 03b5899 | 2019-08-29 17:21:03 -0400 | [diff] [blame^] | 80 | isStreamingRequest, isStreamingResponse := nf.router.IsStreaming(nf.methodInfo.method) |
| 81 | |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 82 | // Get the metadata from the incoming message on the server |
| 83 | md, ok := metadata.FromIncomingContext(serverStream.Context()) |
| 84 | if !ok { |
Kent Hagerman | 03b5899 | 2019-08-29 17:21:03 -0400 | [diff] [blame^] | 85 | return nil, errors.New("could not get a server stream metadata") |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 86 | } |
| 87 | |
Kent Hagerman | 03b5899 | 2019-08-29 17:21:03 -0400 | [diff] [blame^] | 88 | r := &request{ |
| 89 | // Create an outgoing context that includes the incoming metadata and that will cancel if the server's context is canceled |
| 90 | ctx: metadata.AppendToOutgoingContext(metadata.NewOutgoingContext(serverStream.Context(), md.Copy()), "voltha_serial_number", strconv.FormatUint(nf.serialNo, 10)), |
| 91 | |
| 92 | streams: make(map[string]grpc.ClientStream), |
| 93 | responseErrChan: make(chan error, 1), |
| 94 | |
| 95 | backend: be, |
| 96 | serverStream: serverStream, |
| 97 | methodInfo: nf.methodInfo, |
| 98 | requestFrame: nf, |
| 99 | responseFrame: sf, |
| 100 | isStreamingRequest: isStreamingRequest, |
| 101 | isStreamingResponse: isStreamingResponse, |
| 102 | } |
| 103 | |
| 104 | log.Debugf("Opening southbound request for method '%s'", nf.methodInfo.method) |
| 105 | |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 106 | // TODO: Need to check if this is an active/active backend cluster |
| 107 | // with a serial number in the header. |
Kent Hagerman | 03b5899 | 2019-08-29 17:21:03 -0400 | [diff] [blame^] | 108 | log.Debugf("Serial number for transaction allocated: %d", nf.serialNo) |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 109 | // If even one stream can be created then proceed. If none can be |
Kent Hagerman | 03b5899 | 2019-08-29 17:21:03 -0400 | [diff] [blame^] | 110 | // created then report an error because both the primary and redundant |
| 111 | // connections are non-existent. |
Kent Hagerman | 1e9061e | 2019-05-21 16:01:21 -0400 | [diff] [blame] | 112 | var atLeastOne = false |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 113 | var errStr strings.Builder |
Kent Hagerman | 03b5899 | 2019-08-29 17:21:03 -0400 | [diff] [blame^] | 114 | log.Debugf("There are %d/%d streams to open", len(be.openConns), len(be.connections)) |
| 115 | for cn, conn := range be.openConns { |
| 116 | log.Debugf("Opening stream for connection '%s'", cn.name) |
| 117 | if stream, err := grpc.NewClientStream(r.ctx, clientStreamDescForProxying, conn, r.methodInfo.all); err != nil { |
| 118 | log.Debugf("Failed to create a client stream '%s', %v", cn.name, err) |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 119 | } else { |
Kent Hagerman | 03b5899 | 2019-08-29 17:21:03 -0400 | [diff] [blame^] | 120 | r.streams[cn.name] = stream |
| 121 | go r.forwardResponseStream(cn.name, stream) |
| 122 | atLeastOne = true |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 123 | } |
| 124 | } |
Kent Hagerman | 1e9061e | 2019-05-21 16:01:21 -0400 | [diff] [blame] | 125 | if atLeastOne { |
Kent Hagerman | 03b5899 | 2019-08-29 17:21:03 -0400 | [diff] [blame^] | 126 | be.activeRequests[r] = struct{}{} |
| 127 | return r, nil |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 128 | } |
Kent Hagerman | 03b5899 | 2019-08-29 17:21:03 -0400 | [diff] [blame^] | 129 | fmt.Fprintf(&errStr, "{{No open connections for backend '%s' unable to send}} ", be.name) |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 130 | log.Error(errStr.String()) |
| 131 | return nil, errors.New(errStr.String()) |
| 132 | } |
| 133 | |
Kent Hagerman | 03b5899 | 2019-08-29 17:21:03 -0400 | [diff] [blame^] | 134 | func (be *backend) handler(srv interface{}, serverStream grpc.ServerStream, nf *requestFrame, sf *responseFrame) error { |
| 135 | // Set up streams for each open connection |
| 136 | request, err := be.openSouthboundStreams(srv, serverStream, nf, sf) |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 137 | if err != nil { |
Kent Hagerman | 0ab4cb2 | 2019-04-24 13:13:35 -0400 | [diff] [blame] | 138 | log.Errorf("openStreams failed: %v", err) |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 139 | return err |
| 140 | } |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 141 | |
Kent Hagerman | 03b5899 | 2019-08-29 17:21:03 -0400 | [diff] [blame^] | 142 | log.Debug("Starting request stream forwarding") |
| 143 | if s2cErr := request.forwardRequestStream(serverStream); s2cErr != nil { |
| 144 | // exit with an error to the stack |
| 145 | return grpc.Errorf(codes.Internal, "failed proxying s2c: %v", s2cErr) |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 146 | } |
Kent Hagerman | 03b5899 | 2019-08-29 17:21:03 -0400 | [diff] [blame^] | 147 | // wait for response stream to complete |
| 148 | return <-request.responseErrChan |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 149 | } |
| 150 | |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 151 | func newBackend(conf *BackendConfig, clusterName string) (*backend, error) { |
| 152 | var rtrn_err bool = false |
| 153 | |
| 154 | log.Debugf("Configuring the backend with %v", *conf) |
| 155 | // Validate the conifg and configure the backend |
Kent Hagerman | 03b5899 | 2019-08-29 17:21:03 -0400 | [diff] [blame^] | 156 | be := &backend{ |
| 157 | name: conf.Name, |
| 158 | connections: make(map[string]*connection), |
| 159 | openConns: make(map[*connection]*grpc.ClientConn), |
| 160 | activeRequests: make(map[*request]struct{}), |
| 161 | } |
Kent Hagerman | 1e9061e | 2019-05-21 16:01:21 -0400 | [diff] [blame] | 162 | if conf.Type == BackendUndefined { |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 163 | log.Error("Invalid type specified for backend %s in cluster %s", conf.Name, clusterName) |
| 164 | rtrn_err = true |
| 165 | } |
Kent Hagerman | 1e9061e | 2019-05-21 16:01:21 -0400 | [diff] [blame] | 166 | be.beType = conf.Type |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 167 | |
Kent Hagerman | 1e9061e | 2019-05-21 16:01:21 -0400 | [diff] [blame] | 168 | if conf.Association.Strategy == AssociationStrategyUndefined && be.beType == BackendActiveActive { |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 169 | log.Errorf("An association strategy must be provided if the backend "+ |
Kent Hagerman | 0ab4cb2 | 2019-04-24 13:13:35 -0400 | [diff] [blame] | 170 | "type is active/active for backend %s in cluster %s", conf.Name, clusterName) |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 171 | rtrn_err = true |
| 172 | } |
Kent Hagerman | 1e9061e | 2019-05-21 16:01:21 -0400 | [diff] [blame] | 173 | be.activeAssociation.strategy = conf.Association.Strategy |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 174 | |
Kent Hagerman | 1e9061e | 2019-05-21 16:01:21 -0400 | [diff] [blame] | 175 | if conf.Association.Location == AssociationLocationUndefined && be.beType == BackendActiveActive { |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 176 | log.Errorf("An association location must be provided if the backend "+ |
Kent Hagerman | 0ab4cb2 | 2019-04-24 13:13:35 -0400 | [diff] [blame] | 177 | "type is active/active for backend %s in cluster %s", conf.Name, clusterName) |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 178 | rtrn_err = true |
| 179 | } |
Kent Hagerman | 1e9061e | 2019-05-21 16:01:21 -0400 | [diff] [blame] | 180 | be.activeAssociation.location = conf.Association.Location |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 181 | |
Kent Hagerman | 1e9061e | 2019-05-21 16:01:21 -0400 | [diff] [blame] | 182 | if conf.Association.Field == "" && be.activeAssociation.location == AssociationLocationProtobuf { |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 183 | log.Errorf("An association field must be provided if the backend "+ |
Kent Hagerman | 0ab4cb2 | 2019-04-24 13:13:35 -0400 | [diff] [blame] | 184 | "type is active/active and the location is set to protobuf "+ |
| 185 | "for backend %s in cluster %s", conf.Name, clusterName) |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 186 | rtrn_err = true |
| 187 | } |
Kent Hagerman | 1e9061e | 2019-05-21 16:01:21 -0400 | [diff] [blame] | 188 | be.activeAssociation.field = conf.Association.Field |
sslobodr | 8e2ccb5 | 2019-02-05 09:21:47 -0500 | [diff] [blame] | 189 | |
Kent Hagerman | 1e9061e | 2019-05-21 16:01:21 -0400 | [diff] [blame] | 190 | if conf.Association.Key == "" && be.activeAssociation.location == AssociationLocationHeader { |
sslobodr | 8e2ccb5 | 2019-02-05 09:21:47 -0500 | [diff] [blame] | 191 | log.Errorf("An association key must be provided if the backend "+ |
Kent Hagerman | 0ab4cb2 | 2019-04-24 13:13:35 -0400 | [diff] [blame] | 192 | "type is active/active and the location is set to header "+ |
| 193 | "for backend %s in cluster %s", conf.Name, clusterName) |
sslobodr | 8e2ccb5 | 2019-02-05 09:21:47 -0500 | [diff] [blame] | 194 | rtrn_err = true |
| 195 | } |
Kent Hagerman | 1e9061e | 2019-05-21 16:01:21 -0400 | [diff] [blame] | 196 | be.activeAssociation.key = conf.Association.Key |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 197 | if rtrn_err { |
| 198 | return nil, errors.New("Backend configuration failed") |
| 199 | } |
| 200 | // Configure the connections |
| 201 | // Connections can consist of just a name. This allows for dynamic configuration |
| 202 | // at a later time. |
| 203 | // TODO: validate that there is one connection for all but active/active backends |
Kent Hagerman | 1e9061e | 2019-05-21 16:01:21 -0400 | [diff] [blame] | 204 | if len(conf.Connections) > 1 && be.beType != BackendActiveActive { |
Kent Hagerman | 0ab4cb2 | 2019-04-24 13:13:35 -0400 | [diff] [blame] | 205 | log.Errorf("Only one connection must be specified if the association " + |
| 206 | "strategy is not set to 'active_active'") |
sslobodr | 8e2ccb5 | 2019-02-05 09:21:47 -0500 | [diff] [blame] | 207 | rtrn_err = true |
| 208 | } |
| 209 | if len(conf.Connections) == 0 { |
| 210 | log.Errorf("At least one connection must be specified") |
| 211 | rtrn_err = true |
| 212 | } |
Kent Hagerman | 0ab4cb2 | 2019-04-24 13:13:35 -0400 | [diff] [blame] | 213 | for _, cnConf := range conf.Connections { |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 214 | if cnConf.Name == "" { |
| 215 | log.Errorf("A connection must have a name for backend %s in cluster %s", |
Kent Hagerman | 0ab4cb2 | 2019-04-24 13:13:35 -0400 | [diff] [blame] | 216 | conf.Name, clusterName) |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 217 | } else { |
Kent Hagerman | 03b5899 | 2019-08-29 17:21:03 -0400 | [diff] [blame^] | 218 | ctx, cancelFunc := context.WithCancel(context.Background()) |
| 219 | be.connections[cnConf.Name] = &connection{name: cnConf.Name, addr: cnConf.Addr, port: cnConf.Port, backend: be, ctx: ctx, close: cancelFunc} |
| 220 | if _, err := url.Parse(cnConf.Addr); err != nil { |
| 221 | log.Errorf("The address for connection %s in backend %s in cluster %s is invalid: %s", |
| 222 | cnConf.Name, conf.Name, clusterName, err) |
| 223 | rtrn_err = true |
| 224 | } |
| 225 | // Validate the port number. This just validtes that it's a non 0 integer |
| 226 | if n, err := strconv.Atoi(cnConf.Port); err != nil || n <= 0 || n > 65535 { |
| 227 | log.Errorf("Port %s for connection %s in backend %s in cluster %s is invalid", |
| 228 | cnConf.Port, cnConf.Name, conf.Name, clusterName) |
| 229 | rtrn_err = true |
| 230 | } else { |
| 231 | if n <= 0 && n > 65535 { |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 232 | log.Errorf("Port %s for connection %s in backend %s in cluster %s is invalid", |
| 233 | cnConf.Port, cnConf.Name, conf.Name, clusterName) |
| 234 | rtrn_err = true |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 235 | } |
| 236 | } |
| 237 | } |
| 238 | } |
sslobodr | 63d160c | 2019-02-08 14:25:13 -0500 | [diff] [blame] | 239 | |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 240 | if rtrn_err { |
| 241 | return nil, errors.New("Connection configuration failed") |
| 242 | } |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 243 | // All is well start the backend cluster connections |
| 244 | be.connectAll() |
| 245 | |
| 246 | return be, nil |
| 247 | } |
| 248 | |
Kent Hagerman | 03b5899 | 2019-08-29 17:21:03 -0400 | [diff] [blame^] | 249 | func (be *backend) incConn(cn *connection, conn *grpc.ClientConn) { |
Kent Hagerman | 1e9061e | 2019-05-21 16:01:21 -0400 | [diff] [blame] | 250 | be.mutex.Lock() |
| 251 | defer be.mutex.Unlock() |
Kent Hagerman | 03b5899 | 2019-08-29 17:21:03 -0400 | [diff] [blame^] | 252 | |
| 253 | be.openConns[cn] = conn |
| 254 | be.splitActiveStreamsUnsafe(cn, conn) |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 255 | } |
| 256 | |
Kent Hagerman | 03b5899 | 2019-08-29 17:21:03 -0400 | [diff] [blame^] | 257 | func (be *backend) decConn(cn *connection) { |
Kent Hagerman | 1e9061e | 2019-05-21 16:01:21 -0400 | [diff] [blame] | 258 | be.mutex.Lock() |
| 259 | defer be.mutex.Unlock() |
Kent Hagerman | 03b5899 | 2019-08-29 17:21:03 -0400 | [diff] [blame^] | 260 | |
| 261 | delete(be.openConns, cn) |
| 262 | } |
| 263 | |
| 264 | func (be *backend) NumOpenConnections() int { |
| 265 | be.mutex.Lock() |
| 266 | defer be.mutex.Unlock() |
| 267 | |
| 268 | return len(be.openConns) |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 269 | } |
| 270 | |
| 271 | // Attempts to establish all the connections for a backend |
| 272 | // any failures result in an abort. This should only be called |
| 273 | // on a first attempt to connect. Individual connections should be |
| 274 | // handled after that. |
| 275 | func (be *backend) connectAll() { |
Kent Hagerman | 0ab4cb2 | 2019-04-24 13:13:35 -0400 | [diff] [blame] | 276 | for _, cn := range be.connections { |
Kent Hagerman | 03b5899 | 2019-08-29 17:21:03 -0400 | [diff] [blame^] | 277 | go cn.connect() |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 278 | } |
| 279 | } |
| 280 | |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 281 | // Set a callback for connection failure notification |
| 282 | // This is currently not used. |
Kent Hagerman | 1e9061e | 2019-05-21 16:01:21 -0400 | [diff] [blame] | 283 | func (be *backend) setConnFailCallback(cb func(string, *backend) bool) { |
| 284 | be.connFailCallback = cb |
sslobodr | 392ebd5 | 2019-01-18 12:41:49 -0500 | [diff] [blame] | 285 | } |