blob: d02a7eec7c3765c11e326820571bc3768512767f [file] [log] [blame]
William Kurkianea869482019-04-09 15:16:11 -04001// Copyright 2018 The etcd Authors
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
Devmalya Pauleb2e9552019-08-27 19:42:00 -040015// Package balancer implements client balancer.
William Kurkianea869482019-04-09 15:16:11 -040016package balancer
17
18import (
William Kurkianea869482019-04-09 15:16:11 -040019 "strconv"
20 "sync"
21 "time"
22
Devmalya Pauleb2e9552019-08-27 19:42:00 -040023 "go.etcd.io/etcd/clientv3/balancer/connectivity"
William Kurkianea869482019-04-09 15:16:11 -040024 "go.etcd.io/etcd/clientv3/balancer/picker"
25
26 "go.uber.org/zap"
27 "google.golang.org/grpc/balancer"
Devmalya Pauleb2e9552019-08-27 19:42:00 -040028 grpcconnectivity "google.golang.org/grpc/connectivity"
William Kurkianea869482019-04-09 15:16:11 -040029 "google.golang.org/grpc/resolver"
30 _ "google.golang.org/grpc/resolver/dns" // register DNS resolver
31 _ "google.golang.org/grpc/resolver/passthrough" // register passthrough resolver
32)
33
Devmalya Pauleb2e9552019-08-27 19:42:00 -040034// Config defines balancer configurations.
35type Config struct {
36 // Policy configures balancer policy.
37 Policy picker.Policy
38
39 // Picker implements gRPC picker.
40 // Leave empty if "Policy" field is not custom.
41 // TODO: currently custom policy is not supported.
42 // Picker picker.Picker
43
44 // Name defines an additional name for balancer.
45 // Useful for balancer testing to avoid register conflicts.
46 // If empty, defaults to policy name.
47 Name string
48
49 // Logger configures balancer logging.
50 // If nil, logs are discarded.
51 Logger *zap.Logger
52}
53
William Kurkianea869482019-04-09 15:16:11 -040054// RegisterBuilder creates and registers a builder. Since this function calls balancer.Register, it
55// must be invoked at initialization time.
56func RegisterBuilder(cfg Config) {
57 bb := &builder{cfg}
58 balancer.Register(bb)
59
60 bb.cfg.Logger.Debug(
61 "registered balancer",
62 zap.String("policy", bb.cfg.Policy.String()),
63 zap.String("name", bb.cfg.Name),
64 )
65}
66
67type builder struct {
68 cfg Config
69}
70
71// Build is called initially when creating "ccBalancerWrapper".
72// "grpc.Dial" is called to this client connection.
73// Then, resolved addresses will be handled via "HandleResolvedAddrs".
74func (b *builder) Build(cc balancer.ClientConn, opt balancer.BuildOptions) balancer.Balancer {
75 bb := &baseBalancer{
76 id: strconv.FormatInt(time.Now().UnixNano(), 36),
77 policy: b.cfg.Policy,
Abhilash S.L3b494632019-07-16 15:51:09 +053078 name: b.cfg.Name,
William Kurkianea869482019-04-09 15:16:11 -040079 lg: b.cfg.Logger,
80
81 addrToSc: make(map[resolver.Address]balancer.SubConn),
82 scToAddr: make(map[balancer.SubConn]resolver.Address),
Devmalya Pauleb2e9552019-08-27 19:42:00 -040083 scToSt: make(map[balancer.SubConn]grpcconnectivity.State),
William Kurkianea869482019-04-09 15:16:11 -040084
Devmalya Pauleb2e9552019-08-27 19:42:00 -040085 currentConn: nil,
86 connectivityRecorder: connectivity.New(b.cfg.Logger),
William Kurkianea869482019-04-09 15:16:11 -040087
88 // initialize picker always returns "ErrNoSubConnAvailable"
Devmalya Pauleb2e9552019-08-27 19:42:00 -040089 picker: picker.NewErr(balancer.ErrNoSubConnAvailable),
William Kurkianea869482019-04-09 15:16:11 -040090 }
91
92 // TODO: support multiple connections
93 bb.mu.Lock()
94 bb.currentConn = cc
95 bb.mu.Unlock()
96
97 bb.lg.Info(
98 "built balancer",
99 zap.String("balancer-id", bb.id),
100 zap.String("policy", bb.policy.String()),
101 zap.String("resolver-target", cc.Target()),
102 )
103 return bb
104}
105
106// Name implements "grpc/balancer.Builder" interface.
107func (b *builder) Name() string { return b.cfg.Name }
108
109// Balancer defines client balancer interface.
110type Balancer interface {
111 // Balancer is called on specified client connection. Client initiates gRPC
112 // connection with "grpc.Dial(addr, grpc.WithBalancerName)", and then those resolved
113 // addresses are passed to "grpc/balancer.Balancer.HandleResolvedAddrs".
114 // For each resolved address, balancer calls "balancer.ClientConn.NewSubConn".
115 // "grpc/balancer.Balancer.HandleSubConnStateChange" is called when connectivity state
116 // changes, thus requires failover logic in this method.
117 balancer.Balancer
118
119 // Picker calls "Pick" for every client request.
120 picker.Picker
121}
122
123type baseBalancer struct {
124 id string
125 policy picker.Policy
126 name string
127 lg *zap.Logger
128
129 mu sync.RWMutex
130
131 addrToSc map[resolver.Address]balancer.SubConn
132 scToAddr map[balancer.SubConn]resolver.Address
Devmalya Pauleb2e9552019-08-27 19:42:00 -0400133 scToSt map[balancer.SubConn]grpcconnectivity.State
William Kurkianea869482019-04-09 15:16:11 -0400134
Devmalya Pauleb2e9552019-08-27 19:42:00 -0400135 currentConn balancer.ClientConn
136 connectivityRecorder connectivity.Recorder
William Kurkianea869482019-04-09 15:16:11 -0400137
Devmalya Pauleb2e9552019-08-27 19:42:00 -0400138 picker picker.Picker
William Kurkianea869482019-04-09 15:16:11 -0400139}
140
141// HandleResolvedAddrs implements "grpc/balancer.Balancer" interface.
142// gRPC sends initial or updated resolved addresses from "Build".
143func (bb *baseBalancer) HandleResolvedAddrs(addrs []resolver.Address, err error) {
144 if err != nil {
145 bb.lg.Warn("HandleResolvedAddrs called with error", zap.String("balancer-id", bb.id), zap.Error(err))
146 return
147 }
Devmalya Pauleb2e9552019-08-27 19:42:00 -0400148 bb.lg.Info("resolved",
149 zap.String("picker", bb.picker.String()),
150 zap.String("balancer-id", bb.id),
151 zap.Strings("addresses", addrsToStrings(addrs)),
152 )
William Kurkianea869482019-04-09 15:16:11 -0400153
154 bb.mu.Lock()
155 defer bb.mu.Unlock()
156
157 resolved := make(map[resolver.Address]struct{})
158 for _, addr := range addrs {
159 resolved[addr] = struct{}{}
160 if _, ok := bb.addrToSc[addr]; !ok {
161 sc, err := bb.currentConn.NewSubConn([]resolver.Address{addr}, balancer.NewSubConnOptions{})
162 if err != nil {
Devmalya Pauleb2e9552019-08-27 19:42:00 -0400163 bb.lg.Warn("NewSubConn failed", zap.String("picker", bb.picker.String()), zap.String("balancer-id", bb.id), zap.Error(err), zap.String("address", addr.Addr))
William Kurkianea869482019-04-09 15:16:11 -0400164 continue
165 }
Devmalya Pauleb2e9552019-08-27 19:42:00 -0400166 bb.lg.Info("created subconn", zap.String("address", addr.Addr))
William Kurkianea869482019-04-09 15:16:11 -0400167 bb.addrToSc[addr] = sc
168 bb.scToAddr[sc] = addr
Devmalya Pauleb2e9552019-08-27 19:42:00 -0400169 bb.scToSt[sc] = grpcconnectivity.Idle
William Kurkianea869482019-04-09 15:16:11 -0400170 sc.Connect()
171 }
172 }
173
174 for addr, sc := range bb.addrToSc {
175 if _, ok := resolved[addr]; !ok {
176 // was removed by resolver or failed to create subconn
177 bb.currentConn.RemoveSubConn(sc)
178 delete(bb.addrToSc, addr)
179
180 bb.lg.Info(
181 "removed subconn",
Devmalya Pauleb2e9552019-08-27 19:42:00 -0400182 zap.String("picker", bb.picker.String()),
William Kurkianea869482019-04-09 15:16:11 -0400183 zap.String("balancer-id", bb.id),
184 zap.String("address", addr.Addr),
185 zap.String("subconn", scToString(sc)),
186 )
187
188 // Keep the state of this sc in bb.scToSt until sc's state becomes Shutdown.
189 // The entry will be deleted in HandleSubConnStateChange.
190 // (DO NOT) delete(bb.scToAddr, sc)
191 // (DO NOT) delete(bb.scToSt, sc)
192 }
193 }
194}
195
196// HandleSubConnStateChange implements "grpc/balancer.Balancer" interface.
Devmalya Pauleb2e9552019-08-27 19:42:00 -0400197func (bb *baseBalancer) HandleSubConnStateChange(sc balancer.SubConn, s grpcconnectivity.State) {
William Kurkianea869482019-04-09 15:16:11 -0400198 bb.mu.Lock()
199 defer bb.mu.Unlock()
200
201 old, ok := bb.scToSt[sc]
202 if !ok {
203 bb.lg.Warn(
204 "state change for an unknown subconn",
Devmalya Pauleb2e9552019-08-27 19:42:00 -0400205 zap.String("picker", bb.picker.String()),
William Kurkianea869482019-04-09 15:16:11 -0400206 zap.String("balancer-id", bb.id),
207 zap.String("subconn", scToString(sc)),
Devmalya Pauleb2e9552019-08-27 19:42:00 -0400208 zap.Int("subconn-size", len(bb.scToAddr)),
William Kurkianea869482019-04-09 15:16:11 -0400209 zap.String("state", s.String()),
210 )
211 return
212 }
213
214 bb.lg.Info(
215 "state changed",
Devmalya Pauleb2e9552019-08-27 19:42:00 -0400216 zap.String("picker", bb.picker.String()),
William Kurkianea869482019-04-09 15:16:11 -0400217 zap.String("balancer-id", bb.id),
Devmalya Pauleb2e9552019-08-27 19:42:00 -0400218 zap.Bool("connected", s == grpcconnectivity.Ready),
William Kurkianea869482019-04-09 15:16:11 -0400219 zap.String("subconn", scToString(sc)),
Devmalya Pauleb2e9552019-08-27 19:42:00 -0400220 zap.Int("subconn-size", len(bb.scToAddr)),
William Kurkianea869482019-04-09 15:16:11 -0400221 zap.String("address", bb.scToAddr[sc].Addr),
222 zap.String("old-state", old.String()),
223 zap.String("new-state", s.String()),
224 )
225
226 bb.scToSt[sc] = s
227 switch s {
Devmalya Pauleb2e9552019-08-27 19:42:00 -0400228 case grpcconnectivity.Idle:
William Kurkianea869482019-04-09 15:16:11 -0400229 sc.Connect()
Devmalya Pauleb2e9552019-08-27 19:42:00 -0400230 case grpcconnectivity.Shutdown:
William Kurkianea869482019-04-09 15:16:11 -0400231 // When an address was removed by resolver, b called RemoveSubConn but
232 // kept the sc's state in scToSt. Remove state for this sc here.
233 delete(bb.scToAddr, sc)
234 delete(bb.scToSt, sc)
235 }
236
Devmalya Pauleb2e9552019-08-27 19:42:00 -0400237 oldAggrState := bb.connectivityRecorder.GetCurrentState()
238 bb.connectivityRecorder.RecordTransition(old, s)
William Kurkianea869482019-04-09 15:16:11 -0400239
Devmalya Pauleb2e9552019-08-27 19:42:00 -0400240 // Update balancer picker when one of the following happens:
William Kurkianea869482019-04-09 15:16:11 -0400241 // - this sc became ready from not-ready
242 // - this sc became not-ready from ready
243 // - the aggregated state of balancer became TransientFailure from non-TransientFailure
244 // - the aggregated state of balancer became non-TransientFailure from TransientFailure
Devmalya Pauleb2e9552019-08-27 19:42:00 -0400245 if (s == grpcconnectivity.Ready) != (old == grpcconnectivity.Ready) ||
246 (bb.connectivityRecorder.GetCurrentState() == grpcconnectivity.TransientFailure) != (oldAggrState == grpcconnectivity.TransientFailure) {
247 bb.updatePicker()
William Kurkianea869482019-04-09 15:16:11 -0400248 }
249
Devmalya Pauleb2e9552019-08-27 19:42:00 -0400250 bb.currentConn.UpdateBalancerState(bb.connectivityRecorder.GetCurrentState(), bb.picker)
William Kurkianea869482019-04-09 15:16:11 -0400251}
252
Devmalya Pauleb2e9552019-08-27 19:42:00 -0400253func (bb *baseBalancer) updatePicker() {
254 if bb.connectivityRecorder.GetCurrentState() == grpcconnectivity.TransientFailure {
255 bb.picker = picker.NewErr(balancer.ErrTransientFailure)
William Kurkianea869482019-04-09 15:16:11 -0400256 bb.lg.Info(
Devmalya Pauleb2e9552019-08-27 19:42:00 -0400257 "updated picker to transient error picker",
258 zap.String("picker", bb.picker.String()),
William Kurkianea869482019-04-09 15:16:11 -0400259 zap.String("balancer-id", bb.id),
260 zap.String("policy", bb.policy.String()),
261 )
William Kurkianea869482019-04-09 15:16:11 -0400262 return
263 }
264
265 // only pass ready subconns to picker
William Kurkianea869482019-04-09 15:16:11 -0400266 scToAddr := make(map[balancer.SubConn]resolver.Address)
267 for addr, sc := range bb.addrToSc {
Devmalya Pauleb2e9552019-08-27 19:42:00 -0400268 if st, ok := bb.scToSt[sc]; ok && st == grpcconnectivity.Ready {
William Kurkianea869482019-04-09 15:16:11 -0400269 scToAddr[sc] = addr
270 }
271 }
272
Devmalya Pauleb2e9552019-08-27 19:42:00 -0400273 bb.picker = picker.New(picker.Config{
274 Policy: bb.policy,
275 Logger: bb.lg,
276 SubConnToResolverAddress: scToAddr,
277 })
William Kurkianea869482019-04-09 15:16:11 -0400278 bb.lg.Info(
Devmalya Pauleb2e9552019-08-27 19:42:00 -0400279 "updated picker",
280 zap.String("picker", bb.picker.String()),
William Kurkianea869482019-04-09 15:16:11 -0400281 zap.String("balancer-id", bb.id),
282 zap.String("policy", bb.policy.String()),
Devmalya Pauleb2e9552019-08-27 19:42:00 -0400283 zap.Strings("subconn-ready", scsToStrings(scToAddr)),
284 zap.Int("subconn-size", len(scToAddr)),
William Kurkianea869482019-04-09 15:16:11 -0400285 )
286}
287
288// Close implements "grpc/balancer.Balancer" interface.
289// Close is a nop because base balancer doesn't have internal state to clean up,
290// and it doesn't need to call RemoveSubConn for the SubConns.
291func (bb *baseBalancer) Close() {
292 // TODO
293}