blob: f92706cb7a1487c2b9f7f7807cbb92074ac4a11c [file] [log] [blame]
khenaidoo59ce9dd2019-11-11 13:05:32 -05001// Copyright 2015 The etcd Authors
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package etcdserver
16
17import (
18 "context"
19 "encoding/json"
20 "fmt"
21 "io/ioutil"
22 "net/http"
23 "sort"
24 "strings"
25 "time"
26
27 "go.etcd.io/etcd/etcdserver/api/membership"
28 "go.etcd.io/etcd/pkg/types"
29 "go.etcd.io/etcd/version"
30
31 "github.com/coreos/go-semver/semver"
32 "go.uber.org/zap"
33)
34
35// isMemberBootstrapped tries to check if the given member has been bootstrapped
36// in the given cluster.
37func isMemberBootstrapped(lg *zap.Logger, cl *membership.RaftCluster, member string, rt http.RoundTripper, timeout time.Duration) bool {
38 rcl, err := getClusterFromRemotePeers(lg, getRemotePeerURLs(cl, member), timeout, false, rt)
39 if err != nil {
40 return false
41 }
42 id := cl.MemberByName(member).ID
43 m := rcl.Member(id)
44 if m == nil {
45 return false
46 }
47 if len(m.ClientURLs) > 0 {
48 return true
49 }
50 return false
51}
52
53// GetClusterFromRemotePeers takes a set of URLs representing etcd peers, and
54// attempts to construct a Cluster by accessing the members endpoint on one of
55// these URLs. The first URL to provide a response is used. If no URLs provide
56// a response, or a Cluster cannot be successfully created from a received
57// response, an error is returned.
58// Each request has a 10-second timeout. Because the upper limit of TTL is 5s,
59// 10 second is enough for building connection and finishing request.
60func GetClusterFromRemotePeers(lg *zap.Logger, urls []string, rt http.RoundTripper) (*membership.RaftCluster, error) {
61 return getClusterFromRemotePeers(lg, urls, 10*time.Second, true, rt)
62}
63
64// If logerr is true, it prints out more error messages.
65func getClusterFromRemotePeers(lg *zap.Logger, urls []string, timeout time.Duration, logerr bool, rt http.RoundTripper) (*membership.RaftCluster, error) {
66 cc := &http.Client{
67 Transport: rt,
68 Timeout: timeout,
69 }
70 for _, u := range urls {
71 addr := u + "/members"
72 resp, err := cc.Get(addr)
73 if err != nil {
74 if logerr {
75 if lg != nil {
76 lg.Warn("failed to get cluster response", zap.String("address", addr), zap.Error(err))
77 } else {
78 plog.Warningf("could not get cluster response from %s: %v", u, err)
79 }
80 }
81 continue
82 }
83 b, err := ioutil.ReadAll(resp.Body)
84 resp.Body.Close()
85 if err != nil {
86 if logerr {
87 if lg != nil {
88 lg.Warn("failed to read body of cluster response", zap.String("address", addr), zap.Error(err))
89 } else {
90 plog.Warningf("could not read the body of cluster response: %v", err)
91 }
92 }
93 continue
94 }
95 var membs []*membership.Member
96 if err = json.Unmarshal(b, &membs); err != nil {
97 if logerr {
98 if lg != nil {
99 lg.Warn("failed to unmarshal cluster response", zap.String("address", addr), zap.Error(err))
100 } else {
101 plog.Warningf("could not unmarshal cluster response: %v", err)
102 }
103 }
104 continue
105 }
106 id, err := types.IDFromString(resp.Header.Get("X-Etcd-Cluster-ID"))
107 if err != nil {
108 if logerr {
109 if lg != nil {
110 lg.Warn(
111 "failed to parse cluster ID",
112 zap.String("address", addr),
113 zap.String("header", resp.Header.Get("X-Etcd-Cluster-ID")),
114 zap.Error(err),
115 )
116 } else {
117 plog.Warningf("could not parse the cluster ID from cluster res: %v", err)
118 }
119 }
120 continue
121 }
122
123 // check the length of membership members
124 // if the membership members are present then prepare and return raft cluster
125 // if membership members are not present then the raft cluster formed will be
126 // an invalid empty cluster hence return failed to get raft cluster member(s) from the given urls error
127 if len(membs) > 0 {
128 return membership.NewClusterFromMembers(lg, "", id, membs), nil
129 }
130 return nil, fmt.Errorf("failed to get raft cluster member(s) from the given URLs")
131 }
132 return nil, fmt.Errorf("could not retrieve cluster information from the given URLs")
133}
134
135// getRemotePeerURLs returns peer urls of remote members in the cluster. The
136// returned list is sorted in ascending lexicographical order.
137func getRemotePeerURLs(cl *membership.RaftCluster, local string) []string {
138 us := make([]string, 0)
139 for _, m := range cl.Members() {
140 if m.Name == local {
141 continue
142 }
143 us = append(us, m.PeerURLs...)
144 }
145 sort.Strings(us)
146 return us
147}
148
149// getVersions returns the versions of the members in the given cluster.
150// The key of the returned map is the member's ID. The value of the returned map
151// is the semver versions string, including server and cluster.
152// If it fails to get the version of a member, the key will be nil.
153func getVersions(lg *zap.Logger, cl *membership.RaftCluster, local types.ID, rt http.RoundTripper) map[string]*version.Versions {
154 members := cl.Members()
155 vers := make(map[string]*version.Versions)
156 for _, m := range members {
157 if m.ID == local {
158 cv := "not_decided"
159 if cl.Version() != nil {
160 cv = cl.Version().String()
161 }
162 vers[m.ID.String()] = &version.Versions{Server: version.Version, Cluster: cv}
163 continue
164 }
165 ver, err := getVersion(lg, m, rt)
166 if err != nil {
167 if lg != nil {
168 lg.Warn("failed to get version", zap.String("remote-member-id", m.ID.String()), zap.Error(err))
169 } else {
170 plog.Warningf("cannot get the version of member %s (%v)", m.ID, err)
171 }
172 vers[m.ID.String()] = nil
173 } else {
174 vers[m.ID.String()] = ver
175 }
176 }
177 return vers
178}
179
180// decideClusterVersion decides the cluster version based on the versions map.
181// The returned version is the min server version in the map, or nil if the min
182// version in unknown.
183func decideClusterVersion(lg *zap.Logger, vers map[string]*version.Versions) *semver.Version {
184 var cv *semver.Version
185 lv := semver.Must(semver.NewVersion(version.Version))
186
187 for mid, ver := range vers {
188 if ver == nil {
189 return nil
190 }
191 v, err := semver.NewVersion(ver.Server)
192 if err != nil {
193 if lg != nil {
194 lg.Warn(
195 "failed to parse server version of remote member",
196 zap.String("remote-member-id", mid),
197 zap.String("remote-member-version", ver.Server),
198 zap.Error(err),
199 )
200 } else {
201 plog.Errorf("cannot understand the version of member %s (%v)", mid, err)
202 }
203 return nil
204 }
205 if lv.LessThan(*v) {
206 if lg != nil {
207 lg.Warn(
208 "leader found higher-versioned member",
209 zap.String("local-member-version", lv.String()),
210 zap.String("remote-member-id", mid),
211 zap.String("remote-member-version", ver.Server),
212 )
213 } else {
214 plog.Warningf("the local etcd version %s is not up-to-date", lv.String())
215 plog.Warningf("member %s has a higher version %s", mid, ver.Server)
216 }
217 }
218 if cv == nil {
219 cv = v
220 } else if v.LessThan(*cv) {
221 cv = v
222 }
223 }
224 return cv
225}
226
227// isCompatibleWithCluster return true if the local member has a compatible version with
228// the current running cluster.
229// The version is considered as compatible when at least one of the other members in the cluster has a
230// cluster version in the range of [MinClusterVersion, Version] and no known members has a cluster version
231// out of the range.
232// We set this rule since when the local member joins, another member might be offline.
233func isCompatibleWithCluster(lg *zap.Logger, cl *membership.RaftCluster, local types.ID, rt http.RoundTripper) bool {
234 vers := getVersions(lg, cl, local, rt)
235 minV := semver.Must(semver.NewVersion(version.MinClusterVersion))
236 maxV := semver.Must(semver.NewVersion(version.Version))
237 maxV = &semver.Version{
238 Major: maxV.Major,
239 Minor: maxV.Minor,
240 }
241 return isCompatibleWithVers(lg, vers, local, minV, maxV)
242}
243
244func isCompatibleWithVers(lg *zap.Logger, vers map[string]*version.Versions, local types.ID, minV, maxV *semver.Version) bool {
245 var ok bool
246 for id, v := range vers {
247 // ignore comparison with local version
248 if id == local.String() {
249 continue
250 }
251 if v == nil {
252 continue
253 }
254 clusterv, err := semver.NewVersion(v.Cluster)
255 if err != nil {
256 if lg != nil {
257 lg.Warn(
258 "failed to parse cluster version of remote member",
259 zap.String("remote-member-id", id),
260 zap.String("remote-member-cluster-version", v.Cluster),
261 zap.Error(err),
262 )
263 } else {
264 plog.Errorf("cannot understand the cluster version of member %s (%v)", id, err)
265 }
266 continue
267 }
268 if clusterv.LessThan(*minV) {
269 if lg != nil {
270 lg.Warn(
271 "cluster version of remote member is not compatible; too low",
272 zap.String("remote-member-id", id),
273 zap.String("remote-member-cluster-version", clusterv.String()),
274 zap.String("minimum-cluster-version-supported", minV.String()),
275 )
276 } else {
277 plog.Warningf("the running cluster version(%v) is lower than the minimal cluster version(%v) supported", clusterv.String(), minV.String())
278 }
279 return false
280 }
281 if maxV.LessThan(*clusterv) {
282 if lg != nil {
283 lg.Warn(
284 "cluster version of remote member is not compatible; too high",
285 zap.String("remote-member-id", id),
286 zap.String("remote-member-cluster-version", clusterv.String()),
287 zap.String("minimum-cluster-version-supported", minV.String()),
288 )
289 } else {
290 plog.Warningf("the running cluster version(%v) is higher than the maximum cluster version(%v) supported", clusterv.String(), maxV.String())
291 }
292 return false
293 }
294 ok = true
295 }
296 return ok
297}
298
299// getVersion returns the Versions of the given member via its
300// peerURLs. Returns the last error if it fails to get the version.
301func getVersion(lg *zap.Logger, m *membership.Member, rt http.RoundTripper) (*version.Versions, error) {
302 cc := &http.Client{
303 Transport: rt,
304 }
305 var (
306 err error
307 resp *http.Response
308 )
309
310 for _, u := range m.PeerURLs {
311 addr := u + "/version"
312 resp, err = cc.Get(addr)
313 if err != nil {
314 if lg != nil {
315 lg.Warn(
316 "failed to reach the peer URL",
317 zap.String("address", addr),
318 zap.String("remote-member-id", m.ID.String()),
319 zap.Error(err),
320 )
321 } else {
322 plog.Warningf("failed to reach the peerURL(%s) of member %s (%v)", u, m.ID, err)
323 }
324 continue
325 }
326 var b []byte
327 b, err = ioutil.ReadAll(resp.Body)
328 resp.Body.Close()
329 if err != nil {
330 if lg != nil {
331 lg.Warn(
332 "failed to read body of response",
333 zap.String("address", addr),
334 zap.String("remote-member-id", m.ID.String()),
335 zap.Error(err),
336 )
337 } else {
338 plog.Warningf("failed to read out the response body from the peerURL(%s) of member %s (%v)", u, m.ID, err)
339 }
340 continue
341 }
342 var vers version.Versions
343 if err = json.Unmarshal(b, &vers); err != nil {
344 if lg != nil {
345 lg.Warn(
346 "failed to unmarshal response",
347 zap.String("address", addr),
348 zap.String("remote-member-id", m.ID.String()),
349 zap.Error(err),
350 )
351 } else {
352 plog.Warningf("failed to unmarshal the response body got from the peerURL(%s) of member %s (%v)", u, m.ID, err)
353 }
354 continue
355 }
356 return &vers, nil
357 }
358 return nil, err
359}
360
361func promoteMemberHTTP(ctx context.Context, url string, id uint64, peerRt http.RoundTripper) ([]*membership.Member, error) {
362 cc := &http.Client{Transport: peerRt}
363 // TODO: refactor member http handler code
364 // cannot import etcdhttp, so manually construct url
365 requestUrl := url + "/members/promote/" + fmt.Sprintf("%d", id)
366 req, err := http.NewRequest("POST", requestUrl, nil)
367 if err != nil {
368 return nil, err
369 }
370 req = req.WithContext(ctx)
371 resp, err := cc.Do(req)
372 if err != nil {
373 return nil, err
374 }
375 defer resp.Body.Close()
376 b, err := ioutil.ReadAll(resp.Body)
377 if err != nil {
378 return nil, err
379 }
380
381 if resp.StatusCode == http.StatusRequestTimeout {
382 return nil, ErrTimeout
383 }
384 if resp.StatusCode == http.StatusPreconditionFailed {
385 // both ErrMemberNotLearner and ErrLearnerNotReady have same http status code
386 if strings.Contains(string(b), ErrLearnerNotReady.Error()) {
387 return nil, ErrLearnerNotReady
388 }
389 if strings.Contains(string(b), membership.ErrMemberNotLearner.Error()) {
390 return nil, membership.ErrMemberNotLearner
391 }
392 return nil, fmt.Errorf("member promote: unknown error(%s)", string(b))
393 }
394 if resp.StatusCode == http.StatusNotFound {
395 return nil, membership.ErrIDNotFound
396 }
397
398 if resp.StatusCode != http.StatusOK { // all other types of errors
399 return nil, fmt.Errorf("member promote: unknown error(%s)", string(b))
400 }
401
402 var membs []*membership.Member
403 if err := json.Unmarshal(b, &membs); err != nil {
404 return nil, err
405 }
406 return membs, nil
407}