khenaidoo | d948f77 | 2021-08-11 17:49:24 -0400 | [diff] [blame] | 1 | // Copyright 2015 The etcd Authors |
| 2 | // |
| 3 | // Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | // you may not use this file except in compliance with the License. |
| 5 | // You may obtain a copy of the License at |
| 6 | // |
| 7 | // http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | // |
| 9 | // Unless required by applicable law or agreed to in writing, software |
| 10 | // distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | // See the License for the specific language governing permissions and |
| 13 | // limitations under the License. |
| 14 | |
| 15 | package etcdserver |
| 16 | |
| 17 | import ( |
| 18 | "context" |
| 19 | "fmt" |
| 20 | "path/filepath" |
| 21 | "sort" |
| 22 | "strings" |
| 23 | "time" |
| 24 | |
| 25 | "github.com/coreos/etcd/pkg/netutil" |
| 26 | "github.com/coreos/etcd/pkg/transport" |
| 27 | "github.com/coreos/etcd/pkg/types" |
| 28 | ) |
| 29 | |
| 30 | // ServerConfig holds the configuration of etcd as taken from the command line or discovery. |
| 31 | type ServerConfig struct { |
| 32 | Name string |
| 33 | DiscoveryURL string |
| 34 | DiscoveryProxy string |
| 35 | ClientURLs types.URLs |
| 36 | PeerURLs types.URLs |
| 37 | DataDir string |
| 38 | // DedicatedWALDir config will make the etcd to write the WAL to the WALDir |
| 39 | // rather than the dataDir/member/wal. |
| 40 | DedicatedWALDir string |
| 41 | SnapCount uint64 |
| 42 | MaxSnapFiles uint |
| 43 | MaxWALFiles uint |
| 44 | InitialPeerURLsMap types.URLsMap |
| 45 | InitialClusterToken string |
| 46 | NewCluster bool |
| 47 | ForceNewCluster bool |
| 48 | PeerTLSInfo transport.TLSInfo |
| 49 | |
| 50 | TickMs uint |
| 51 | ElectionTicks int |
| 52 | |
| 53 | // InitialElectionTickAdvance is true, then local member fast-forwards |
| 54 | // election ticks to speed up "initial" leader election trigger. This |
| 55 | // benefits the case of larger election ticks. For instance, cross |
| 56 | // datacenter deployment may require longer election timeout of 10-second. |
| 57 | // If true, local node does not need wait up to 10-second. Instead, |
| 58 | // forwards its election ticks to 8-second, and have only 2-second left |
| 59 | // before leader election. |
| 60 | // |
| 61 | // Major assumptions are that: |
| 62 | // - cluster has no active leader thus advancing ticks enables faster |
| 63 | // leader election, or |
| 64 | // - cluster already has an established leader, and rejoining follower |
| 65 | // is likely to receive heartbeats from the leader after tick advance |
| 66 | // and before election timeout. |
| 67 | // |
| 68 | // However, when network from leader to rejoining follower is congested, |
| 69 | // and the follower does not receive leader heartbeat within left election |
| 70 | // ticks, disruptive election has to happen thus affecting cluster |
| 71 | // availabilities. |
| 72 | // |
| 73 | // Disabling this would slow down initial bootstrap process for cross |
| 74 | // datacenter deployments. Make your own tradeoffs by configuring |
| 75 | // --initial-election-tick-advance at the cost of slow initial bootstrap. |
| 76 | // |
| 77 | // If single-node, it advances ticks regardless. |
| 78 | // |
| 79 | // See https://github.com/coreos/etcd/issues/9333 for more detail. |
| 80 | InitialElectionTickAdvance bool |
| 81 | |
| 82 | BootstrapTimeout time.Duration |
| 83 | |
| 84 | AutoCompactionRetention time.Duration |
| 85 | AutoCompactionMode string |
| 86 | QuotaBackendBytes int64 |
| 87 | MaxTxnOps uint |
| 88 | |
| 89 | // MaxRequestBytes is the maximum request size to send over raft. |
| 90 | MaxRequestBytes uint |
| 91 | |
| 92 | StrictReconfigCheck bool |
| 93 | |
| 94 | // ClientCertAuthEnabled is true when cert has been signed by the client CA. |
| 95 | ClientCertAuthEnabled bool |
| 96 | |
| 97 | AuthToken string |
| 98 | TokenTTL uint |
| 99 | |
| 100 | // InitialCorruptCheck is true to check data corruption on boot |
| 101 | // before serving any peer/client traffic. |
| 102 | InitialCorruptCheck bool |
| 103 | CorruptCheckTime time.Duration |
| 104 | |
| 105 | Debug bool |
| 106 | } |
| 107 | |
| 108 | // VerifyBootstrap sanity-checks the initial config for bootstrap case |
| 109 | // and returns an error for things that should never happen. |
| 110 | func (c *ServerConfig) VerifyBootstrap() error { |
| 111 | if err := c.hasLocalMember(); err != nil { |
| 112 | return err |
| 113 | } |
| 114 | if err := c.advertiseMatchesCluster(); err != nil { |
| 115 | return err |
| 116 | } |
| 117 | if checkDuplicateURL(c.InitialPeerURLsMap) { |
| 118 | return fmt.Errorf("initial cluster %s has duplicate url", c.InitialPeerURLsMap) |
| 119 | } |
| 120 | if c.InitialPeerURLsMap.String() == "" && c.DiscoveryURL == "" { |
| 121 | return fmt.Errorf("initial cluster unset and no discovery URL found") |
| 122 | } |
| 123 | return nil |
| 124 | } |
| 125 | |
| 126 | // VerifyJoinExisting sanity-checks the initial config for join existing cluster |
| 127 | // case and returns an error for things that should never happen. |
| 128 | func (c *ServerConfig) VerifyJoinExisting() error { |
| 129 | // The member has announced its peer urls to the cluster before starting; no need to |
| 130 | // set the configuration again. |
| 131 | if err := c.hasLocalMember(); err != nil { |
| 132 | return err |
| 133 | } |
| 134 | if checkDuplicateURL(c.InitialPeerURLsMap) { |
| 135 | return fmt.Errorf("initial cluster %s has duplicate url", c.InitialPeerURLsMap) |
| 136 | } |
| 137 | if c.DiscoveryURL != "" { |
| 138 | return fmt.Errorf("discovery URL should not be set when joining existing initial cluster") |
| 139 | } |
| 140 | return nil |
| 141 | } |
| 142 | |
| 143 | // hasLocalMember checks that the cluster at least contains the local server. |
| 144 | func (c *ServerConfig) hasLocalMember() error { |
| 145 | if urls := c.InitialPeerURLsMap[c.Name]; urls == nil { |
| 146 | return fmt.Errorf("couldn't find local name %q in the initial cluster configuration", c.Name) |
| 147 | } |
| 148 | return nil |
| 149 | } |
| 150 | |
| 151 | // advertiseMatchesCluster confirms peer URLs match those in the cluster peer list. |
| 152 | func (c *ServerConfig) advertiseMatchesCluster() error { |
| 153 | urls, apurls := c.InitialPeerURLsMap[c.Name], c.PeerURLs.StringSlice() |
| 154 | urls.Sort() |
| 155 | sort.Strings(apurls) |
| 156 | ctx, cancel := context.WithTimeout(context.TODO(), 30*time.Second) |
| 157 | defer cancel() |
| 158 | ok, err := netutil.URLStringsEqual(ctx, apurls, urls.StringSlice()) |
| 159 | if ok { |
| 160 | return nil |
| 161 | } |
| 162 | |
| 163 | initMap, apMap := make(map[string]struct{}), make(map[string]struct{}) |
| 164 | for _, url := range c.PeerURLs { |
| 165 | apMap[url.String()] = struct{}{} |
| 166 | } |
| 167 | for _, url := range c.InitialPeerURLsMap[c.Name] { |
| 168 | initMap[url.String()] = struct{}{} |
| 169 | } |
| 170 | |
| 171 | missing := []string{} |
| 172 | for url := range initMap { |
| 173 | if _, ok := apMap[url]; !ok { |
| 174 | missing = append(missing, url) |
| 175 | } |
| 176 | } |
| 177 | if len(missing) > 0 { |
| 178 | for i := range missing { |
| 179 | missing[i] = c.Name + "=" + missing[i] |
| 180 | } |
| 181 | mstr := strings.Join(missing, ",") |
| 182 | apStr := strings.Join(apurls, ",") |
| 183 | return fmt.Errorf("--initial-cluster has %s but missing from --initial-advertise-peer-urls=%s (%v)", mstr, apStr, err) |
| 184 | } |
| 185 | |
| 186 | for url := range apMap { |
| 187 | if _, ok := initMap[url]; !ok { |
| 188 | missing = append(missing, url) |
| 189 | } |
| 190 | } |
| 191 | if len(missing) > 0 { |
| 192 | mstr := strings.Join(missing, ",") |
| 193 | umap := types.URLsMap(map[string]types.URLs{c.Name: c.PeerURLs}) |
| 194 | return fmt.Errorf("--initial-advertise-peer-urls has %s but missing from --initial-cluster=%s", mstr, umap.String()) |
| 195 | } |
| 196 | |
| 197 | // resolved URLs from "--initial-advertise-peer-urls" and "--initial-cluster" did not match or failed |
| 198 | apStr := strings.Join(apurls, ",") |
| 199 | umap := types.URLsMap(map[string]types.URLs{c.Name: c.PeerURLs}) |
| 200 | return fmt.Errorf("failed to resolve %s to match --initial-cluster=%s (%v)", apStr, umap.String(), err) |
| 201 | } |
| 202 | |
| 203 | func (c *ServerConfig) MemberDir() string { return filepath.Join(c.DataDir, "member") } |
| 204 | |
| 205 | func (c *ServerConfig) WALDir() string { |
| 206 | if c.DedicatedWALDir != "" { |
| 207 | return c.DedicatedWALDir |
| 208 | } |
| 209 | return filepath.Join(c.MemberDir(), "wal") |
| 210 | } |
| 211 | |
| 212 | func (c *ServerConfig) SnapDir() string { return filepath.Join(c.MemberDir(), "snap") } |
| 213 | |
| 214 | func (c *ServerConfig) ShouldDiscover() bool { return c.DiscoveryURL != "" } |
| 215 | |
| 216 | // ReqTimeout returns timeout for request to finish. |
| 217 | func (c *ServerConfig) ReqTimeout() time.Duration { |
| 218 | // 5s for queue waiting, computation and disk IO delay |
| 219 | // + 2 * election timeout for possible leader election |
| 220 | return 5*time.Second + 2*time.Duration(c.ElectionTicks*int(c.TickMs))*time.Millisecond |
| 221 | } |
| 222 | |
| 223 | func (c *ServerConfig) electionTimeout() time.Duration { |
| 224 | return time.Duration(c.ElectionTicks*int(c.TickMs)) * time.Millisecond |
| 225 | } |
| 226 | |
| 227 | func (c *ServerConfig) peerDialTimeout() time.Duration { |
| 228 | // 1s for queue wait and election timeout |
| 229 | return time.Second + time.Duration(c.ElectionTicks*int(c.TickMs))*time.Millisecond |
| 230 | } |
| 231 | |
| 232 | func (c *ServerConfig) PrintWithInitial() { c.print(true) } |
| 233 | |
| 234 | func (c *ServerConfig) Print() { c.print(false) } |
| 235 | |
| 236 | func (c *ServerConfig) print(initial bool) { |
| 237 | plog.Infof("name = %s", c.Name) |
| 238 | if c.ForceNewCluster { |
| 239 | plog.Infof("force new cluster") |
| 240 | } |
| 241 | plog.Infof("data dir = %s", c.DataDir) |
| 242 | plog.Infof("member dir = %s", c.MemberDir()) |
| 243 | if c.DedicatedWALDir != "" { |
| 244 | plog.Infof("dedicated WAL dir = %s", c.DedicatedWALDir) |
| 245 | } |
| 246 | plog.Infof("heartbeat = %dms", c.TickMs) |
| 247 | plog.Infof("election = %dms", c.ElectionTicks*int(c.TickMs)) |
| 248 | plog.Infof("snapshot count = %d", c.SnapCount) |
| 249 | if len(c.DiscoveryURL) != 0 { |
| 250 | plog.Infof("discovery URL= %s", c.DiscoveryURL) |
| 251 | if len(c.DiscoveryProxy) != 0 { |
| 252 | plog.Infof("discovery proxy = %s", c.DiscoveryProxy) |
| 253 | } |
| 254 | } |
| 255 | plog.Infof("advertise client URLs = %s", c.ClientURLs) |
| 256 | if initial { |
| 257 | plog.Infof("initial advertise peer URLs = %s", c.PeerURLs) |
| 258 | plog.Infof("initial cluster = %s", c.InitialPeerURLsMap) |
| 259 | } |
| 260 | } |
| 261 | |
| 262 | func checkDuplicateURL(urlsmap types.URLsMap) bool { |
| 263 | um := make(map[string]bool) |
| 264 | for _, urls := range urlsmap { |
| 265 | for _, url := range urls { |
| 266 | u := url.String() |
| 267 | if um[u] { |
| 268 | return true |
| 269 | } |
| 270 | um[u] = true |
| 271 | } |
| 272 | } |
| 273 | return false |
| 274 | } |
| 275 | |
| 276 | func (c *ServerConfig) bootstrapTimeout() time.Duration { |
| 277 | if c.BootstrapTimeout != 0 { |
| 278 | return c.BootstrapTimeout |
| 279 | } |
| 280 | return time.Second |
| 281 | } |
| 282 | |
| 283 | func (c *ServerConfig) backendPath() string { return filepath.Join(c.SnapDir(), "db") } |