VOL-2112 move to voltha-lib-go
Change-Id: Ic1af08003c1d2c698c0cce371e64f47b47b8d875
diff --git a/vendor/go.etcd.io/etcd/NOTICE b/vendor/go.etcd.io/etcd/NOTICE
deleted file mode 100644
index b39ddfa..0000000
--- a/vendor/go.etcd.io/etcd/NOTICE
+++ /dev/null
@@ -1,5 +0,0 @@
-CoreOS Project
-Copyright 2014 CoreOS, Inc
-
-This product includes software developed at CoreOS, Inc.
-(http://www.coreos.com/).
diff --git a/vendor/go.etcd.io/etcd/auth/authpb/auth.pb.go b/vendor/go.etcd.io/etcd/auth/authpb/auth.pb.go
index 1a940c3..7e038df 100644
--- a/vendor/go.etcd.io/etcd/auth/authpb/auth.pb.go
+++ b/vendor/go.etcd.io/etcd/auth/authpb/auth.pb.go
@@ -8,6 +8,7 @@
auth.proto
It has these top-level messages:
+ UserAddOptions
User
Permission
Role
@@ -59,19 +60,29 @@
func (x Permission_Type) String() string {
return proto.EnumName(Permission_Type_name, int32(x))
}
-func (Permission_Type) EnumDescriptor() ([]byte, []int) { return fileDescriptorAuth, []int{1, 0} }
+func (Permission_Type) EnumDescriptor() ([]byte, []int) { return fileDescriptorAuth, []int{2, 0} }
+
+type UserAddOptions struct {
+ NoPassword bool `protobuf:"varint,1,opt,name=no_password,json=noPassword,proto3" json:"no_password,omitempty"`
+}
+
+func (m *UserAddOptions) Reset() { *m = UserAddOptions{} }
+func (m *UserAddOptions) String() string { return proto.CompactTextString(m) }
+func (*UserAddOptions) ProtoMessage() {}
+func (*UserAddOptions) Descriptor() ([]byte, []int) { return fileDescriptorAuth, []int{0} }
// User is a single entry in the bucket authUsers
type User struct {
- Name []byte `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
- Password []byte `protobuf:"bytes,2,opt,name=password,proto3" json:"password,omitempty"`
- Roles []string `protobuf:"bytes,3,rep,name=roles" json:"roles,omitempty"`
+ Name []byte `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
+ Password []byte `protobuf:"bytes,2,opt,name=password,proto3" json:"password,omitempty"`
+ Roles []string `protobuf:"bytes,3,rep,name=roles" json:"roles,omitempty"`
+ Options *UserAddOptions `protobuf:"bytes,4,opt,name=options" json:"options,omitempty"`
}
func (m *User) Reset() { *m = User{} }
func (m *User) String() string { return proto.CompactTextString(m) }
func (*User) ProtoMessage() {}
-func (*User) Descriptor() ([]byte, []int) { return fileDescriptorAuth, []int{0} }
+func (*User) Descriptor() ([]byte, []int) { return fileDescriptorAuth, []int{1} }
// Permission is a single entity
type Permission struct {
@@ -83,7 +94,7 @@
func (m *Permission) Reset() { *m = Permission{} }
func (m *Permission) String() string { return proto.CompactTextString(m) }
func (*Permission) ProtoMessage() {}
-func (*Permission) Descriptor() ([]byte, []int) { return fileDescriptorAuth, []int{1} }
+func (*Permission) Descriptor() ([]byte, []int) { return fileDescriptorAuth, []int{2} }
// Role is a single entry in the bucket authRoles
type Role struct {
@@ -94,14 +105,43 @@
func (m *Role) Reset() { *m = Role{} }
func (m *Role) String() string { return proto.CompactTextString(m) }
func (*Role) ProtoMessage() {}
-func (*Role) Descriptor() ([]byte, []int) { return fileDescriptorAuth, []int{2} }
+func (*Role) Descriptor() ([]byte, []int) { return fileDescriptorAuth, []int{3} }
func init() {
+ proto.RegisterType((*UserAddOptions)(nil), "authpb.UserAddOptions")
proto.RegisterType((*User)(nil), "authpb.User")
proto.RegisterType((*Permission)(nil), "authpb.Permission")
proto.RegisterType((*Role)(nil), "authpb.Role")
proto.RegisterEnum("authpb.Permission_Type", Permission_Type_name, Permission_Type_value)
}
+func (m *UserAddOptions) Marshal() (dAtA []byte, err error) {
+ size := m.Size()
+ dAtA = make([]byte, size)
+ n, err := m.MarshalTo(dAtA)
+ if err != nil {
+ return nil, err
+ }
+ return dAtA[:n], nil
+}
+
+func (m *UserAddOptions) MarshalTo(dAtA []byte) (int, error) {
+ var i int
+ _ = i
+ var l int
+ _ = l
+ if m.NoPassword {
+ dAtA[i] = 0x8
+ i++
+ if m.NoPassword {
+ dAtA[i] = 1
+ } else {
+ dAtA[i] = 0
+ }
+ i++
+ }
+ return i, nil
+}
+
func (m *User) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
@@ -144,6 +184,16 @@
i += copy(dAtA[i:], s)
}
}
+ if m.Options != nil {
+ dAtA[i] = 0x22
+ i++
+ i = encodeVarintAuth(dAtA, i, uint64(m.Options.Size()))
+ n1, err := m.Options.MarshalTo(dAtA[i:])
+ if err != nil {
+ return 0, err
+ }
+ i += n1
+ }
return i, nil
}
@@ -227,6 +277,15 @@
dAtA[offset] = uint8(v)
return offset + 1
}
+func (m *UserAddOptions) Size() (n int) {
+ var l int
+ _ = l
+ if m.NoPassword {
+ n += 2
+ }
+ return n
+}
+
func (m *User) Size() (n int) {
var l int
_ = l
@@ -244,6 +303,10 @@
n += 1 + l + sovAuth(uint64(l))
}
}
+ if m.Options != nil {
+ l = m.Options.Size()
+ n += 1 + l + sovAuth(uint64(l))
+ }
return n
}
@@ -293,6 +356,76 @@
func sozAuth(x uint64) (n int) {
return sovAuth(uint64((x << 1) ^ uint64((int64(x) >> 63))))
}
+func (m *UserAddOptions) Unmarshal(dAtA []byte) error {
+ l := len(dAtA)
+ iNdEx := 0
+ for iNdEx < l {
+ preIndex := iNdEx
+ var wire uint64
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowAuth
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ wire |= (uint64(b) & 0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ fieldNum := int32(wire >> 3)
+ wireType := int(wire & 0x7)
+ if wireType == 4 {
+ return fmt.Errorf("proto: UserAddOptions: wiretype end group for non-group")
+ }
+ if fieldNum <= 0 {
+ return fmt.Errorf("proto: UserAddOptions: illegal tag %d (wire type %d)", fieldNum, wire)
+ }
+ switch fieldNum {
+ case 1:
+ if wireType != 0 {
+ return fmt.Errorf("proto: wrong wireType = %d for field NoPassword", wireType)
+ }
+ var v int
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowAuth
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ v |= (int(b) & 0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ m.NoPassword = bool(v != 0)
+ default:
+ iNdEx = preIndex
+ skippy, err := skipAuth(dAtA[iNdEx:])
+ if err != nil {
+ return err
+ }
+ if skippy < 0 {
+ return ErrInvalidLengthAuth
+ }
+ if (iNdEx + skippy) > l {
+ return io.ErrUnexpectedEOF
+ }
+ iNdEx += skippy
+ }
+ }
+
+ if iNdEx > l {
+ return io.ErrUnexpectedEOF
+ }
+ return nil
+}
func (m *User) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
@@ -413,6 +546,39 @@
}
m.Roles = append(m.Roles, string(dAtA[iNdEx:postIndex]))
iNdEx = postIndex
+ case 4:
+ if wireType != 2 {
+ return fmt.Errorf("proto: wrong wireType = %d for field Options", wireType)
+ }
+ var msglen int
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowAuth
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ msglen |= (int(b) & 0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ if msglen < 0 {
+ return ErrInvalidLengthAuth
+ }
+ postIndex := iNdEx + msglen
+ if postIndex > l {
+ return io.ErrUnexpectedEOF
+ }
+ if m.Options == nil {
+ m.Options = &UserAddOptions{}
+ }
+ if err := m.Options.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+ return err
+ }
+ iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipAuth(dAtA[iNdEx:])
@@ -785,23 +951,27 @@
func init() { proto.RegisterFile("auth.proto", fileDescriptorAuth) }
var fileDescriptorAuth = []byte{
- // 288 bytes of a gzipped FileDescriptorProto
- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x90, 0xc1, 0x4a, 0xc3, 0x30,
- 0x1c, 0xc6, 0x9b, 0xb6, 0x1b, 0xed, 0x5f, 0x27, 0x25, 0x0c, 0x0c, 0x13, 0x42, 0xe9, 0xa9, 0x78,
- 0xa8, 0xb0, 0x5d, 0xbc, 0x2a, 0xf6, 0x20, 0x78, 0x90, 0x50, 0xf1, 0x28, 0x1d, 0x0d, 0x75, 0x6c,
- 0x6d, 0x4a, 0x32, 0x91, 0xbe, 0x89, 0x07, 0x1f, 0x68, 0xc7, 0x3d, 0x82, 0xab, 0x2f, 0x22, 0x4d,
- 0x64, 0x43, 0xdc, 0xed, 0xfb, 0xbe, 0xff, 0x97, 0xe4, 0x97, 0x3f, 0x40, 0xfe, 0xb6, 0x7e, 0x4d,
- 0x1a, 0x29, 0xd6, 0x02, 0x0f, 0x7b, 0xdd, 0xcc, 0x27, 0xe3, 0x52, 0x94, 0x42, 0x47, 0x57, 0xbd,
- 0x32, 0xd3, 0xe8, 0x01, 0xdc, 0x27, 0xc5, 0x25, 0xc6, 0xe0, 0xd6, 0x79, 0xc5, 0x09, 0x0a, 0x51,
- 0x7c, 0xca, 0xb4, 0xc6, 0x13, 0xf0, 0x9a, 0x5c, 0xa9, 0x77, 0x21, 0x0b, 0x62, 0xeb, 0x7c, 0xef,
- 0xf1, 0x18, 0x06, 0x52, 0xac, 0xb8, 0x22, 0x4e, 0xe8, 0xc4, 0x3e, 0x33, 0x26, 0xfa, 0x44, 0x00,
- 0x8f, 0x5c, 0x56, 0x0b, 0xa5, 0x16, 0xa2, 0xc6, 0x33, 0xf0, 0x1a, 0x2e, 0xab, 0xac, 0x6d, 0xcc,
- 0xc5, 0x67, 0xd3, 0xf3, 0xc4, 0xd0, 0x24, 0x87, 0x56, 0xd2, 0x8f, 0xd9, 0xbe, 0x88, 0x03, 0x70,
- 0x96, 0xbc, 0xfd, 0x7d, 0xb0, 0x97, 0xf8, 0x02, 0x7c, 0x99, 0xd7, 0x25, 0x7f, 0xe1, 0x75, 0x41,
- 0x1c, 0x03, 0xa2, 0x83, 0xb4, 0x2e, 0xa2, 0x4b, 0x70, 0xf5, 0x31, 0x0f, 0x5c, 0x96, 0xde, 0xdc,
- 0x05, 0x16, 0xf6, 0x61, 0xf0, 0xcc, 0xee, 0xb3, 0x34, 0x40, 0x78, 0x04, 0x7e, 0x1f, 0x1a, 0x6b,
- 0x47, 0x19, 0xb8, 0x4c, 0xac, 0xf8, 0xd1, 0xcf, 0x5e, 0xc3, 0x68, 0xc9, 0xdb, 0x03, 0x16, 0xb1,
- 0x43, 0x27, 0x3e, 0x99, 0xe2, 0xff, 0xc0, 0xec, 0x6f, 0xf1, 0x96, 0x6c, 0x76, 0xd4, 0xda, 0xee,
- 0xa8, 0xb5, 0xe9, 0x28, 0xda, 0x76, 0x14, 0x7d, 0x75, 0x14, 0x7d, 0x7c, 0x53, 0x6b, 0x3e, 0xd4,
- 0x3b, 0x9e, 0xfd, 0x04, 0x00, 0x00, 0xff, 0xff, 0xcc, 0x76, 0x8d, 0x4f, 0x8f, 0x01, 0x00, 0x00,
+ // 338 bytes of a gzipped FileDescriptorProto
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x91, 0xcf, 0x4e, 0xea, 0x40,
+ 0x14, 0xc6, 0x3b, 0xb4, 0x70, 0xdb, 0xc3, 0x85, 0x90, 0x13, 0x72, 0x6f, 0x83, 0x49, 0x6d, 0xba,
+ 0x6a, 0x5c, 0x54, 0x85, 0x8d, 0x5b, 0x8c, 0x2c, 0x5c, 0x49, 0x26, 0x18, 0x97, 0xa4, 0xa4, 0x13,
+ 0x24, 0xc0, 0x4c, 0x33, 0x83, 0x31, 0x6c, 0x7c, 0x0e, 0x17, 0x3e, 0x10, 0x4b, 0x1e, 0x41, 0xf0,
+ 0x45, 0x4c, 0x67, 0xf8, 0x13, 0xa2, 0xbb, 0xef, 0x7c, 0xe7, 0xfb, 0x66, 0x7e, 0x99, 0x01, 0x48,
+ 0x5f, 0x16, 0xcf, 0x49, 0x2e, 0xc5, 0x42, 0x60, 0xa5, 0xd0, 0xf9, 0xa8, 0xd5, 0x1c, 0x8b, 0xb1,
+ 0xd0, 0xd6, 0x65, 0xa1, 0xcc, 0x36, 0xba, 0x86, 0xfa, 0xa3, 0x62, 0xb2, 0x9b, 0x65, 0x0f, 0xf9,
+ 0x62, 0x22, 0xb8, 0xc2, 0x73, 0xa8, 0x72, 0x31, 0xcc, 0x53, 0xa5, 0x5e, 0x85, 0xcc, 0x7c, 0x12,
+ 0x92, 0xd8, 0xa5, 0xc0, 0x45, 0x7f, 0xe7, 0x44, 0x6f, 0xe0, 0x14, 0x15, 0x44, 0x70, 0x78, 0x3a,
+ 0x67, 0x3a, 0xf1, 0x97, 0x6a, 0x8d, 0x2d, 0x70, 0x0f, 0xcd, 0x92, 0xf6, 0x0f, 0x33, 0x36, 0xa1,
+ 0x2c, 0xc5, 0x8c, 0x29, 0xdf, 0x0e, 0xed, 0xd8, 0xa3, 0x66, 0xc0, 0x2b, 0xf8, 0x23, 0xcc, 0xcd,
+ 0xbe, 0x13, 0x92, 0xb8, 0xda, 0xfe, 0x97, 0x18, 0xe0, 0xe4, 0x94, 0x8b, 0xee, 0x63, 0xd1, 0x07,
+ 0x01, 0xe8, 0x33, 0x39, 0x9f, 0x28, 0x35, 0x11, 0x1c, 0x3b, 0xe0, 0xe6, 0x4c, 0xce, 0x07, 0xcb,
+ 0xdc, 0xa0, 0xd4, 0xdb, 0xff, 0xf7, 0x27, 0x1c, 0x53, 0x49, 0xb1, 0xa6, 0x87, 0x20, 0x36, 0xc0,
+ 0x9e, 0xb2, 0xe5, 0x0e, 0xb1, 0x90, 0x78, 0x06, 0x9e, 0x4c, 0xf9, 0x98, 0x0d, 0x19, 0xcf, 0x7c,
+ 0xdb, 0xa0, 0x6b, 0xa3, 0xc7, 0xb3, 0xe8, 0x02, 0x1c, 0x5d, 0x73, 0xc1, 0xa1, 0xbd, 0xee, 0x5d,
+ 0xc3, 0x42, 0x0f, 0xca, 0x4f, 0xf4, 0x7e, 0xd0, 0x6b, 0x10, 0xac, 0x81, 0x57, 0x98, 0x66, 0x2c,
+ 0x45, 0x03, 0x70, 0xa8, 0x98, 0xb1, 0x5f, 0x9f, 0xe7, 0x06, 0x6a, 0x53, 0xb6, 0x3c, 0x62, 0xf9,
+ 0xa5, 0xd0, 0x8e, 0xab, 0x6d, 0xfc, 0x09, 0x4c, 0x4f, 0x83, 0xb7, 0xfe, 0x6a, 0x13, 0x58, 0xeb,
+ 0x4d, 0x60, 0xad, 0xb6, 0x01, 0x59, 0x6f, 0x03, 0xf2, 0xb9, 0x0d, 0xc8, 0xfb, 0x57, 0x60, 0x8d,
+ 0x2a, 0xfa, 0x23, 0x3b, 0xdf, 0x01, 0x00, 0x00, 0xff, 0xff, 0x61, 0x66, 0xc6, 0x9d, 0xf4, 0x01,
+ 0x00, 0x00,
}
diff --git a/vendor/go.etcd.io/etcd/auth/authpb/auth.proto b/vendor/go.etcd.io/etcd/auth/authpb/auth.proto
index 001d334..8f82b7c 100644
--- a/vendor/go.etcd.io/etcd/auth/authpb/auth.proto
+++ b/vendor/go.etcd.io/etcd/auth/authpb/auth.proto
@@ -9,11 +9,16 @@
option (gogoproto.goproto_getters_all) = false;
option (gogoproto.goproto_enum_prefix_all) = false;
+message UserAddOptions {
+ bool no_password = 1;
+};
+
// User is a single entry in the bucket authUsers
message User {
bytes name = 1;
bytes password = 2;
repeated string roles = 3;
+ UserAddOptions options = 4;
}
// Permission is a single entity
diff --git a/vendor/go.etcd.io/etcd/clientv3/README.md b/vendor/go.etcd.io/etcd/clientv3/README.md
index 0b2dc9a..6c6fe7c 100644
--- a/vendor/go.etcd.io/etcd/clientv3/README.md
+++ b/vendor/go.etcd.io/etcd/clientv3/README.md
@@ -1,12 +1,10 @@
# etcd/clientv3
-[](https://etcd.readthedocs.io/en/latest/?badge=latest)
+[](https://etcd.io/docs)
[](https://godoc.org/go.etcd.io/etcd/clientv3)
`etcd/clientv3` is the official Go etcd client for v3.
-See https://etcd.readthedocs.io/en/latest for latest client architecture.
-
## Install
```bash
diff --git a/vendor/go.etcd.io/etcd/clientv3/auth.go b/vendor/go.etcd.io/etcd/clientv3/auth.go
index 921f50f..c954f1b 100644
--- a/vendor/go.etcd.io/etcd/clientv3/auth.go
+++ b/vendor/go.etcd.io/etcd/clientv3/auth.go
@@ -52,6 +52,8 @@
PermReadWrite = authpb.READWRITE
)
+type UserAddOptions authpb.UserAddOptions
+
type Auth interface {
// AuthEnable enables auth of an etcd cluster.
AuthEnable(ctx context.Context) (*AuthEnableResponse, error)
@@ -62,6 +64,9 @@
// UserAdd adds a new user to an etcd cluster.
UserAdd(ctx context.Context, name string, password string) (*AuthUserAddResponse, error)
+ // UserAddWithOptions adds a new user to an etcd cluster with some options.
+ UserAddWithOptions(ctx context.Context, name string, password string, opt *UserAddOptions) (*AuthUserAddResponse, error)
+
// UserDelete deletes a user from an etcd cluster.
UserDelete(ctx context.Context, name string) (*AuthUserDeleteResponse, error)
@@ -123,7 +128,12 @@
}
func (auth *authClient) UserAdd(ctx context.Context, name string, password string) (*AuthUserAddResponse, error) {
- resp, err := auth.remote.UserAdd(ctx, &pb.AuthUserAddRequest{Name: name, Password: password}, auth.callOpts...)
+ resp, err := auth.remote.UserAdd(ctx, &pb.AuthUserAddRequest{Name: name, Password: password, Options: &authpb.UserAddOptions{NoPassword: false}}, auth.callOpts...)
+ return (*AuthUserAddResponse)(resp), toErr(ctx, err)
+}
+
+func (auth *authClient) UserAddWithOptions(ctx context.Context, name string, password string, options *UserAddOptions) (*AuthUserAddResponse, error) {
+ resp, err := auth.remote.UserAdd(ctx, &pb.AuthUserAddRequest{Name: name, Password: password, Options: (*authpb.UserAddOptions)(options)}, auth.callOpts...)
return (*AuthUserAddResponse)(resp), toErr(ctx, err)
}
diff --git a/vendor/go.etcd.io/etcd/clientv3/balancer/balancer.go b/vendor/go.etcd.io/etcd/clientv3/balancer/balancer.go
index 3c44e70..d02a7ee 100644
--- a/vendor/go.etcd.io/etcd/clientv3/balancer/balancer.go
+++ b/vendor/go.etcd.io/etcd/clientv3/balancer/balancer.go
@@ -12,24 +12,45 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+// Package balancer implements client balancer.
package balancer
import (
- "fmt"
"strconv"
"sync"
"time"
+ "go.etcd.io/etcd/clientv3/balancer/connectivity"
"go.etcd.io/etcd/clientv3/balancer/picker"
"go.uber.org/zap"
"google.golang.org/grpc/balancer"
- "google.golang.org/grpc/connectivity"
+ grpcconnectivity "google.golang.org/grpc/connectivity"
"google.golang.org/grpc/resolver"
_ "google.golang.org/grpc/resolver/dns" // register DNS resolver
_ "google.golang.org/grpc/resolver/passthrough" // register passthrough resolver
)
+// Config defines balancer configurations.
+type Config struct {
+ // Policy configures balancer policy.
+ Policy picker.Policy
+
+ // Picker implements gRPC picker.
+ // Leave empty if "Policy" field is not custom.
+ // TODO: currently custom policy is not supported.
+ // Picker picker.Picker
+
+ // Name defines an additional name for balancer.
+ // Useful for balancer testing to avoid register conflicts.
+ // If empty, defaults to policy name.
+ Name string
+
+ // Logger configures balancer logging.
+ // If nil, logs are discarded.
+ Logger *zap.Logger
+}
+
// RegisterBuilder creates and registers a builder. Since this function calls balancer.Register, it
// must be invoked at initialization time.
func RegisterBuilder(cfg Config) {
@@ -54,24 +75,18 @@
bb := &baseBalancer{
id: strconv.FormatInt(time.Now().UnixNano(), 36),
policy: b.cfg.Policy,
- name: b.cfg.Policy.String(),
+ name: b.cfg.Name,
lg: b.cfg.Logger,
addrToSc: make(map[resolver.Address]balancer.SubConn),
scToAddr: make(map[balancer.SubConn]resolver.Address),
- scToSt: make(map[balancer.SubConn]connectivity.State),
+ scToSt: make(map[balancer.SubConn]grpcconnectivity.State),
- currentConn: nil,
- csEvltr: &connectivityStateEvaluator{},
+ currentConn: nil,
+ connectivityRecorder: connectivity.New(b.cfg.Logger),
// initialize picker always returns "ErrNoSubConnAvailable"
- Picker: picker.NewErr(balancer.ErrNoSubConnAvailable),
- }
- if b.cfg.Name != "" {
- bb.name = b.cfg.Name
- }
- if bb.lg == nil {
- bb.lg = zap.NewNop()
+ picker: picker.NewErr(balancer.ErrNoSubConnAvailable),
}
// TODO: support multiple connections
@@ -115,13 +130,12 @@
addrToSc map[resolver.Address]balancer.SubConn
scToAddr map[balancer.SubConn]resolver.Address
- scToSt map[balancer.SubConn]connectivity.State
+ scToSt map[balancer.SubConn]grpcconnectivity.State
- currentConn balancer.ClientConn
- currentState connectivity.State
- csEvltr *connectivityStateEvaluator
+ currentConn balancer.ClientConn
+ connectivityRecorder connectivity.Recorder
- picker.Picker
+ picker picker.Picker
}
// HandleResolvedAddrs implements "grpc/balancer.Balancer" interface.
@@ -131,7 +145,11 @@
bb.lg.Warn("HandleResolvedAddrs called with error", zap.String("balancer-id", bb.id), zap.Error(err))
return
}
- bb.lg.Info("resolved", zap.String("balancer-id", bb.id), zap.Strings("addresses", addrsToStrings(addrs)))
+ bb.lg.Info("resolved",
+ zap.String("picker", bb.picker.String()),
+ zap.String("balancer-id", bb.id),
+ zap.Strings("addresses", addrsToStrings(addrs)),
+ )
bb.mu.Lock()
defer bb.mu.Unlock()
@@ -142,12 +160,13 @@
if _, ok := bb.addrToSc[addr]; !ok {
sc, err := bb.currentConn.NewSubConn([]resolver.Address{addr}, balancer.NewSubConnOptions{})
if err != nil {
- bb.lg.Warn("NewSubConn failed", zap.String("balancer-id", bb.id), zap.Error(err), zap.String("address", addr.Addr))
+ bb.lg.Warn("NewSubConn failed", zap.String("picker", bb.picker.String()), zap.String("balancer-id", bb.id), zap.Error(err), zap.String("address", addr.Addr))
continue
}
+ bb.lg.Info("created subconn", zap.String("address", addr.Addr))
bb.addrToSc[addr] = sc
bb.scToAddr[sc] = addr
- bb.scToSt[sc] = connectivity.Idle
+ bb.scToSt[sc] = grpcconnectivity.Idle
sc.Connect()
}
}
@@ -160,6 +179,7 @@
bb.lg.Info(
"removed subconn",
+ zap.String("picker", bb.picker.String()),
zap.String("balancer-id", bb.id),
zap.String("address", addr.Addr),
zap.String("subconn", scToString(sc)),
@@ -174,7 +194,7 @@
}
// HandleSubConnStateChange implements "grpc/balancer.Balancer" interface.
-func (bb *baseBalancer) HandleSubConnStateChange(sc balancer.SubConn, s connectivity.State) {
+func (bb *baseBalancer) HandleSubConnStateChange(sc balancer.SubConn, s grpcconnectivity.State) {
bb.mu.Lock()
defer bb.mu.Unlock()
@@ -182,8 +202,10 @@
if !ok {
bb.lg.Warn(
"state change for an unknown subconn",
+ zap.String("picker", bb.picker.String()),
zap.String("balancer-id", bb.id),
zap.String("subconn", scToString(sc)),
+ zap.Int("subconn-size", len(bb.scToAddr)),
zap.String("state", s.String()),
)
return
@@ -191,9 +213,11 @@
bb.lg.Info(
"state changed",
+ zap.String("picker", bb.picker.String()),
zap.String("balancer-id", bb.id),
- zap.Bool("connected", s == connectivity.Ready),
+ zap.Bool("connected", s == grpcconnectivity.Ready),
zap.String("subconn", scToString(sc)),
+ zap.Int("subconn-size", len(bb.scToAddr)),
zap.String("address", bb.scToAddr[sc].Addr),
zap.String("old-state", old.String()),
zap.String("new-state", s.String()),
@@ -201,69 +225,63 @@
bb.scToSt[sc] = s
switch s {
- case connectivity.Idle:
+ case grpcconnectivity.Idle:
sc.Connect()
- case connectivity.Shutdown:
+ case grpcconnectivity.Shutdown:
// When an address was removed by resolver, b called RemoveSubConn but
// kept the sc's state in scToSt. Remove state for this sc here.
delete(bb.scToAddr, sc)
delete(bb.scToSt, sc)
}
- oldAggrState := bb.currentState
- bb.currentState = bb.csEvltr.recordTransition(old, s)
+ oldAggrState := bb.connectivityRecorder.GetCurrentState()
+ bb.connectivityRecorder.RecordTransition(old, s)
- // Regenerate picker when one of the following happens:
+ // Update balancer picker when one of the following happens:
// - this sc became ready from not-ready
// - this sc became not-ready from ready
// - the aggregated state of balancer became TransientFailure from non-TransientFailure
// - the aggregated state of balancer became non-TransientFailure from TransientFailure
- if (s == connectivity.Ready) != (old == connectivity.Ready) ||
- (bb.currentState == connectivity.TransientFailure) != (oldAggrState == connectivity.TransientFailure) {
- bb.regeneratePicker()
+ if (s == grpcconnectivity.Ready) != (old == grpcconnectivity.Ready) ||
+ (bb.connectivityRecorder.GetCurrentState() == grpcconnectivity.TransientFailure) != (oldAggrState == grpcconnectivity.TransientFailure) {
+ bb.updatePicker()
}
- bb.currentConn.UpdateBalancerState(bb.currentState, bb.Picker)
- return
+ bb.currentConn.UpdateBalancerState(bb.connectivityRecorder.GetCurrentState(), bb.picker)
}
-func (bb *baseBalancer) regeneratePicker() {
- if bb.currentState == connectivity.TransientFailure {
+func (bb *baseBalancer) updatePicker() {
+ if bb.connectivityRecorder.GetCurrentState() == grpcconnectivity.TransientFailure {
+ bb.picker = picker.NewErr(balancer.ErrTransientFailure)
bb.lg.Info(
- "generated transient error picker",
+ "updated picker to transient error picker",
+ zap.String("picker", bb.picker.String()),
zap.String("balancer-id", bb.id),
zap.String("policy", bb.policy.String()),
)
- bb.Picker = picker.NewErr(balancer.ErrTransientFailure)
return
}
// only pass ready subconns to picker
- scs := make([]balancer.SubConn, 0)
- addrToSc := make(map[resolver.Address]balancer.SubConn)
scToAddr := make(map[balancer.SubConn]resolver.Address)
for addr, sc := range bb.addrToSc {
- if st, ok := bb.scToSt[sc]; ok && st == connectivity.Ready {
- scs = append(scs, sc)
- addrToSc[addr] = sc
+ if st, ok := bb.scToSt[sc]; ok && st == grpcconnectivity.Ready {
scToAddr[sc] = addr
}
}
- switch bb.policy {
- case picker.RoundrobinBalanced:
- bb.Picker = picker.NewRoundrobinBalanced(bb.lg, scs, addrToSc, scToAddr)
-
- default:
- panic(fmt.Errorf("invalid balancer picker policy (%d)", bb.policy))
- }
-
+ bb.picker = picker.New(picker.Config{
+ Policy: bb.policy,
+ Logger: bb.lg,
+ SubConnToResolverAddress: scToAddr,
+ })
bb.lg.Info(
- "generated picker",
+ "updated picker",
+ zap.String("picker", bb.picker.String()),
zap.String("balancer-id", bb.id),
zap.String("policy", bb.policy.String()),
- zap.Strings("subconn-ready", scsToStrings(addrToSc)),
- zap.Int("subconn-size", len(addrToSc)),
+ zap.Strings("subconn-ready", scsToStrings(scToAddr)),
+ zap.Int("subconn-size", len(scToAddr)),
)
}
diff --git a/vendor/go.etcd.io/etcd/clientv3/balancer/config.go b/vendor/go.etcd.io/etcd/clientv3/balancer/config.go
deleted file mode 100644
index 0339a84..0000000
--- a/vendor/go.etcd.io/etcd/clientv3/balancer/config.go
+++ /dev/null
@@ -1,36 +0,0 @@
-// Copyright 2018 The etcd Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package balancer
-
-import (
- "go.etcd.io/etcd/clientv3/balancer/picker"
-
- "go.uber.org/zap"
-)
-
-// Config defines balancer configurations.
-type Config struct {
- // Policy configures balancer policy.
- Policy picker.Policy
-
- // Name defines an additional name for balancer.
- // Useful for balancer testing to avoid register conflicts.
- // If empty, defaults to policy name.
- Name string
-
- // Logger configures balancer logging.
- // If nil, logs are discarded.
- Logger *zap.Logger
-}
diff --git a/vendor/go.etcd.io/etcd/clientv3/balancer/connectivity.go b/vendor/go.etcd.io/etcd/clientv3/balancer/connectivity.go
deleted file mode 100644
index 6cdeb3f..0000000
--- a/vendor/go.etcd.io/etcd/clientv3/balancer/connectivity.go
+++ /dev/null
@@ -1,58 +0,0 @@
-// Copyright 2018 The etcd Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package balancer
-
-import "google.golang.org/grpc/connectivity"
-
-// connectivityStateEvaluator gets updated by addrConns when their
-// states transition, based on which it evaluates the state of
-// ClientConn.
-type connectivityStateEvaluator struct {
- numReady uint64 // Number of addrConns in ready state.
- numConnecting uint64 // Number of addrConns in connecting state.
- numTransientFailure uint64 // Number of addrConns in transientFailure.
-}
-
-// recordTransition records state change happening in every subConn and based on
-// that it evaluates what aggregated state should be.
-// It can only transition between Ready, Connecting and TransientFailure. Other states,
-// Idle and Shutdown are transitioned into by ClientConn; in the beginning of the connection
-// before any subConn is created ClientConn is in idle state. In the end when ClientConn
-// closes it is in Shutdown state.
-//
-// recordTransition should only be called synchronously from the same goroutine.
-func (cse *connectivityStateEvaluator) recordTransition(oldState, newState connectivity.State) connectivity.State {
- // Update counters.
- for idx, state := range []connectivity.State{oldState, newState} {
- updateVal := 2*uint64(idx) - 1 // -1 for oldState and +1 for new.
- switch state {
- case connectivity.Ready:
- cse.numReady += updateVal
- case connectivity.Connecting:
- cse.numConnecting += updateVal
- case connectivity.TransientFailure:
- cse.numTransientFailure += updateVal
- }
- }
-
- // Evaluate.
- if cse.numReady > 0 {
- return connectivity.Ready
- }
- if cse.numConnecting > 0 {
- return connectivity.Connecting
- }
- return connectivity.TransientFailure
-}
diff --git a/vendor/go.etcd.io/etcd/clientv3/balancer/connectivity/connectivity.go b/vendor/go.etcd.io/etcd/clientv3/balancer/connectivity/connectivity.go
new file mode 100644
index 0000000..4c4ad36
--- /dev/null
+++ b/vendor/go.etcd.io/etcd/clientv3/balancer/connectivity/connectivity.go
@@ -0,0 +1,93 @@
+// Copyright 2019 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package connectivity implements client connectivity operations.
+package connectivity
+
+import (
+ "sync"
+
+ "go.uber.org/zap"
+ "google.golang.org/grpc/connectivity"
+)
+
+// Recorder records gRPC connectivity.
+type Recorder interface {
+ GetCurrentState() connectivity.State
+ RecordTransition(oldState, newState connectivity.State)
+}
+
+// New returns a new Recorder.
+func New(lg *zap.Logger) Recorder {
+ return &recorder{lg: lg}
+}
+
+// recorder takes the connectivity states of multiple SubConns
+// and returns one aggregated connectivity state.
+// ref. https://github.com/grpc/grpc-go/blob/master/balancer/balancer.go
+type recorder struct {
+ lg *zap.Logger
+
+ mu sync.RWMutex
+
+ cur connectivity.State
+
+ numReady uint64 // Number of addrConns in ready state.
+ numConnecting uint64 // Number of addrConns in connecting state.
+ numTransientFailure uint64 // Number of addrConns in transientFailure.
+}
+
+func (rc *recorder) GetCurrentState() (state connectivity.State) {
+ rc.mu.RLock()
+ defer rc.mu.RUnlock()
+ return rc.cur
+}
+
+// RecordTransition records state change happening in subConn and based on that
+// it evaluates what aggregated state should be.
+//
+// - If at least one SubConn in Ready, the aggregated state is Ready;
+// - Else if at least one SubConn in Connecting, the aggregated state is Connecting;
+// - Else the aggregated state is TransientFailure.
+//
+// Idle and Shutdown are not considered.
+//
+// ref. https://github.com/grpc/grpc-go/blob/master/balancer/balancer.go
+func (rc *recorder) RecordTransition(oldState, newState connectivity.State) {
+ rc.mu.Lock()
+ defer rc.mu.Unlock()
+
+ for idx, state := range []connectivity.State{oldState, newState} {
+ updateVal := 2*uint64(idx) - 1 // -1 for oldState and +1 for new.
+ switch state {
+ case connectivity.Ready:
+ rc.numReady += updateVal
+ case connectivity.Connecting:
+ rc.numConnecting += updateVal
+ case connectivity.TransientFailure:
+ rc.numTransientFailure += updateVal
+ default:
+ rc.lg.Warn("connectivity recorder received unknown state", zap.String("connectivity-state", state.String()))
+ }
+ }
+
+ switch { // must be exclusive, no overlap
+ case rc.numReady > 0:
+ rc.cur = connectivity.Ready
+ case rc.numConnecting > 0:
+ rc.cur = connectivity.Connecting
+ default:
+ rc.cur = connectivity.TransientFailure
+ }
+}
diff --git a/vendor/go.etcd.io/etcd/clientv3/balancer/doc.go b/vendor/go.etcd.io/etcd/clientv3/balancer/doc.go
deleted file mode 100644
index 45af5e9..0000000
--- a/vendor/go.etcd.io/etcd/clientv3/balancer/doc.go
+++ /dev/null
@@ -1,16 +0,0 @@
-// Copyright 2018 The etcd Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Package balancer implements client balancer.
-package balancer
diff --git a/vendor/go.etcd.io/etcd/clientv3/balancer/grpc1.7-health.go b/vendor/go.etcd.io/etcd/clientv3/balancer/grpc1.7-health.go
deleted file mode 100644
index 2153767..0000000
--- a/vendor/go.etcd.io/etcd/clientv3/balancer/grpc1.7-health.go
+++ /dev/null
@@ -1,657 +0,0 @@
-// Copyright 2018 The etcd Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package balancer
-
-import (
- "context"
- "errors"
- "io/ioutil"
- "net/url"
- "strings"
- "sync"
- "time"
-
- "google.golang.org/grpc"
- "google.golang.org/grpc/codes"
- "google.golang.org/grpc/grpclog"
- healthpb "google.golang.org/grpc/health/grpc_health_v1"
- "google.golang.org/grpc/status"
-)
-
-// TODO: replace with something better
-var lg = grpclog.NewLoggerV2(ioutil.Discard, ioutil.Discard, ioutil.Discard)
-
-const (
- minHealthRetryDuration = 3 * time.Second
- unknownService = "unknown service grpc.health.v1.Health"
-)
-
-// ErrNoAddrAvailable is returned by Get() when the balancer does not have
-// any active connection to endpoints at the time.
-// This error is returned only when opts.BlockingWait is true.
-var ErrNoAddrAvailable = status.Error(codes.Unavailable, "there is no address available")
-
-type NotifyMsg int
-
-const (
- NotifyReset NotifyMsg = iota
- NotifyNext
-)
-
-// GRPC17Health does the bare minimum to expose multiple eps
-// to the grpc reconnection code path
-type GRPC17Health struct {
- // addrs are the client's endpoint addresses for grpc
- addrs []grpc.Address
-
- // eps holds the raw endpoints from the client
- eps []string
-
- // notifyCh notifies grpc of the set of addresses for connecting
- notifyCh chan []grpc.Address
-
- // readyc closes once the first connection is up
- readyc chan struct{}
- readyOnce sync.Once
-
- // healthCheck checks an endpoint's health.
- healthCheck func(ep string) (bool, error)
- healthCheckTimeout time.Duration
-
- unhealthyMu sync.RWMutex
- unhealthyHostPorts map[string]time.Time
-
- // mu protects all fields below.
- mu sync.RWMutex
-
- // upc closes when pinAddr transitions from empty to non-empty or the balancer closes.
- upc chan struct{}
-
- // downc closes when grpc calls down() on pinAddr
- downc chan struct{}
-
- // stopc is closed to signal updateNotifyLoop should stop.
- stopc chan struct{}
- stopOnce sync.Once
- wg sync.WaitGroup
-
- // donec closes when all goroutines are exited
- donec chan struct{}
-
- // updateAddrsC notifies updateNotifyLoop to update addrs.
- updateAddrsC chan NotifyMsg
-
- // grpc issues TLS cert checks using the string passed into dial so
- // that string must be the host. To recover the full scheme://host URL,
- // have a map from hosts to the original endpoint.
- hostPort2ep map[string]string
-
- // pinAddr is the currently pinned address; set to the empty string on
- // initialization and shutdown.
- pinAddr string
-
- closed bool
-}
-
-// DialFunc defines gRPC dial function.
-type DialFunc func(ep string, dopts ...grpc.DialOption) (*grpc.ClientConn, error)
-
-// NewGRPC17Health returns a new health balancer with gRPC v1.7.
-func NewGRPC17Health(
- eps []string,
- timeout time.Duration,
- dialFunc DialFunc,
-) *GRPC17Health {
- notifyCh := make(chan []grpc.Address)
- addrs := eps2addrs(eps)
- hb := &GRPC17Health{
- addrs: addrs,
- eps: eps,
- notifyCh: notifyCh,
- readyc: make(chan struct{}),
- healthCheck: func(ep string) (bool, error) { return grpcHealthCheck(ep, dialFunc) },
- unhealthyHostPorts: make(map[string]time.Time),
- upc: make(chan struct{}),
- stopc: make(chan struct{}),
- downc: make(chan struct{}),
- donec: make(chan struct{}),
- updateAddrsC: make(chan NotifyMsg),
- hostPort2ep: getHostPort2ep(eps),
- }
- if timeout < minHealthRetryDuration {
- timeout = minHealthRetryDuration
- }
- hb.healthCheckTimeout = timeout
-
- close(hb.downc)
- go hb.updateNotifyLoop()
- hb.wg.Add(1)
- go func() {
- defer hb.wg.Done()
- hb.updateUnhealthy()
- }()
- return hb
-}
-
-func (b *GRPC17Health) Start(target string, config grpc.BalancerConfig) error { return nil }
-
-func (b *GRPC17Health) ConnectNotify() <-chan struct{} {
- b.mu.Lock()
- defer b.mu.Unlock()
- return b.upc
-}
-
-func (b *GRPC17Health) UpdateAddrsC() chan NotifyMsg { return b.updateAddrsC }
-func (b *GRPC17Health) StopC() chan struct{} { return b.stopc }
-
-func (b *GRPC17Health) Ready() <-chan struct{} { return b.readyc }
-
-func (b *GRPC17Health) Endpoint(hostPort string) string {
- b.mu.RLock()
- defer b.mu.RUnlock()
- return b.hostPort2ep[hostPort]
-}
-
-func (b *GRPC17Health) Pinned() string {
- b.mu.RLock()
- defer b.mu.RUnlock()
- return b.pinAddr
-}
-
-func (b *GRPC17Health) HostPortError(hostPort string, err error) {
- if b.Endpoint(hostPort) == "" {
- lg.Infof("clientv3/balancer: %q is stale (skip marking as unhealthy on %q)", hostPort, err.Error())
- return
- }
-
- b.unhealthyMu.Lock()
- b.unhealthyHostPorts[hostPort] = time.Now()
- b.unhealthyMu.Unlock()
- lg.Infof("clientv3/balancer: %q is marked unhealthy (%q)", hostPort, err.Error())
-}
-
-func (b *GRPC17Health) removeUnhealthy(hostPort, msg string) {
- if b.Endpoint(hostPort) == "" {
- lg.Infof("clientv3/balancer: %q was not in unhealthy (%q)", hostPort, msg)
- return
- }
-
- b.unhealthyMu.Lock()
- delete(b.unhealthyHostPorts, hostPort)
- b.unhealthyMu.Unlock()
- lg.Infof("clientv3/balancer: %q is removed from unhealthy (%q)", hostPort, msg)
-}
-
-func (b *GRPC17Health) countUnhealthy() (count int) {
- b.unhealthyMu.RLock()
- count = len(b.unhealthyHostPorts)
- b.unhealthyMu.RUnlock()
- return count
-}
-
-func (b *GRPC17Health) isUnhealthy(hostPort string) (unhealthy bool) {
- b.unhealthyMu.RLock()
- _, unhealthy = b.unhealthyHostPorts[hostPort]
- b.unhealthyMu.RUnlock()
- return unhealthy
-}
-
-func (b *GRPC17Health) cleanupUnhealthy() {
- b.unhealthyMu.Lock()
- for k, v := range b.unhealthyHostPorts {
- if time.Since(v) > b.healthCheckTimeout {
- delete(b.unhealthyHostPorts, k)
- lg.Infof("clientv3/balancer: removed %q from unhealthy after %v", k, b.healthCheckTimeout)
- }
- }
- b.unhealthyMu.Unlock()
-}
-
-func (b *GRPC17Health) liveAddrs() ([]grpc.Address, map[string]struct{}) {
- unhealthyCnt := b.countUnhealthy()
-
- b.mu.RLock()
- defer b.mu.RUnlock()
-
- hbAddrs := b.addrs
- if len(b.addrs) == 1 || unhealthyCnt == 0 || unhealthyCnt == len(b.addrs) {
- liveHostPorts := make(map[string]struct{}, len(b.hostPort2ep))
- for k := range b.hostPort2ep {
- liveHostPorts[k] = struct{}{}
- }
- return hbAddrs, liveHostPorts
- }
-
- addrs := make([]grpc.Address, 0, len(b.addrs)-unhealthyCnt)
- liveHostPorts := make(map[string]struct{}, len(addrs))
- for _, addr := range b.addrs {
- if !b.isUnhealthy(addr.Addr) {
- addrs = append(addrs, addr)
- liveHostPorts[addr.Addr] = struct{}{}
- }
- }
- return addrs, liveHostPorts
-}
-
-func (b *GRPC17Health) updateUnhealthy() {
- for {
- select {
- case <-time.After(b.healthCheckTimeout):
- b.cleanupUnhealthy()
- pinned := b.Pinned()
- if pinned == "" || b.isUnhealthy(pinned) {
- select {
- case b.updateAddrsC <- NotifyNext:
- case <-b.stopc:
- return
- }
- }
- case <-b.stopc:
- return
- }
- }
-}
-
-// NeedUpdate returns true if all connections are down or
-// addresses do not include current pinned address.
-func (b *GRPC17Health) NeedUpdate() bool {
- // updating notifyCh can trigger new connections,
- // need update addrs if all connections are down
- // or addrs does not include pinAddr.
- b.mu.RLock()
- update := !hasAddr(b.addrs, b.pinAddr)
- b.mu.RUnlock()
- return update
-}
-
-func (b *GRPC17Health) UpdateAddrs(eps ...string) {
- np := getHostPort2ep(eps)
-
- b.mu.Lock()
- defer b.mu.Unlock()
-
- match := len(np) == len(b.hostPort2ep)
- if match {
- for k, v := range np {
- if b.hostPort2ep[k] != v {
- match = false
- break
- }
- }
- }
- if match {
- // same endpoints, so no need to update address
- return
- }
-
- b.hostPort2ep = np
- b.addrs, b.eps = eps2addrs(eps), eps
-
- b.unhealthyMu.Lock()
- b.unhealthyHostPorts = make(map[string]time.Time)
- b.unhealthyMu.Unlock()
-}
-
-func (b *GRPC17Health) Next() {
- b.mu.RLock()
- downc := b.downc
- b.mu.RUnlock()
- select {
- case b.updateAddrsC <- NotifyNext:
- case <-b.stopc:
- }
- // wait until disconnect so new RPCs are not issued on old connection
- select {
- case <-downc:
- case <-b.stopc:
- }
-}
-
-func (b *GRPC17Health) updateNotifyLoop() {
- defer close(b.donec)
-
- for {
- b.mu.RLock()
- upc, downc, addr := b.upc, b.downc, b.pinAddr
- b.mu.RUnlock()
- // downc or upc should be closed
- select {
- case <-downc:
- downc = nil
- default:
- }
- select {
- case <-upc:
- upc = nil
- default:
- }
- switch {
- case downc == nil && upc == nil:
- // stale
- select {
- case <-b.stopc:
- return
- default:
- }
- case downc == nil:
- b.notifyAddrs(NotifyReset)
- select {
- case <-upc:
- case msg := <-b.updateAddrsC:
- b.notifyAddrs(msg)
- case <-b.stopc:
- return
- }
- case upc == nil:
- select {
- // close connections that are not the pinned address
- case b.notifyCh <- []grpc.Address{{Addr: addr}}:
- case <-downc:
- case <-b.stopc:
- return
- }
- select {
- case <-downc:
- b.notifyAddrs(NotifyReset)
- case msg := <-b.updateAddrsC:
- b.notifyAddrs(msg)
- case <-b.stopc:
- return
- }
- }
- }
-}
-
-func (b *GRPC17Health) notifyAddrs(msg NotifyMsg) {
- if msg == NotifyNext {
- select {
- case b.notifyCh <- []grpc.Address{}:
- case <-b.stopc:
- return
- }
- }
- b.mu.RLock()
- pinAddr := b.pinAddr
- downc := b.downc
- b.mu.RUnlock()
- addrs, hostPorts := b.liveAddrs()
-
- var waitDown bool
- if pinAddr != "" {
- _, ok := hostPorts[pinAddr]
- waitDown = !ok
- }
-
- select {
- case b.notifyCh <- addrs:
- if waitDown {
- select {
- case <-downc:
- case <-b.stopc:
- }
- }
- case <-b.stopc:
- }
-}
-
-func (b *GRPC17Health) Up(addr grpc.Address) func(error) {
- if !b.mayPin(addr) {
- return func(err error) {}
- }
-
- b.mu.Lock()
- defer b.mu.Unlock()
-
- // gRPC might call Up after it called Close. We add this check
- // to "fix" it up at application layer. Otherwise, will panic
- // if b.upc is already closed.
- if b.closed {
- return func(err error) {}
- }
-
- // gRPC might call Up on a stale address.
- // Prevent updating pinAddr with a stale address.
- if !hasAddr(b.addrs, addr.Addr) {
- return func(err error) {}
- }
-
- if b.pinAddr != "" {
- lg.Infof("clientv3/balancer: %q is up but not pinned (already pinned %q)", addr.Addr, b.pinAddr)
- return func(err error) {}
- }
-
- // notify waiting Get()s and pin first connected address
- close(b.upc)
- b.downc = make(chan struct{})
- b.pinAddr = addr.Addr
- lg.Infof("clientv3/balancer: pin %q", addr.Addr)
-
- // notify client that a connection is up
- b.readyOnce.Do(func() { close(b.readyc) })
-
- return func(err error) {
- // If connected to a black hole endpoint or a killed server, the gRPC ping
- // timeout will induce a network I/O error, and retrying until success;
- // finding healthy endpoint on retry could take several timeouts and redials.
- // To avoid wasting retries, gray-list unhealthy endpoints.
- b.HostPortError(addr.Addr, err)
-
- b.mu.Lock()
- b.upc = make(chan struct{})
- close(b.downc)
- b.pinAddr = ""
- b.mu.Unlock()
- lg.Infof("clientv3/balancer: unpin %q (%q)", addr.Addr, err.Error())
- }
-}
-
-func (b *GRPC17Health) mayPin(addr grpc.Address) bool {
- if b.Endpoint(addr.Addr) == "" { // stale host:port
- return false
- }
-
- b.unhealthyMu.RLock()
- unhealthyCnt := len(b.unhealthyHostPorts)
- failedTime, bad := b.unhealthyHostPorts[addr.Addr]
- b.unhealthyMu.RUnlock()
-
- b.mu.RLock()
- skip := len(b.addrs) == 1 || unhealthyCnt == 0 || len(b.addrs) == unhealthyCnt
- b.mu.RUnlock()
- if skip || !bad {
- return true
- }
-
- // prevent isolated member's endpoint from being infinitely retried, as follows:
- // 1. keepalive pings detects GoAway with http2.ErrCodeEnhanceYourCalm
- // 2. balancer 'Up' unpins with grpc: failed with network I/O error
- // 3. grpc-healthcheck still SERVING, thus retry to pin
- // instead, return before grpc-healthcheck if failed within healthcheck timeout
- if elapsed := time.Since(failedTime); elapsed < b.healthCheckTimeout {
- lg.Infof("clientv3/balancer: %q is up but not pinned (failed %v ago, require minimum %v after failure)", addr.Addr, elapsed, b.healthCheckTimeout)
- return false
- }
-
- if ok, _ := b.healthCheck(addr.Addr); ok {
- b.removeUnhealthy(addr.Addr, "health check success")
- return true
- }
-
- b.HostPortError(addr.Addr, errors.New("health check failed"))
- return false
-}
-
-func (b *GRPC17Health) Get(ctx context.Context, opts grpc.BalancerGetOptions) (grpc.Address, func(), error) {
- var (
- addr string
- closed bool
- )
-
- // If opts.BlockingWait is false (for fail-fast RPCs), it should return
- // an address it has notified via Notify immediately instead of blocking.
- if !opts.BlockingWait {
- b.mu.RLock()
- closed = b.closed
- addr = b.pinAddr
- b.mu.RUnlock()
- if closed {
- return grpc.Address{Addr: ""}, nil, grpc.ErrClientConnClosing
- }
- if addr == "" {
- return grpc.Address{Addr: ""}, nil, ErrNoAddrAvailable
- }
- return grpc.Address{Addr: addr}, func() {}, nil
- }
-
- for {
- b.mu.RLock()
- ch := b.upc
- b.mu.RUnlock()
- select {
- case <-ch:
- case <-b.donec:
- return grpc.Address{Addr: ""}, nil, grpc.ErrClientConnClosing
- case <-ctx.Done():
- return grpc.Address{Addr: ""}, nil, ctx.Err()
- }
- b.mu.RLock()
- closed = b.closed
- addr = b.pinAddr
- b.mu.RUnlock()
- // Close() which sets b.closed = true can be called before Get(), Get() must exit if balancer is closed.
- if closed {
- return grpc.Address{Addr: ""}, nil, grpc.ErrClientConnClosing
- }
- if addr != "" {
- break
- }
- }
- return grpc.Address{Addr: addr}, func() {}, nil
-}
-
-func (b *GRPC17Health) Notify() <-chan []grpc.Address { return b.notifyCh }
-
-func (b *GRPC17Health) Close() error {
- b.mu.Lock()
- // In case gRPC calls close twice. TODO: remove the checking
- // when we are sure that gRPC wont call close twice.
- if b.closed {
- b.mu.Unlock()
- <-b.donec
- return nil
- }
- b.closed = true
- b.stopOnce.Do(func() { close(b.stopc) })
- b.pinAddr = ""
-
- // In the case of following scenario:
- // 1. upc is not closed; no pinned address
- // 2. client issues an RPC, calling invoke(), which calls Get(), enters for loop, blocks
- // 3. client.conn.Close() calls balancer.Close(); closed = true
- // 4. for loop in Get() never exits since ctx is the context passed in by the client and may not be canceled
- // we must close upc so Get() exits from blocking on upc
- select {
- case <-b.upc:
- default:
- // terminate all waiting Get()s
- close(b.upc)
- }
-
- b.mu.Unlock()
- b.wg.Wait()
-
- // wait for updateNotifyLoop to finish
- <-b.donec
- close(b.notifyCh)
-
- return nil
-}
-
-func grpcHealthCheck(ep string, dialFunc func(ep string, dopts ...grpc.DialOption) (*grpc.ClientConn, error)) (bool, error) {
- conn, err := dialFunc(ep)
- if err != nil {
- return false, err
- }
- defer conn.Close()
- cli := healthpb.NewHealthClient(conn)
- ctx, cancel := context.WithTimeout(context.Background(), time.Second)
- resp, err := cli.Check(ctx, &healthpb.HealthCheckRequest{})
- cancel()
- if err != nil {
- if s, ok := status.FromError(err); ok && s.Code() == codes.Unavailable {
- if s.Message() == unknownService { // etcd < v3.3.0
- return true, nil
- }
- }
- return false, err
- }
- return resp.Status == healthpb.HealthCheckResponse_SERVING, nil
-}
-
-func hasAddr(addrs []grpc.Address, targetAddr string) bool {
- for _, addr := range addrs {
- if targetAddr == addr.Addr {
- return true
- }
- }
- return false
-}
-
-func getHost(ep string) string {
- url, uerr := url.Parse(ep)
- if uerr != nil || !strings.Contains(ep, "://") {
- return ep
- }
- return url.Host
-}
-
-func eps2addrs(eps []string) []grpc.Address {
- addrs := make([]grpc.Address, len(eps))
- for i := range eps {
- addrs[i].Addr = getHost(eps[i])
- }
- return addrs
-}
-
-func getHostPort2ep(eps []string) map[string]string {
- hm := make(map[string]string, len(eps))
- for i := range eps {
- _, host, _ := parseEndpoint(eps[i])
- hm[host] = eps[i]
- }
- return hm
-}
-
-func parseEndpoint(endpoint string) (proto string, host string, scheme string) {
- proto = "tcp"
- host = endpoint
- url, uerr := url.Parse(endpoint)
- if uerr != nil || !strings.Contains(endpoint, "://") {
- return proto, host, scheme
- }
- scheme = url.Scheme
-
- // strip scheme:// prefix since grpc dials by host
- host = url.Host
- switch url.Scheme {
- case "http", "https":
- case "unix", "unixs":
- proto = "unix"
- host = url.Host + url.Path
- default:
- proto, host = "", ""
- }
- return proto, host, scheme
-}
diff --git a/vendor/go.etcd.io/etcd/clientv3/balancer/picker/err.go b/vendor/go.etcd.io/etcd/clientv3/balancer/picker/err.go
index c70ce15..9e04378 100644
--- a/vendor/go.etcd.io/etcd/clientv3/balancer/picker/err.go
+++ b/vendor/go.etcd.io/etcd/clientv3/balancer/picker/err.go
@@ -22,13 +22,18 @@
// NewErr returns a picker that always returns err on "Pick".
func NewErr(err error) Picker {
- return &errPicker{err: err}
+ return &errPicker{p: Error, err: err}
}
type errPicker struct {
+ p Policy
err error
}
-func (p *errPicker) Pick(context.Context, balancer.PickOptions) (balancer.SubConn, func(balancer.DoneInfo), error) {
- return nil, nil, p.err
+func (ep *errPicker) String() string {
+ return ep.p.String()
+}
+
+func (ep *errPicker) Pick(context.Context, balancer.PickOptions) (balancer.SubConn, func(balancer.DoneInfo), error) {
+ return nil, nil, ep.err
}
diff --git a/vendor/go.etcd.io/etcd/clientv3/balancer/picker/picker.go b/vendor/go.etcd.io/etcd/clientv3/balancer/picker/picker.go
index 7ea761b..bd1a5d2 100644
--- a/vendor/go.etcd.io/etcd/clientv3/balancer/picker/picker.go
+++ b/vendor/go.etcd.io/etcd/clientv3/balancer/picker/picker.go
@@ -15,10 +15,77 @@
package picker
import (
+ "fmt"
+
+ "go.uber.org/zap"
"google.golang.org/grpc/balancer"
+ "google.golang.org/grpc/resolver"
)
// Picker defines balancer Picker methods.
type Picker interface {
balancer.Picker
+ String() string
+}
+
+// Config defines picker configuration.
+type Config struct {
+ // Policy specifies etcd clientv3's built in balancer policy.
+ Policy Policy
+
+ // Logger defines picker logging object.
+ Logger *zap.Logger
+
+ // SubConnToResolverAddress maps each gRPC sub-connection to an address.
+ // Basically, it is a list of addresses that the Picker can pick from.
+ SubConnToResolverAddress map[balancer.SubConn]resolver.Address
+}
+
+// Policy defines balancer picker policy.
+type Policy uint8
+
+const (
+ // Error is error picker policy.
+ Error Policy = iota
+
+ // RoundrobinBalanced balances loads over multiple endpoints
+ // and implements failover in roundrobin fashion.
+ RoundrobinBalanced
+
+ // Custom defines custom balancer picker.
+ // TODO: custom picker is not supported yet.
+ Custom
+)
+
+func (p Policy) String() string {
+ switch p {
+ case Error:
+ return "picker-error"
+
+ case RoundrobinBalanced:
+ return "picker-roundrobin-balanced"
+
+ case Custom:
+ panic("'custom' picker policy is not supported yet")
+
+ default:
+ panic(fmt.Errorf("invalid balancer picker policy (%d)", p))
+ }
+}
+
+// New creates a new Picker.
+func New(cfg Config) Picker {
+ switch cfg.Policy {
+ case Error:
+ panic("'error' picker policy is not supported here; use 'picker.NewErr'")
+
+ case RoundrobinBalanced:
+ return newRoundrobinBalanced(cfg)
+
+ case Custom:
+ panic("'custom' picker policy is not supported yet")
+
+ default:
+ panic(fmt.Errorf("invalid balancer picker policy (%d)", cfg.Policy))
+ }
}
diff --git a/vendor/go.etcd.io/etcd/clientv3/balancer/picker/picker_policy.go b/vendor/go.etcd.io/etcd/clientv3/balancer/picker/picker_policy.go
deleted file mode 100644
index 463ddc2..0000000
--- a/vendor/go.etcd.io/etcd/clientv3/balancer/picker/picker_policy.go
+++ /dev/null
@@ -1,49 +0,0 @@
-// Copyright 2018 The etcd Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package picker
-
-import "fmt"
-
-// Policy defines balancer picker policy.
-type Policy uint8
-
-const (
- // TODO: custom picker is not supported yet.
- // custom defines custom balancer picker.
- custom Policy = iota
-
- // RoundrobinBalanced balance loads over multiple endpoints
- // and implements failover in roundrobin fashion.
- RoundrobinBalanced Policy = iota
-
- // TODO: only send loads to pinned address "RoundrobinFailover"
- // just like how 3.3 client works
- //
- // TODO: priotize leader
- // TODO: health-check
- // TODO: weighted roundrobin
- // TODO: power of two random choice
-)
-
-func (p Policy) String() string {
- switch p {
- case custom:
- panic("'custom' picker policy is not supported yet")
- case RoundrobinBalanced:
- return "etcd-client-roundrobin-balanced"
- default:
- panic(fmt.Errorf("invalid balancer picker policy (%d)", p))
- }
-}
diff --git a/vendor/go.etcd.io/etcd/clientv3/balancer/picker/roundrobin_balanced.go b/vendor/go.etcd.io/etcd/clientv3/balancer/picker/roundrobin_balanced.go
index b043d57..1b8b285 100644
--- a/vendor/go.etcd.io/etcd/clientv3/balancer/picker/roundrobin_balanced.go
+++ b/vendor/go.etcd.io/etcd/clientv3/balancer/picker/roundrobin_balanced.go
@@ -24,32 +24,33 @@
"google.golang.org/grpc/resolver"
)
-// NewRoundrobinBalanced returns a new roundrobin balanced picker.
-func NewRoundrobinBalanced(
- lg *zap.Logger,
- scs []balancer.SubConn,
- addrToSc map[resolver.Address]balancer.SubConn,
- scToAddr map[balancer.SubConn]resolver.Address,
-) Picker {
+// newRoundrobinBalanced returns a new roundrobin balanced picker.
+func newRoundrobinBalanced(cfg Config) Picker {
+ scs := make([]balancer.SubConn, 0, len(cfg.SubConnToResolverAddress))
+ for sc := range cfg.SubConnToResolverAddress {
+ scs = append(scs, sc)
+ }
return &rrBalanced{
- lg: lg,
+ p: RoundrobinBalanced,
+ lg: cfg.Logger,
scs: scs,
- addrToSc: addrToSc,
- scToAddr: scToAddr,
+ scToAddr: cfg.SubConnToResolverAddress,
}
}
type rrBalanced struct {
+ p Policy
+
lg *zap.Logger
- mu sync.RWMutex
- next int
- scs []balancer.SubConn
-
- addrToSc map[resolver.Address]balancer.SubConn
+ mu sync.RWMutex
+ next int
+ scs []balancer.SubConn
scToAddr map[balancer.SubConn]resolver.Address
}
+func (rb *rrBalanced) String() string { return rb.p.String() }
+
// Pick is called for every client request.
func (rb *rrBalanced) Pick(ctx context.Context, opts balancer.PickOptions) (balancer.SubConn, func(balancer.DoneInfo), error) {
rb.mu.RLock()
@@ -68,6 +69,7 @@
rb.lg.Debug(
"picked",
+ zap.String("picker", rb.p.String()),
zap.String("address", picked),
zap.Int("subconn-index", cur),
zap.Int("subconn-size", n),
@@ -77,6 +79,7 @@
// TODO: error handling?
fss := []zapcore.Field{
zap.Error(info.Err),
+ zap.String("picker", rb.p.String()),
zap.String("address", picked),
zap.Bool("success", info.Err == nil),
zap.Bool("bytes-sent", info.BytesSent),
diff --git a/vendor/go.etcd.io/etcd/clientv3/balancer/utils.go b/vendor/go.etcd.io/etcd/clientv3/balancer/utils.go
index a11faeb..48eb875 100644
--- a/vendor/go.etcd.io/etcd/clientv3/balancer/utils.go
+++ b/vendor/go.etcd.io/etcd/clientv3/balancer/utils.go
@@ -29,9 +29,9 @@
return fmt.Sprintf("%p", sc)
}
-func scsToStrings(scs map[resolver.Address]balancer.SubConn) (ss []string) {
+func scsToStrings(scs map[balancer.SubConn]resolver.Address) (ss []string) {
ss = make([]string, 0, len(scs))
- for a, sc := range scs {
+ for sc, a := range scs {
ss = append(ss, fmt.Sprintf("%s (%s)", a.Addr, scToString(sc)))
}
sort.Strings(ss)
diff --git a/vendor/go.etcd.io/etcd/clientv3/client.go b/vendor/go.etcd.io/etcd/clientv3/client.go
index 445ecfe..d6000a8 100644
--- a/vendor/go.etcd.io/etcd/clientv3/client.go
+++ b/vendor/go.etcd.io/etcd/clientv3/client.go
@@ -16,7 +16,6 @@
import (
"context"
- "crypto/tls"
"errors"
"fmt"
"net"
@@ -30,12 +29,13 @@
"go.etcd.io/etcd/clientv3/balancer"
"go.etcd.io/etcd/clientv3/balancer/picker"
"go.etcd.io/etcd/clientv3/balancer/resolver/endpoint"
+ "go.etcd.io/etcd/clientv3/credentials"
"go.etcd.io/etcd/etcdserver/api/v3rpc/rpctypes"
"go.etcd.io/etcd/pkg/logutil"
"go.uber.org/zap"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
- "google.golang.org/grpc/credentials"
+ grpccredentials "google.golang.org/grpc/credentials"
"google.golang.org/grpc/keepalive"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/status"
@@ -51,12 +51,17 @@
func init() {
lg := zap.NewNop()
if os.Getenv("ETCD_CLIENT_DEBUG") != "" {
+ lcfg := logutil.DefaultZapLoggerConfig
+ lcfg.Level = zap.NewAtomicLevelAt(zap.DebugLevel)
+
var err error
- lg, err = zap.NewProductionConfig().Build() // info level logging
+ lg, err = lcfg.Build() // info level logging
if err != nil {
panic(err)
}
}
+
+ // TODO: support custom balancer
balancer.RegisterBuilder(balancer.Config{
Policy: picker.RoundrobinBalanced,
Name: roundRobinBalancerName,
@@ -76,10 +81,9 @@
conn *grpc.ClientConn
cfg Config
- creds *credentials.TransportCredentials
- balancer balancer.Balancer
+ creds grpccredentials.TransportCredentials
resolverGroup *endpoint.ResolverGroup
- mu *sync.Mutex
+ mu *sync.RWMutex
ctx context.Context
cancel context.CancelFunc
@@ -87,9 +91,8 @@
// Username is a user name for authentication.
Username string
// Password is a password for authentication.
- Password string
- // tokenCred is an instance of WithPerRPCCredentials()'s argument
- tokenCred *authTokenCredential
+ Password string
+ authTokenBundle credentials.Bundle
callOpts []grpc.CallOption
@@ -126,8 +129,12 @@
// Close shuts down the client's etcd connections.
func (c *Client) Close() error {
c.cancel()
- c.Watcher.Close()
- c.Lease.Close()
+ if c.Watcher != nil {
+ c.Watcher.Close()
+ }
+ if c.Lease != nil {
+ c.Lease.Close()
+ }
if c.resolverGroup != nil {
c.resolverGroup.Close()
}
@@ -143,11 +150,13 @@
func (c *Client) Ctx() context.Context { return c.ctx }
// Endpoints lists the registered endpoints for the client.
-func (c *Client) Endpoints() (eps []string) {
+func (c *Client) Endpoints() []string {
// copy the slice; protect original endpoints from being changed
- eps = make([]string, len(c.cfg.Endpoints))
+ c.mu.RLock()
+ defer c.mu.RUnlock()
+ eps := make([]string, len(c.cfg.Endpoints))
copy(eps, c.cfg.Endpoints)
- return
+ return eps
}
// SetEndpoints updates client's endpoints.
@@ -192,24 +201,7 @@
}
}
-type authTokenCredential struct {
- token string
- tokenMu *sync.RWMutex
-}
-
-func (cred authTokenCredential) RequireTransportSecurity() bool {
- return false
-}
-
-func (cred authTokenCredential) GetRequestMetadata(ctx context.Context, s ...string) (map[string]string, error) {
- cred.tokenMu.RLock()
- defer cred.tokenMu.RUnlock()
- return map[string]string{
- rpctypes.TokenFieldNameGRPC: cred.token,
- }, nil
-}
-
-func (c *Client) processCreds(scheme string) (creds *credentials.TransportCredentials) {
+func (c *Client) processCreds(scheme string) (creds grpccredentials.TransportCredentials) {
creds = c.creds
switch scheme {
case "unix":
@@ -219,9 +211,7 @@
if creds != nil {
break
}
- tlsconfig := &tls.Config{}
- emptyCreds := credentials.NewTLS(tlsconfig)
- creds = &emptyCreds
+ creds = credentials.NewBundle(credentials.Config{}).TransportCredentials()
default:
creds = nil
}
@@ -229,7 +219,7 @@
}
// dialSetupOpts gives the dial opts prior to any authentication.
-func (c *Client) dialSetupOpts(creds *credentials.TransportCredentials, dopts ...grpc.DialOption) (opts []grpc.DialOption, err error) {
+func (c *Client) dialSetupOpts(creds grpccredentials.TransportCredentials, dopts ...grpc.DialOption) (opts []grpc.DialOption, err error) {
if c.cfg.DialKeepAliveTime > 0 {
params := keepalive.ClientParameters{
Time: c.cfg.DialKeepAliveTime,
@@ -254,7 +244,7 @@
opts = append(opts, grpc.WithDialer(f))
if creds != nil {
- opts = append(opts, grpc.WithTransportCredentials(*creds))
+ opts = append(opts, grpc.WithTransportCredentials(creds))
} else {
opts = append(opts, grpc.WithInsecure())
}
@@ -288,8 +278,8 @@
var err error // return last error in a case of fail
var auth *authenticator
- for i := 0; i < len(c.cfg.Endpoints); i++ {
- ep := c.cfg.Endpoints[i]
+ eps := c.Endpoints()
+ for _, ep := range eps {
// use dial options without dopts to avoid reusing the client balancer
var dOpts []grpc.DialOption
_, host, _ := endpoint.ParseEndpoint(ep)
@@ -317,10 +307,7 @@
continue
}
- c.tokenCred.tokenMu.Lock()
- c.tokenCred.token = resp.Token
- c.tokenCred.tokenMu.Unlock()
-
+ c.authTokenBundle.UpdateAuthToken(resp.Token)
return nil
}
@@ -337,16 +324,14 @@
}
// dial configures and dials any grpc balancer target.
-func (c *Client) dial(target string, creds *credentials.TransportCredentials, dopts ...grpc.DialOption) (*grpc.ClientConn, error) {
+func (c *Client) dial(target string, creds grpccredentials.TransportCredentials, dopts ...grpc.DialOption) (*grpc.ClientConn, error) {
opts, err := c.dialSetupOpts(creds, dopts...)
if err != nil {
return nil, fmt.Errorf("failed to configure dialer: %v", err)
}
if c.Username != "" && c.Password != "" {
- c.tokenCred = &authTokenCredential{
- tokenMu: &sync.RWMutex{},
- }
+ c.authTokenBundle = credentials.NewBundle(credentials.Config{})
ctx, cancel := c.ctx, func() {}
if c.cfg.DialTimeout > 0 {
@@ -363,7 +348,7 @@
return nil, err
}
} else {
- opts = append(opts, grpc.WithPerRPCCredentials(c.tokenCred))
+ opts = append(opts, grpc.WithPerRPCCredentials(c.authTokenBundle.PerRPCCredentials()))
}
cancel()
}
@@ -384,26 +369,25 @@
return conn, nil
}
-func (c *Client) directDialCreds(ep string) *credentials.TransportCredentials {
+func (c *Client) directDialCreds(ep string) grpccredentials.TransportCredentials {
_, hostPort, scheme := endpoint.ParseEndpoint(ep)
creds := c.creds
if len(scheme) != 0 {
creds = c.processCreds(scheme)
if creds != nil {
- c := *creds
- clone := c.Clone()
+ clone := creds.Clone()
// Set the server name must to the endpoint hostname without port since grpc
// otherwise attempts to check if x509 cert is valid for the full endpoint
// including the scheme and port, which fails.
host, _ := endpoint.ParseHostPort(hostPort)
clone.OverrideServerName(host)
- creds = &clone
+ creds = clone
}
}
return creds
}
-func (c *Client) dialWithBalancerCreds(ep string) *credentials.TransportCredentials {
+func (c *Client) dialWithBalancerCreds(ep string) grpccredentials.TransportCredentials {
_, _, scheme := endpoint.ParseEndpoint(ep)
creds := c.creds
if len(scheme) != 0 {
@@ -423,10 +407,9 @@
if cfg == nil {
cfg = &Config{}
}
- var creds *credentials.TransportCredentials
+ var creds grpccredentials.TransportCredentials
if cfg.TLS != nil {
- c := credentials.NewTLS(cfg.TLS)
- creds = &c
+ creds = credentials.NewBundle(credentials.Config{TLSConfig: cfg.TLS}).TransportCredentials()
}
// use a temporary skeleton client to bootstrap first connection
@@ -442,7 +425,7 @@
creds: creds,
ctx: ctx,
cancel: cancel,
- mu: new(sync.Mutex),
+ mu: new(sync.RWMutex),
callOpts: defaultCallOpts,
}
@@ -540,13 +523,17 @@
func (c *Client) checkVersion() (err error) {
var wg sync.WaitGroup
- errc := make(chan error, len(c.cfg.Endpoints))
+
+ eps := c.Endpoints()
+ errc := make(chan error, len(eps))
ctx, cancel := context.WithCancel(c.ctx)
if c.cfg.DialTimeout > 0 {
- ctx, cancel = context.WithTimeout(ctx, c.cfg.DialTimeout)
+ cancel()
+ ctx, cancel = context.WithTimeout(c.ctx, c.cfg.DialTimeout)
}
- wg.Add(len(c.cfg.Endpoints))
- for _, ep := range c.cfg.Endpoints {
+
+ wg.Add(len(eps))
+ for _, ep := range eps {
// if cluster is current, any endpoint gives a recent version
go func(e string) {
defer wg.Done()
@@ -558,8 +545,15 @@
vs := strings.Split(resp.Version, ".")
maj, min := 0, 0
if len(vs) >= 2 {
- maj, _ = strconv.Atoi(vs[0])
- min, rerr = strconv.Atoi(vs[1])
+ var serr error
+ if maj, serr = strconv.Atoi(vs[0]); serr != nil {
+ errc <- serr
+ return
+ }
+ if min, serr = strconv.Atoi(vs[1]); serr != nil {
+ errc <- serr
+ return
+ }
}
if maj < 3 || (maj == 3 && min < 2) {
rerr = ErrOldCluster
@@ -568,7 +562,7 @@
}(ep)
}
// wait for success
- for i := 0; i < len(c.cfg.Endpoints); i++ {
+ for range eps {
if err = <-errc; err == nil {
break
}
@@ -608,10 +602,13 @@
if err == nil {
return false
}
- ev, _ := status.FromError(err)
- // Unavailable codes mean the system will be right back.
- // (e.g., can't connect, lost leader)
- return ev.Code() == codes.Unavailable
+ ev, ok := status.FromError(err)
+ if ok {
+ // Unavailable codes mean the system will be right back.
+ // (e.g., can't connect, lost leader)
+ return ev.Code() == codes.Unavailable
+ }
+ return false
}
func toErr(ctx context.Context, err error) error {
@@ -631,9 +628,6 @@
if ctx.Err() != nil {
err = ctx.Err()
}
- case codes.Unavailable:
- case codes.FailedPrecondition:
- err = grpc.ErrClientConnClosing
}
}
return err
@@ -653,16 +647,19 @@
if err == nil {
return false
}
- // >= gRPC v1.10.x
+
+ // >= gRPC v1.23.x
s, ok := status.FromError(err)
if ok {
// connection is canceled or server has already closed the connection
return s.Code() == codes.Canceled || s.Message() == "transport is closing"
}
+
// >= gRPC v1.10.x
if err == context.Canceled {
return true
}
+
// <= gRPC v1.7.x returns 'errors.New("grpc: the client connection is closing")'
return strings.Contains(err.Error(), "grpc: the client connection is closing")
}
diff --git a/vendor/go.etcd.io/etcd/clientv3/cluster.go b/vendor/go.etcd.io/etcd/clientv3/cluster.go
index d497c05..ce97e5c 100644
--- a/vendor/go.etcd.io/etcd/clientv3/cluster.go
+++ b/vendor/go.etcd.io/etcd/clientv3/cluster.go
@@ -24,11 +24,12 @@
)
type (
- Member pb.Member
- MemberListResponse pb.MemberListResponse
- MemberAddResponse pb.MemberAddResponse
- MemberRemoveResponse pb.MemberRemoveResponse
- MemberUpdateResponse pb.MemberUpdateResponse
+ Member pb.Member
+ MemberListResponse pb.MemberListResponse
+ MemberAddResponse pb.MemberAddResponse
+ MemberRemoveResponse pb.MemberRemoveResponse
+ MemberUpdateResponse pb.MemberUpdateResponse
+ MemberPromoteResponse pb.MemberPromoteResponse
)
type Cluster interface {
@@ -38,11 +39,17 @@
// MemberAdd adds a new member into the cluster.
MemberAdd(ctx context.Context, peerAddrs []string) (*MemberAddResponse, error)
+ // MemberAddAsLearner adds a new learner member into the cluster.
+ MemberAddAsLearner(ctx context.Context, peerAddrs []string) (*MemberAddResponse, error)
+
// MemberRemove removes an existing member from the cluster.
MemberRemove(ctx context.Context, id uint64) (*MemberRemoveResponse, error)
// MemberUpdate updates the peer addresses of the member.
MemberUpdate(ctx context.Context, id uint64, peerAddrs []string) (*MemberUpdateResponse, error)
+
+ // MemberPromote promotes a member from raft learner (non-voting) to raft voting member.
+ MemberPromote(ctx context.Context, id uint64) (*MemberPromoteResponse, error)
}
type cluster struct {
@@ -67,12 +74,23 @@
}
func (c *cluster) MemberAdd(ctx context.Context, peerAddrs []string) (*MemberAddResponse, error) {
+ return c.memberAdd(ctx, peerAddrs, false)
+}
+
+func (c *cluster) MemberAddAsLearner(ctx context.Context, peerAddrs []string) (*MemberAddResponse, error) {
+ return c.memberAdd(ctx, peerAddrs, true)
+}
+
+func (c *cluster) memberAdd(ctx context.Context, peerAddrs []string, isLearner bool) (*MemberAddResponse, error) {
// fail-fast before panic in rafthttp
if _, err := types.NewURLs(peerAddrs); err != nil {
return nil, err
}
- r := &pb.MemberAddRequest{PeerURLs: peerAddrs}
+ r := &pb.MemberAddRequest{
+ PeerURLs: peerAddrs,
+ IsLearner: isLearner,
+ }
resp, err := c.remote.MemberAdd(ctx, r, c.callOpts...)
if err != nil {
return nil, toErr(ctx, err)
@@ -112,3 +130,12 @@
}
return nil, toErr(ctx, err)
}
+
+func (c *cluster) MemberPromote(ctx context.Context, id uint64) (*MemberPromoteResponse, error) {
+ r := &pb.MemberPromoteRequest{ID: id}
+ resp, err := c.remote.MemberPromote(ctx, r, c.callOpts...)
+ if err != nil {
+ return nil, toErr(ctx, err)
+ }
+ return (*MemberPromoteResponse)(resp), nil
+}
diff --git a/vendor/go.etcd.io/etcd/clientv3/concurrency/mutex.go b/vendor/go.etcd.io/etcd/clientv3/concurrency/mutex.go
index 0135341..306470b 100644
--- a/vendor/go.etcd.io/etcd/clientv3/concurrency/mutex.go
+++ b/vendor/go.etcd.io/etcd/clientv3/concurrency/mutex.go
@@ -16,6 +16,7 @@
import (
"context"
+ "errors"
"fmt"
"sync"
@@ -23,6 +24,9 @@
pb "go.etcd.io/etcd/etcdserver/etcdserverpb"
)
+// ErrLocked is returned by TryLock when Mutex is already locked by another session.
+var ErrLocked = errors.New("mutex: Locked by another session")
+
// Mutex implements the sync Locker interface with etcd
type Mutex struct {
s *Session
@@ -37,9 +41,56 @@
return &Mutex{s, pfx + "/", "", -1, nil}
}
+// TryLock locks the mutex if not already locked by another session.
+// If lock is held by another session, return immediately after attempting necessary cleanup
+// The ctx argument is used for the sending/receiving Txn RPC.
+func (m *Mutex) TryLock(ctx context.Context) error {
+ resp, err := m.tryAcquire(ctx)
+ if err != nil {
+ return err
+ }
+ // if no key on prefix / the minimum rev is key, already hold the lock
+ ownerKey := resp.Responses[1].GetResponseRange().Kvs
+ if len(ownerKey) == 0 || ownerKey[0].CreateRevision == m.myRev {
+ m.hdr = resp.Header
+ return nil
+ }
+ client := m.s.Client()
+ // Cannot lock, so delete the key
+ if _, err := client.Delete(ctx, m.myKey); err != nil {
+ return err
+ }
+ m.myKey = "\x00"
+ m.myRev = -1
+ return ErrLocked
+}
+
// Lock locks the mutex with a cancelable context. If the context is canceled
// while trying to acquire the lock, the mutex tries to clean its stale lock entry.
func (m *Mutex) Lock(ctx context.Context) error {
+ resp, err := m.tryAcquire(ctx)
+ if err != nil {
+ return err
+ }
+ // if no key on prefix / the minimum rev is key, already hold the lock
+ ownerKey := resp.Responses[1].GetResponseRange().Kvs
+ if len(ownerKey) == 0 || ownerKey[0].CreateRevision == m.myRev {
+ m.hdr = resp.Header
+ return nil
+ }
+ client := m.s.Client()
+ // wait for deletion revisions prior to myKey
+ hdr, werr := waitDeletes(ctx, client, m.pfx, m.myRev-1)
+ // release lock key if wait failed
+ if werr != nil {
+ m.Unlock(client.Ctx())
+ } else {
+ m.hdr = hdr
+ }
+ return werr
+}
+
+func (m *Mutex) tryAcquire(ctx context.Context) (*v3.TxnResponse, error) {
s := m.s
client := m.s.Client()
@@ -53,28 +104,13 @@
getOwner := v3.OpGet(m.pfx, v3.WithFirstCreate()...)
resp, err := client.Txn(ctx).If(cmp).Then(put, getOwner).Else(get, getOwner).Commit()
if err != nil {
- return err
+ return nil, err
}
m.myRev = resp.Header.Revision
if !resp.Succeeded {
m.myRev = resp.Responses[0].GetResponseRange().Kvs[0].CreateRevision
}
- // if no key on prefix / the minimum rev is key, already hold the lock
- ownerKey := resp.Responses[1].GetResponseRange().Kvs
- if len(ownerKey) == 0 || ownerKey[0].CreateRevision == m.myRev {
- m.hdr = resp.Header
- return nil
- }
-
- // wait for deletion revisions prior to myKey
- hdr, werr := waitDeletes(ctx, client, m.pfx, m.myRev-1)
- // release lock key if wait failed
- if werr != nil {
- m.Unlock(client.Ctx())
- } else {
- m.hdr = hdr
- }
- return werr
+ return resp, nil
}
func (m *Mutex) Unlock(ctx context.Context) error {
diff --git a/vendor/go.etcd.io/etcd/clientv3/concurrency/session.go b/vendor/go.etcd.io/etcd/clientv3/concurrency/session.go
index 598ec0e..97eb763 100644
--- a/vendor/go.etcd.io/etcd/clientv3/concurrency/session.go
+++ b/vendor/go.etcd.io/etcd/clientv3/concurrency/session.go
@@ -47,7 +47,7 @@
if err != nil {
return nil, err
}
- id = v3.LeaseID(resp.ID)
+ id = resp.ID
}
ctx, cancel := context.WithCancel(ops.ctx)
diff --git a/vendor/go.etcd.io/etcd/clientv3/config.go b/vendor/go.etcd.io/etcd/clientv3/config.go
index bd03768..11d447d 100644
--- a/vendor/go.etcd.io/etcd/clientv3/config.go
+++ b/vendor/go.etcd.io/etcd/clientv3/config.go
@@ -68,6 +68,8 @@
RejectOldCluster bool `json:"reject-old-cluster"`
// DialOptions is a list of dial options for the grpc client (e.g., for interceptors).
+ // For example, pass "grpc.WithBlock()" to block until the underlying connection is up.
+ // Without this, Dial returns immediately and connecting the server happens in background.
DialOptions []grpc.DialOption
// Context is the default client context; it can be used to cancel grpc dial out and
@@ -81,4 +83,6 @@
// PermitWithoutStream when set will allow client to send keepalive pings to server without any active streams(RPCs).
PermitWithoutStream bool `json:"permit-without-stream"`
+
+ // TODO: support custom balancer picker
}
diff --git a/vendor/go.etcd.io/etcd/clientv3/credentials/credentials.go b/vendor/go.etcd.io/etcd/clientv3/credentials/credentials.go
new file mode 100644
index 0000000..e6fd75c
--- /dev/null
+++ b/vendor/go.etcd.io/etcd/clientv3/credentials/credentials.go
@@ -0,0 +1,155 @@
+// Copyright 2019 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package credentials implements gRPC credential interface with etcd specific logic.
+// e.g., client handshake with custom authority parameter
+package credentials
+
+import (
+ "context"
+ "crypto/tls"
+ "net"
+ "sync"
+
+ "go.etcd.io/etcd/etcdserver/api/v3rpc/rpctypes"
+ grpccredentials "google.golang.org/grpc/credentials"
+)
+
+// Config defines gRPC credential configuration.
+type Config struct {
+ TLSConfig *tls.Config
+}
+
+// Bundle defines gRPC credential interface.
+type Bundle interface {
+ grpccredentials.Bundle
+ UpdateAuthToken(token string)
+}
+
+// NewBundle constructs a new gRPC credential bundle.
+func NewBundle(cfg Config) Bundle {
+ return &bundle{
+ tc: newTransportCredential(cfg.TLSConfig),
+ rc: newPerRPCCredential(),
+ }
+}
+
+// bundle implements "grpccredentials.Bundle" interface.
+type bundle struct {
+ tc *transportCredential
+ rc *perRPCCredential
+}
+
+func (b *bundle) TransportCredentials() grpccredentials.TransportCredentials {
+ return b.tc
+}
+
+func (b *bundle) PerRPCCredentials() grpccredentials.PerRPCCredentials {
+ return b.rc
+}
+
+func (b *bundle) NewWithMode(mode string) (grpccredentials.Bundle, error) {
+ // no-op
+ return nil, nil
+}
+
+// transportCredential implements "grpccredentials.TransportCredentials" interface.
+type transportCredential struct {
+ gtc grpccredentials.TransportCredentials
+}
+
+func newTransportCredential(cfg *tls.Config) *transportCredential {
+ return &transportCredential{
+ gtc: grpccredentials.NewTLS(cfg),
+ }
+}
+
+func (tc *transportCredential) ClientHandshake(ctx context.Context, authority string, rawConn net.Conn) (net.Conn, grpccredentials.AuthInfo, error) {
+ // Only overwrite when authority is an IP address!
+ // Let's say, a server runs SRV records on "etcd.local" that resolves
+ // to "m1.etcd.local", and its SAN field also includes "m1.etcd.local".
+ // But what if SAN does not include its resolved IP address (e.g. 127.0.0.1)?
+ // Then, the server should only authenticate using its DNS hostname "m1.etcd.local",
+ // instead of overwriting it with its IP address.
+ // And we do not overwrite "localhost" either. Only overwrite IP addresses!
+ if isIP(authority) {
+ target := rawConn.RemoteAddr().String()
+ if authority != target {
+ // When user dials with "grpc.WithDialer", "grpc.DialContext" "cc.parsedTarget"
+ // update only happens once. This is problematic, because when TLS is enabled,
+ // retries happen through "grpc.WithDialer" with static "cc.parsedTarget" from
+ // the initial dial call.
+ // If the server authenticates by IP addresses, we want to set a new endpoint as
+ // a new authority. Otherwise
+ // "transport: authentication handshake failed: x509: certificate is valid for 127.0.0.1, 192.168.121.180, not 192.168.223.156"
+ // when the new dial target is "192.168.121.180" whose certificate host name is also "192.168.121.180"
+ // but client tries to authenticate with previously set "cc.parsedTarget" field "192.168.223.156"
+ authority = target
+ }
+ }
+ return tc.gtc.ClientHandshake(ctx, authority, rawConn)
+}
+
+// return true if given string is an IP.
+func isIP(ep string) bool {
+ return net.ParseIP(ep) != nil
+}
+
+func (tc *transportCredential) ServerHandshake(rawConn net.Conn) (net.Conn, grpccredentials.AuthInfo, error) {
+ return tc.gtc.ServerHandshake(rawConn)
+}
+
+func (tc *transportCredential) Info() grpccredentials.ProtocolInfo {
+ return tc.gtc.Info()
+}
+
+func (tc *transportCredential) Clone() grpccredentials.TransportCredentials {
+ return &transportCredential{
+ gtc: tc.gtc.Clone(),
+ }
+}
+
+func (tc *transportCredential) OverrideServerName(serverNameOverride string) error {
+ return tc.gtc.OverrideServerName(serverNameOverride)
+}
+
+// perRPCCredential implements "grpccredentials.PerRPCCredentials" interface.
+type perRPCCredential struct {
+ authToken string
+ authTokenMu sync.RWMutex
+}
+
+func newPerRPCCredential() *perRPCCredential { return &perRPCCredential{} }
+
+func (rc *perRPCCredential) RequireTransportSecurity() bool { return false }
+
+func (rc *perRPCCredential) GetRequestMetadata(ctx context.Context, s ...string) (map[string]string, error) {
+ rc.authTokenMu.RLock()
+ authToken := rc.authToken
+ rc.authTokenMu.RUnlock()
+ return map[string]string{rpctypes.TokenFieldNameGRPC: authToken}, nil
+}
+
+func (b *bundle) UpdateAuthToken(token string) {
+ if b.rc == nil {
+ return
+ }
+ b.rc.UpdateAuthToken(token)
+}
+
+func (rc *perRPCCredential) UpdateAuthToken(token string) {
+ rc.authTokenMu.Lock()
+ rc.authToken = token
+ rc.authTokenMu.Unlock()
+}
diff --git a/vendor/go.etcd.io/etcd/clientv3/doc.go b/vendor/go.etcd.io/etcd/clientv3/doc.go
index 01a3f59..913cd28 100644
--- a/vendor/go.etcd.io/etcd/clientv3/doc.go
+++ b/vendor/go.etcd.io/etcd/clientv3/doc.go
@@ -90,7 +90,7 @@
// // with etcd clientv3 <= v3.3
// if err == context.Canceled {
// // grpc balancer calls 'Get' with an inflight client.Close
-// } else if err == grpc.ErrClientConnClosing {
+// } else if err == grpc.ErrClientConnClosing { // <= gRCP v1.7.x
// // grpc balancer calls 'Get' after client.Close.
// }
// // with etcd clientv3 >= v3.4
diff --git a/vendor/go.etcd.io/etcd/clientv3/op.go b/vendor/go.etcd.io/etcd/clientv3/op.go
index 085dd28..81ae31f 100644
--- a/vendor/go.etcd.io/etcd/clientv3/op.go
+++ b/vendor/go.etcd.io/etcd/clientv3/op.go
@@ -113,13 +113,13 @@
func (op Op) IsDelete() bool { return op.t == tDeleteRange }
// IsSerializable returns true if the serializable field is true.
-func (op Op) IsSerializable() bool { return op.serializable == true }
+func (op Op) IsSerializable() bool { return op.serializable }
// IsKeysOnly returns whether keysOnly is set.
-func (op Op) IsKeysOnly() bool { return op.keysOnly == true }
+func (op Op) IsKeysOnly() bool { return op.keysOnly }
// IsCountOnly returns whether countOnly is set.
-func (op Op) IsCountOnly() bool { return op.countOnly == true }
+func (op Op) IsCountOnly() bool { return op.countOnly }
// MinModRev returns the operation's minimum modify revision.
func (op Op) MinModRev() int64 { return op.minModRev }
diff --git a/vendor/go.etcd.io/etcd/clientv3/options.go b/vendor/go.etcd.io/etcd/clientv3/options.go
index 4660ace..700714c 100644
--- a/vendor/go.etcd.io/etcd/clientv3/options.go
+++ b/vendor/go.etcd.io/etcd/clientv3/options.go
@@ -47,7 +47,7 @@
// client-side streaming retry limit, only applied to requests where server responds with
// a error code clearly indicating it was unable to process the request such as codes.Unavailable.
// If set to 0, retry is disabled.
- defaultStreamMaxRetries = uint(^uint(0)) // max uint
+ defaultStreamMaxRetries = ^uint(0) // max uint
// client-side retry backoff wait between requests.
defaultBackoffWaitBetween = 25 * time.Millisecond
diff --git a/vendor/go.etcd.io/etcd/clientv3/ready_wait.go b/vendor/go.etcd.io/etcd/clientv3/ready_wait.go
deleted file mode 100644
index c6ef585..0000000
--- a/vendor/go.etcd.io/etcd/clientv3/ready_wait.go
+++ /dev/null
@@ -1,30 +0,0 @@
-// Copyright 2017 The etcd Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package clientv3
-
-import "context"
-
-// TODO: remove this when "FailFast=false" is fixed.
-// See https://github.com/grpc/grpc-go/issues/1532.
-func readyWait(rpcCtx, clientCtx context.Context, ready <-chan struct{}) error {
- select {
- case <-ready:
- return nil
- case <-rpcCtx.Done():
- return rpcCtx.Err()
- case <-clientCtx.Done():
- return clientCtx.Err()
- }
-}
diff --git a/vendor/go.etcd.io/etcd/clientv3/retry.go b/vendor/go.etcd.io/etcd/clientv3/retry.go
index 38ad00a..7e855de 100644
--- a/vendor/go.etcd.io/etcd/clientv3/retry.go
+++ b/vendor/go.etcd.io/etcd/clientv3/retry.go
@@ -43,10 +43,6 @@
}
}
-type rpcFunc func(ctx context.Context) error
-type retryRPCFunc func(context.Context, rpcFunc, retryPolicy) error
-type retryStopErrFunc func(error) bool
-
// isSafeRetryImmutableRPC returns "true" when an immutable request is safe for retry.
//
// immutable requests (e.g. Get) should be retried unless it's
@@ -183,6 +179,10 @@
return rcc.cc.MemberUpdate(ctx, in, opts...)
}
+func (rcc *retryClusterClient) MemberPromote(ctx context.Context, in *pb.MemberPromoteRequest, opts ...grpc.CallOption) (resp *pb.MemberPromoteResponse, err error) {
+ return rcc.cc.MemberPromote(ctx, in, opts...)
+}
+
type retryMaintenanceClient struct {
mc pb.MaintenanceClient
}
diff --git a/vendor/go.etcd.io/etcd/clientv3/retry_interceptor.go b/vendor/go.etcd.io/etcd/clientv3/retry_interceptor.go
index e48a003..080490a 100644
--- a/vendor/go.etcd.io/etcd/clientv3/retry_interceptor.go
+++ b/vendor/go.etcd.io/etcd/clientv3/retry_interceptor.go
@@ -28,6 +28,7 @@
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/metadata"
+ "google.golang.org/grpc/status"
)
// unaryClientInterceptor returns a new retrying unary client interceptor.
@@ -79,7 +80,7 @@
zap.String("target", cc.Target()),
zap.Error(gterr),
)
- return lastErr // return the original error for simplicity
+ return gterr // lastErr must be invalid auth token
}
continue
}
@@ -109,7 +110,7 @@
return streamer(ctx, desc, cc, method, grpcOpts...)
}
if desc.ClientStreams {
- return nil, grpc.Errorf(codes.Unimplemented, "clientv3/retry_interceptor: cannot retry on ClientStreams, set Disable()")
+ return nil, status.Errorf(codes.Unimplemented, "clientv3/retry_interceptor: cannot retry on ClientStreams, set Disable()")
}
newStreamer, err := streamer(ctx, desc, cc, method, grpcOpts...)
logger.Warn("retry stream intercept", zap.Error(err))
@@ -296,11 +297,11 @@
func contextErrToGrpcErr(err error) error {
switch err {
case context.DeadlineExceeded:
- return grpc.Errorf(codes.DeadlineExceeded, err.Error())
+ return status.Errorf(codes.DeadlineExceeded, err.Error())
case context.Canceled:
- return grpc.Errorf(codes.Canceled, err.Error())
+ return status.Errorf(codes.Canceled, err.Error())
default:
- return grpc.Errorf(codes.Unknown, err.Error())
+ return status.Errorf(codes.Unknown, err.Error())
}
}
@@ -328,13 +329,6 @@
}}
}
-// withAuthRetry sets enables authentication retries.
-func withAuthRetry(retryAuth bool) retryOption {
- return retryOption{applyFunc: func(o *options) {
- o.retryAuth = retryAuth
- }}
-}
-
// withMax sets the maximum number of retries on this call, or this interceptor.
func withMax(maxRetries uint) retryOption {
return retryOption{applyFunc: func(o *options) {
diff --git a/vendor/go.etcd.io/etcd/clientv3/watch.go b/vendor/go.etcd.io/etcd/clientv3/watch.go
index d50acbc..87d222d 100644
--- a/vendor/go.etcd.io/etcd/clientv3/watch.go
+++ b/vendor/go.etcd.io/etcd/clientv3/watch.go
@@ -384,6 +384,7 @@
w.mu.Lock()
if w.streams == nil {
+ w.mu.Unlock()
return fmt.Errorf("no stream found for context")
}
wgs := w.streams[ctxKey]
diff --git a/vendor/go.etcd.io/etcd/etcdserver/api/v3rpc/rpctypes/error.go b/vendor/go.etcd.io/etcd/etcdserver/api/v3rpc/rpctypes/error.go
index 9e45cea..e6a2814 100644
--- a/vendor/go.etcd.io/etcd/etcdserver/api/v3rpc/rpctypes/error.go
+++ b/vendor/go.etcd.io/etcd/etcdserver/api/v3rpc/rpctypes/error.go
@@ -40,6 +40,9 @@
ErrGRPCMemberNotEnoughStarted = status.New(codes.FailedPrecondition, "etcdserver: re-configuration failed due to not enough started members").Err()
ErrGRPCMemberBadURLs = status.New(codes.InvalidArgument, "etcdserver: given member URLs are invalid").Err()
ErrGRPCMemberNotFound = status.New(codes.NotFound, "etcdserver: member not found").Err()
+ ErrGRPCMemberNotLearner = status.New(codes.FailedPrecondition, "etcdserver: can only promote a learner member").Err()
+ ErrGRPCLearnerNotReady = status.New(codes.FailedPrecondition, "etcdserver: can only promote a learner member which is in sync with leader").Err()
+ ErrGRPCTooManyLearners = status.New(codes.FailedPrecondition, "etcdserver: too many learner members in cluster").Err()
ErrGRPCRequestTooLarge = status.New(codes.InvalidArgument, "etcdserver: request is too large").Err()
ErrGRPCRequestTooManyRequests = status.New(codes.ResourceExhausted, "etcdserver: too many requests").Err()
@@ -51,6 +54,7 @@
ErrGRPCUserNotFound = status.New(codes.FailedPrecondition, "etcdserver: user name not found").Err()
ErrGRPCRoleAlreadyExist = status.New(codes.FailedPrecondition, "etcdserver: role name already exists").Err()
ErrGRPCRoleNotFound = status.New(codes.FailedPrecondition, "etcdserver: role name not found").Err()
+ ErrGRPCRoleEmpty = status.New(codes.InvalidArgument, "etcdserver: role name is empty").Err()
ErrGRPCAuthFailed = status.New(codes.InvalidArgument, "etcdserver: authentication failed, invalid user ID or password").Err()
ErrGRPCPermissionDenied = status.New(codes.PermissionDenied, "etcdserver: permission denied").Err()
ErrGRPCRoleNotGranted = status.New(codes.FailedPrecondition, "etcdserver: role is not granted to the user").Err()
@@ -69,6 +73,8 @@
ErrGRPCTimeoutDueToConnectionLost = status.New(codes.Unavailable, "etcdserver: request timed out, possibly due to connection lost").Err()
ErrGRPCUnhealthy = status.New(codes.Unavailable, "etcdserver: unhealthy cluster").Err()
ErrGRPCCorrupt = status.New(codes.DataLoss, "etcdserver: corrupt cluster").Err()
+ ErrGPRCNotSupportedForLearner = status.New(codes.Unavailable, "etcdserver: rpc not supported for learner").Err()
+ ErrGRPCBadLeaderTransferee = status.New(codes.FailedPrecondition, "etcdserver: bad leader transferee").Err()
errStringToError = map[string]error{
ErrorDesc(ErrGRPCEmptyKey): ErrGRPCEmptyKey,
@@ -91,6 +97,9 @@
ErrorDesc(ErrGRPCMemberNotEnoughStarted): ErrGRPCMemberNotEnoughStarted,
ErrorDesc(ErrGRPCMemberBadURLs): ErrGRPCMemberBadURLs,
ErrorDesc(ErrGRPCMemberNotFound): ErrGRPCMemberNotFound,
+ ErrorDesc(ErrGRPCMemberNotLearner): ErrGRPCMemberNotLearner,
+ ErrorDesc(ErrGRPCLearnerNotReady): ErrGRPCLearnerNotReady,
+ ErrorDesc(ErrGRPCTooManyLearners): ErrGRPCTooManyLearners,
ErrorDesc(ErrGRPCRequestTooLarge): ErrGRPCRequestTooLarge,
ErrorDesc(ErrGRPCRequestTooManyRequests): ErrGRPCRequestTooManyRequests,
@@ -102,6 +111,7 @@
ErrorDesc(ErrGRPCUserNotFound): ErrGRPCUserNotFound,
ErrorDesc(ErrGRPCRoleAlreadyExist): ErrGRPCRoleAlreadyExist,
ErrorDesc(ErrGRPCRoleNotFound): ErrGRPCRoleNotFound,
+ ErrorDesc(ErrGRPCRoleEmpty): ErrGRPCRoleEmpty,
ErrorDesc(ErrGRPCAuthFailed): ErrGRPCAuthFailed,
ErrorDesc(ErrGRPCPermissionDenied): ErrGRPCPermissionDenied,
ErrorDesc(ErrGRPCRoleNotGranted): ErrGRPCRoleNotGranted,
@@ -120,6 +130,8 @@
ErrorDesc(ErrGRPCTimeoutDueToConnectionLost): ErrGRPCTimeoutDueToConnectionLost,
ErrorDesc(ErrGRPCUnhealthy): ErrGRPCUnhealthy,
ErrorDesc(ErrGRPCCorrupt): ErrGRPCCorrupt,
+ ErrorDesc(ErrGPRCNotSupportedForLearner): ErrGPRCNotSupportedForLearner,
+ ErrorDesc(ErrGRPCBadLeaderTransferee): ErrGRPCBadLeaderTransferee,
}
)
@@ -144,6 +156,9 @@
ErrMemberNotEnoughStarted = Error(ErrGRPCMemberNotEnoughStarted)
ErrMemberBadURLs = Error(ErrGRPCMemberBadURLs)
ErrMemberNotFound = Error(ErrGRPCMemberNotFound)
+ ErrMemberNotLearner = Error(ErrGRPCMemberNotLearner)
+ ErrMemberLearnerNotReady = Error(ErrGRPCLearnerNotReady)
+ ErrTooManyLearners = Error(ErrGRPCTooManyLearners)
ErrRequestTooLarge = Error(ErrGRPCRequestTooLarge)
ErrTooManyRequests = Error(ErrGRPCRequestTooManyRequests)
@@ -155,6 +170,7 @@
ErrUserNotFound = Error(ErrGRPCUserNotFound)
ErrRoleAlreadyExist = Error(ErrGRPCRoleAlreadyExist)
ErrRoleNotFound = Error(ErrGRPCRoleNotFound)
+ ErrRoleEmpty = Error(ErrGRPCRoleEmpty)
ErrAuthFailed = Error(ErrGRPCAuthFailed)
ErrPermissionDenied = Error(ErrGRPCPermissionDenied)
ErrRoleNotGranted = Error(ErrGRPCRoleNotGranted)
@@ -173,6 +189,7 @@
ErrTimeoutDueToConnectionLost = Error(ErrGRPCTimeoutDueToConnectionLost)
ErrUnhealthy = Error(ErrGRPCUnhealthy)
ErrCorrupt = Error(ErrGRPCCorrupt)
+ ErrBadLeaderTransferee = Error(ErrGRPCBadLeaderTransferee)
)
// EtcdError defines gRPC server errors.
diff --git a/vendor/go.etcd.io/etcd/etcdserver/etcdserverpb/etcdserver.pb.go b/vendor/go.etcd.io/etcd/etcdserver/etcdserverpb/etcdserver.pb.go
index f5134b9..9e9b42c 100644
--- a/vendor/go.etcd.io/etcd/etcdserver/etcdserverpb/etcdserver.pb.go
+++ b/vendor/go.etcd.io/etcd/etcdserver/etcdserverpb/etcdserver.pb.go
@@ -64,6 +64,8 @@
MemberUpdateResponse
MemberListRequest
MemberListResponse
+ MemberPromoteRequest
+ MemberPromoteResponse
DefragmentRequest
DefragmentResponse
MoveLeaderRequest
diff --git a/vendor/go.etcd.io/etcd/etcdserver/etcdserverpb/rpc.pb.go b/vendor/go.etcd.io/etcd/etcdserver/etcdserverpb/rpc.pb.go
index 3e15079..199ee62 100644
--- a/vendor/go.etcd.io/etcd/etcdserver/etcdserverpb/rpc.pb.go
+++ b/vendor/go.etcd.io/etcd/etcdserver/etcdserverpb/rpc.pb.go
@@ -211,7 +211,7 @@
return proto.EnumName(AlarmRequest_AlarmAction_name, int32(x))
}
func (AlarmRequest_AlarmAction) EnumDescriptor() ([]byte, []int) {
- return fileDescriptorRpc, []int{52, 0}
+ return fileDescriptorRpc, []int{54, 0}
}
type ResponseHeader struct {
@@ -2186,6 +2186,8 @@
PeerURLs []string `protobuf:"bytes,3,rep,name=peerURLs" json:"peerURLs,omitempty"`
// clientURLs is the list of URLs the member exposes to clients for communication. If the member is not started, clientURLs will be empty.
ClientURLs []string `protobuf:"bytes,4,rep,name=clientURLs" json:"clientURLs,omitempty"`
+ // isLearner indicates if the member is raft learner.
+ IsLearner bool `protobuf:"varint,5,opt,name=isLearner,proto3" json:"isLearner,omitempty"`
}
func (m *Member) Reset() { *m = Member{} }
@@ -2221,9 +2223,18 @@
return nil
}
+func (m *Member) GetIsLearner() bool {
+ if m != nil {
+ return m.IsLearner
+ }
+ return false
+}
+
type MemberAddRequest struct {
// peerURLs is the list of URLs the added member will use to communicate with the cluster.
PeerURLs []string `protobuf:"bytes,1,rep,name=peerURLs" json:"peerURLs,omitempty"`
+ // isLearner indicates if the added member is raft learner.
+ IsLearner bool `protobuf:"varint,2,opt,name=isLearner,proto3" json:"isLearner,omitempty"`
}
func (m *MemberAddRequest) Reset() { *m = MemberAddRequest{} }
@@ -2238,6 +2249,13 @@
return nil
}
+func (m *MemberAddRequest) GetIsLearner() bool {
+ if m != nil {
+ return m.IsLearner
+ }
+ return false
+}
+
type MemberAddResponse struct {
Header *ResponseHeader `protobuf:"bytes,1,opt,name=header" json:"header,omitempty"`
// member is the member information for the added member.
@@ -2398,13 +2416,55 @@
return nil
}
+type MemberPromoteRequest struct {
+ // ID is the member ID of the member to promote.
+ ID uint64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"`
+}
+
+func (m *MemberPromoteRequest) Reset() { *m = MemberPromoteRequest{} }
+func (m *MemberPromoteRequest) String() string { return proto.CompactTextString(m) }
+func (*MemberPromoteRequest) ProtoMessage() {}
+func (*MemberPromoteRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{48} }
+
+func (m *MemberPromoteRequest) GetID() uint64 {
+ if m != nil {
+ return m.ID
+ }
+ return 0
+}
+
+type MemberPromoteResponse struct {
+ Header *ResponseHeader `protobuf:"bytes,1,opt,name=header" json:"header,omitempty"`
+ // members is a list of all members after promoting the member.
+ Members []*Member `protobuf:"bytes,2,rep,name=members" json:"members,omitempty"`
+}
+
+func (m *MemberPromoteResponse) Reset() { *m = MemberPromoteResponse{} }
+func (m *MemberPromoteResponse) String() string { return proto.CompactTextString(m) }
+func (*MemberPromoteResponse) ProtoMessage() {}
+func (*MemberPromoteResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{49} }
+
+func (m *MemberPromoteResponse) GetHeader() *ResponseHeader {
+ if m != nil {
+ return m.Header
+ }
+ return nil
+}
+
+func (m *MemberPromoteResponse) GetMembers() []*Member {
+ if m != nil {
+ return m.Members
+ }
+ return nil
+}
+
type DefragmentRequest struct {
}
func (m *DefragmentRequest) Reset() { *m = DefragmentRequest{} }
func (m *DefragmentRequest) String() string { return proto.CompactTextString(m) }
func (*DefragmentRequest) ProtoMessage() {}
-func (*DefragmentRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{48} }
+func (*DefragmentRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{50} }
type DefragmentResponse struct {
Header *ResponseHeader `protobuf:"bytes,1,opt,name=header" json:"header,omitempty"`
@@ -2413,7 +2473,7 @@
func (m *DefragmentResponse) Reset() { *m = DefragmentResponse{} }
func (m *DefragmentResponse) String() string { return proto.CompactTextString(m) }
func (*DefragmentResponse) ProtoMessage() {}
-func (*DefragmentResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{49} }
+func (*DefragmentResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{51} }
func (m *DefragmentResponse) GetHeader() *ResponseHeader {
if m != nil {
@@ -2430,7 +2490,7 @@
func (m *MoveLeaderRequest) Reset() { *m = MoveLeaderRequest{} }
func (m *MoveLeaderRequest) String() string { return proto.CompactTextString(m) }
func (*MoveLeaderRequest) ProtoMessage() {}
-func (*MoveLeaderRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{50} }
+func (*MoveLeaderRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{52} }
func (m *MoveLeaderRequest) GetTargetID() uint64 {
if m != nil {
@@ -2446,7 +2506,7 @@
func (m *MoveLeaderResponse) Reset() { *m = MoveLeaderResponse{} }
func (m *MoveLeaderResponse) String() string { return proto.CompactTextString(m) }
func (*MoveLeaderResponse) ProtoMessage() {}
-func (*MoveLeaderResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{51} }
+func (*MoveLeaderResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{53} }
func (m *MoveLeaderResponse) GetHeader() *ResponseHeader {
if m != nil {
@@ -2470,7 +2530,7 @@
func (m *AlarmRequest) Reset() { *m = AlarmRequest{} }
func (m *AlarmRequest) String() string { return proto.CompactTextString(m) }
func (*AlarmRequest) ProtoMessage() {}
-func (*AlarmRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{52} }
+func (*AlarmRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{54} }
func (m *AlarmRequest) GetAction() AlarmRequest_AlarmAction {
if m != nil {
@@ -2503,7 +2563,7 @@
func (m *AlarmMember) Reset() { *m = AlarmMember{} }
func (m *AlarmMember) String() string { return proto.CompactTextString(m) }
func (*AlarmMember) ProtoMessage() {}
-func (*AlarmMember) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{53} }
+func (*AlarmMember) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{55} }
func (m *AlarmMember) GetMemberID() uint64 {
if m != nil {
@@ -2528,7 +2588,7 @@
func (m *AlarmResponse) Reset() { *m = AlarmResponse{} }
func (m *AlarmResponse) String() string { return proto.CompactTextString(m) }
func (*AlarmResponse) ProtoMessage() {}
-func (*AlarmResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{54} }
+func (*AlarmResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{56} }
func (m *AlarmResponse) GetHeader() *ResponseHeader {
if m != nil {
@@ -2550,7 +2610,7 @@
func (m *StatusRequest) Reset() { *m = StatusRequest{} }
func (m *StatusRequest) String() string { return proto.CompactTextString(m) }
func (*StatusRequest) ProtoMessage() {}
-func (*StatusRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{55} }
+func (*StatusRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{57} }
type StatusResponse struct {
Header *ResponseHeader `protobuf:"bytes,1,opt,name=header" json:"header,omitempty"`
@@ -2570,12 +2630,14 @@
Errors []string `protobuf:"bytes,8,rep,name=errors" json:"errors,omitempty"`
// dbSizeInUse is the size of the backend database logically in use, in bytes, of the responding member.
DbSizeInUse int64 `protobuf:"varint,9,opt,name=dbSizeInUse,proto3" json:"dbSizeInUse,omitempty"`
+ // isLearner indicates if the member is raft learner.
+ IsLearner bool `protobuf:"varint,10,opt,name=isLearner,proto3" json:"isLearner,omitempty"`
}
func (m *StatusResponse) Reset() { *m = StatusResponse{} }
func (m *StatusResponse) String() string { return proto.CompactTextString(m) }
func (*StatusResponse) ProtoMessage() {}
-func (*StatusResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{56} }
+func (*StatusResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{58} }
func (m *StatusResponse) GetHeader() *ResponseHeader {
if m != nil {
@@ -2640,13 +2702,20 @@
return 0
}
+func (m *StatusResponse) GetIsLearner() bool {
+ if m != nil {
+ return m.IsLearner
+ }
+ return false
+}
+
type AuthEnableRequest struct {
}
func (m *AuthEnableRequest) Reset() { *m = AuthEnableRequest{} }
func (m *AuthEnableRequest) String() string { return proto.CompactTextString(m) }
func (*AuthEnableRequest) ProtoMessage() {}
-func (*AuthEnableRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{57} }
+func (*AuthEnableRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{59} }
type AuthDisableRequest struct {
}
@@ -2654,7 +2723,7 @@
func (m *AuthDisableRequest) Reset() { *m = AuthDisableRequest{} }
func (m *AuthDisableRequest) String() string { return proto.CompactTextString(m) }
func (*AuthDisableRequest) ProtoMessage() {}
-func (*AuthDisableRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{58} }
+func (*AuthDisableRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{60} }
type AuthenticateRequest struct {
Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
@@ -2664,7 +2733,7 @@
func (m *AuthenticateRequest) Reset() { *m = AuthenticateRequest{} }
func (m *AuthenticateRequest) String() string { return proto.CompactTextString(m) }
func (*AuthenticateRequest) ProtoMessage() {}
-func (*AuthenticateRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{59} }
+func (*AuthenticateRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{61} }
func (m *AuthenticateRequest) GetName() string {
if m != nil {
@@ -2681,14 +2750,15 @@
}
type AuthUserAddRequest struct {
- Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
- Password string `protobuf:"bytes,2,opt,name=password,proto3" json:"password,omitempty"`
+ Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
+ Password string `protobuf:"bytes,2,opt,name=password,proto3" json:"password,omitempty"`
+ Options *authpb.UserAddOptions `protobuf:"bytes,3,opt,name=options" json:"options,omitempty"`
}
func (m *AuthUserAddRequest) Reset() { *m = AuthUserAddRequest{} }
func (m *AuthUserAddRequest) String() string { return proto.CompactTextString(m) }
func (*AuthUserAddRequest) ProtoMessage() {}
-func (*AuthUserAddRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{60} }
+func (*AuthUserAddRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{62} }
func (m *AuthUserAddRequest) GetName() string {
if m != nil {
@@ -2704,6 +2774,13 @@
return ""
}
+func (m *AuthUserAddRequest) GetOptions() *authpb.UserAddOptions {
+ if m != nil {
+ return m.Options
+ }
+ return nil
+}
+
type AuthUserGetRequest struct {
Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
}
@@ -2711,7 +2788,7 @@
func (m *AuthUserGetRequest) Reset() { *m = AuthUserGetRequest{} }
func (m *AuthUserGetRequest) String() string { return proto.CompactTextString(m) }
func (*AuthUserGetRequest) ProtoMessage() {}
-func (*AuthUserGetRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{61} }
+func (*AuthUserGetRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{63} }
func (m *AuthUserGetRequest) GetName() string {
if m != nil {
@@ -2728,7 +2805,7 @@
func (m *AuthUserDeleteRequest) Reset() { *m = AuthUserDeleteRequest{} }
func (m *AuthUserDeleteRequest) String() string { return proto.CompactTextString(m) }
func (*AuthUserDeleteRequest) ProtoMessage() {}
-func (*AuthUserDeleteRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{62} }
+func (*AuthUserDeleteRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{64} }
func (m *AuthUserDeleteRequest) GetName() string {
if m != nil {
@@ -2748,7 +2825,7 @@
func (m *AuthUserChangePasswordRequest) String() string { return proto.CompactTextString(m) }
func (*AuthUserChangePasswordRequest) ProtoMessage() {}
func (*AuthUserChangePasswordRequest) Descriptor() ([]byte, []int) {
- return fileDescriptorRpc, []int{63}
+ return fileDescriptorRpc, []int{65}
}
func (m *AuthUserChangePasswordRequest) GetName() string {
@@ -2775,7 +2852,7 @@
func (m *AuthUserGrantRoleRequest) Reset() { *m = AuthUserGrantRoleRequest{} }
func (m *AuthUserGrantRoleRequest) String() string { return proto.CompactTextString(m) }
func (*AuthUserGrantRoleRequest) ProtoMessage() {}
-func (*AuthUserGrantRoleRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{64} }
+func (*AuthUserGrantRoleRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{66} }
func (m *AuthUserGrantRoleRequest) GetUser() string {
if m != nil {
@@ -2799,7 +2876,7 @@
func (m *AuthUserRevokeRoleRequest) Reset() { *m = AuthUserRevokeRoleRequest{} }
func (m *AuthUserRevokeRoleRequest) String() string { return proto.CompactTextString(m) }
func (*AuthUserRevokeRoleRequest) ProtoMessage() {}
-func (*AuthUserRevokeRoleRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{65} }
+func (*AuthUserRevokeRoleRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{67} }
func (m *AuthUserRevokeRoleRequest) GetName() string {
if m != nil {
@@ -2823,7 +2900,7 @@
func (m *AuthRoleAddRequest) Reset() { *m = AuthRoleAddRequest{} }
func (m *AuthRoleAddRequest) String() string { return proto.CompactTextString(m) }
func (*AuthRoleAddRequest) ProtoMessage() {}
-func (*AuthRoleAddRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{66} }
+func (*AuthRoleAddRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{68} }
func (m *AuthRoleAddRequest) GetName() string {
if m != nil {
@@ -2839,7 +2916,7 @@
func (m *AuthRoleGetRequest) Reset() { *m = AuthRoleGetRequest{} }
func (m *AuthRoleGetRequest) String() string { return proto.CompactTextString(m) }
func (*AuthRoleGetRequest) ProtoMessage() {}
-func (*AuthRoleGetRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{67} }
+func (*AuthRoleGetRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{69} }
func (m *AuthRoleGetRequest) GetRole() string {
if m != nil {
@@ -2854,7 +2931,7 @@
func (m *AuthUserListRequest) Reset() { *m = AuthUserListRequest{} }
func (m *AuthUserListRequest) String() string { return proto.CompactTextString(m) }
func (*AuthUserListRequest) ProtoMessage() {}
-func (*AuthUserListRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{68} }
+func (*AuthUserListRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{70} }
type AuthRoleListRequest struct {
}
@@ -2862,7 +2939,7 @@
func (m *AuthRoleListRequest) Reset() { *m = AuthRoleListRequest{} }
func (m *AuthRoleListRequest) String() string { return proto.CompactTextString(m) }
func (*AuthRoleListRequest) ProtoMessage() {}
-func (*AuthRoleListRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{69} }
+func (*AuthRoleListRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{71} }
type AuthRoleDeleteRequest struct {
Role string `protobuf:"bytes,1,opt,name=role,proto3" json:"role,omitempty"`
@@ -2871,7 +2948,7 @@
func (m *AuthRoleDeleteRequest) Reset() { *m = AuthRoleDeleteRequest{} }
func (m *AuthRoleDeleteRequest) String() string { return proto.CompactTextString(m) }
func (*AuthRoleDeleteRequest) ProtoMessage() {}
-func (*AuthRoleDeleteRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{70} }
+func (*AuthRoleDeleteRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{72} }
func (m *AuthRoleDeleteRequest) GetRole() string {
if m != nil {
@@ -2891,7 +2968,7 @@
func (m *AuthRoleGrantPermissionRequest) String() string { return proto.CompactTextString(m) }
func (*AuthRoleGrantPermissionRequest) ProtoMessage() {}
func (*AuthRoleGrantPermissionRequest) Descriptor() ([]byte, []int) {
- return fileDescriptorRpc, []int{71}
+ return fileDescriptorRpc, []int{73}
}
func (m *AuthRoleGrantPermissionRequest) GetName() string {
@@ -2918,7 +2995,7 @@
func (m *AuthRoleRevokePermissionRequest) String() string { return proto.CompactTextString(m) }
func (*AuthRoleRevokePermissionRequest) ProtoMessage() {}
func (*AuthRoleRevokePermissionRequest) Descriptor() ([]byte, []int) {
- return fileDescriptorRpc, []int{72}
+ return fileDescriptorRpc, []int{74}
}
func (m *AuthRoleRevokePermissionRequest) GetRole() string {
@@ -2949,7 +3026,7 @@
func (m *AuthEnableResponse) Reset() { *m = AuthEnableResponse{} }
func (m *AuthEnableResponse) String() string { return proto.CompactTextString(m) }
func (*AuthEnableResponse) ProtoMessage() {}
-func (*AuthEnableResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{73} }
+func (*AuthEnableResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{75} }
func (m *AuthEnableResponse) GetHeader() *ResponseHeader {
if m != nil {
@@ -2965,7 +3042,7 @@
func (m *AuthDisableResponse) Reset() { *m = AuthDisableResponse{} }
func (m *AuthDisableResponse) String() string { return proto.CompactTextString(m) }
func (*AuthDisableResponse) ProtoMessage() {}
-func (*AuthDisableResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{74} }
+func (*AuthDisableResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{76} }
func (m *AuthDisableResponse) GetHeader() *ResponseHeader {
if m != nil {
@@ -2983,7 +3060,7 @@
func (m *AuthenticateResponse) Reset() { *m = AuthenticateResponse{} }
func (m *AuthenticateResponse) String() string { return proto.CompactTextString(m) }
func (*AuthenticateResponse) ProtoMessage() {}
-func (*AuthenticateResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{75} }
+func (*AuthenticateResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{77} }
func (m *AuthenticateResponse) GetHeader() *ResponseHeader {
if m != nil {
@@ -3006,7 +3083,7 @@
func (m *AuthUserAddResponse) Reset() { *m = AuthUserAddResponse{} }
func (m *AuthUserAddResponse) String() string { return proto.CompactTextString(m) }
func (*AuthUserAddResponse) ProtoMessage() {}
-func (*AuthUserAddResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{76} }
+func (*AuthUserAddResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{78} }
func (m *AuthUserAddResponse) GetHeader() *ResponseHeader {
if m != nil {
@@ -3023,7 +3100,7 @@
func (m *AuthUserGetResponse) Reset() { *m = AuthUserGetResponse{} }
func (m *AuthUserGetResponse) String() string { return proto.CompactTextString(m) }
func (*AuthUserGetResponse) ProtoMessage() {}
-func (*AuthUserGetResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{77} }
+func (*AuthUserGetResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{79} }
func (m *AuthUserGetResponse) GetHeader() *ResponseHeader {
if m != nil {
@@ -3046,7 +3123,7 @@
func (m *AuthUserDeleteResponse) Reset() { *m = AuthUserDeleteResponse{} }
func (m *AuthUserDeleteResponse) String() string { return proto.CompactTextString(m) }
func (*AuthUserDeleteResponse) ProtoMessage() {}
-func (*AuthUserDeleteResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{78} }
+func (*AuthUserDeleteResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{80} }
func (m *AuthUserDeleteResponse) GetHeader() *ResponseHeader {
if m != nil {
@@ -3063,7 +3140,7 @@
func (m *AuthUserChangePasswordResponse) String() string { return proto.CompactTextString(m) }
func (*AuthUserChangePasswordResponse) ProtoMessage() {}
func (*AuthUserChangePasswordResponse) Descriptor() ([]byte, []int) {
- return fileDescriptorRpc, []int{79}
+ return fileDescriptorRpc, []int{81}
}
func (m *AuthUserChangePasswordResponse) GetHeader() *ResponseHeader {
@@ -3080,7 +3157,7 @@
func (m *AuthUserGrantRoleResponse) Reset() { *m = AuthUserGrantRoleResponse{} }
func (m *AuthUserGrantRoleResponse) String() string { return proto.CompactTextString(m) }
func (*AuthUserGrantRoleResponse) ProtoMessage() {}
-func (*AuthUserGrantRoleResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{80} }
+func (*AuthUserGrantRoleResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{82} }
func (m *AuthUserGrantRoleResponse) GetHeader() *ResponseHeader {
if m != nil {
@@ -3096,7 +3173,7 @@
func (m *AuthUserRevokeRoleResponse) Reset() { *m = AuthUserRevokeRoleResponse{} }
func (m *AuthUserRevokeRoleResponse) String() string { return proto.CompactTextString(m) }
func (*AuthUserRevokeRoleResponse) ProtoMessage() {}
-func (*AuthUserRevokeRoleResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{81} }
+func (*AuthUserRevokeRoleResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{83} }
func (m *AuthUserRevokeRoleResponse) GetHeader() *ResponseHeader {
if m != nil {
@@ -3112,7 +3189,7 @@
func (m *AuthRoleAddResponse) Reset() { *m = AuthRoleAddResponse{} }
func (m *AuthRoleAddResponse) String() string { return proto.CompactTextString(m) }
func (*AuthRoleAddResponse) ProtoMessage() {}
-func (*AuthRoleAddResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{82} }
+func (*AuthRoleAddResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{84} }
func (m *AuthRoleAddResponse) GetHeader() *ResponseHeader {
if m != nil {
@@ -3129,7 +3206,7 @@
func (m *AuthRoleGetResponse) Reset() { *m = AuthRoleGetResponse{} }
func (m *AuthRoleGetResponse) String() string { return proto.CompactTextString(m) }
func (*AuthRoleGetResponse) ProtoMessage() {}
-func (*AuthRoleGetResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{83} }
+func (*AuthRoleGetResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{85} }
func (m *AuthRoleGetResponse) GetHeader() *ResponseHeader {
if m != nil {
@@ -3153,7 +3230,7 @@
func (m *AuthRoleListResponse) Reset() { *m = AuthRoleListResponse{} }
func (m *AuthRoleListResponse) String() string { return proto.CompactTextString(m) }
func (*AuthRoleListResponse) ProtoMessage() {}
-func (*AuthRoleListResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{84} }
+func (*AuthRoleListResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{86} }
func (m *AuthRoleListResponse) GetHeader() *ResponseHeader {
if m != nil {
@@ -3177,7 +3254,7 @@
func (m *AuthUserListResponse) Reset() { *m = AuthUserListResponse{} }
func (m *AuthUserListResponse) String() string { return proto.CompactTextString(m) }
func (*AuthUserListResponse) ProtoMessage() {}
-func (*AuthUserListResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{85} }
+func (*AuthUserListResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{87} }
func (m *AuthUserListResponse) GetHeader() *ResponseHeader {
if m != nil {
@@ -3200,7 +3277,7 @@
func (m *AuthRoleDeleteResponse) Reset() { *m = AuthRoleDeleteResponse{} }
func (m *AuthRoleDeleteResponse) String() string { return proto.CompactTextString(m) }
func (*AuthRoleDeleteResponse) ProtoMessage() {}
-func (*AuthRoleDeleteResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{86} }
+func (*AuthRoleDeleteResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{88} }
func (m *AuthRoleDeleteResponse) GetHeader() *ResponseHeader {
if m != nil {
@@ -3217,7 +3294,7 @@
func (m *AuthRoleGrantPermissionResponse) String() string { return proto.CompactTextString(m) }
func (*AuthRoleGrantPermissionResponse) ProtoMessage() {}
func (*AuthRoleGrantPermissionResponse) Descriptor() ([]byte, []int) {
- return fileDescriptorRpc, []int{87}
+ return fileDescriptorRpc, []int{89}
}
func (m *AuthRoleGrantPermissionResponse) GetHeader() *ResponseHeader {
@@ -3235,7 +3312,7 @@
func (m *AuthRoleRevokePermissionResponse) String() string { return proto.CompactTextString(m) }
func (*AuthRoleRevokePermissionResponse) ProtoMessage() {}
func (*AuthRoleRevokePermissionResponse) Descriptor() ([]byte, []int) {
- return fileDescriptorRpc, []int{88}
+ return fileDescriptorRpc, []int{90}
}
func (m *AuthRoleRevokePermissionResponse) GetHeader() *ResponseHeader {
@@ -3294,6 +3371,8 @@
proto.RegisterType((*MemberUpdateResponse)(nil), "etcdserverpb.MemberUpdateResponse")
proto.RegisterType((*MemberListRequest)(nil), "etcdserverpb.MemberListRequest")
proto.RegisterType((*MemberListResponse)(nil), "etcdserverpb.MemberListResponse")
+ proto.RegisterType((*MemberPromoteRequest)(nil), "etcdserverpb.MemberPromoteRequest")
+ proto.RegisterType((*MemberPromoteResponse)(nil), "etcdserverpb.MemberPromoteResponse")
proto.RegisterType((*DefragmentRequest)(nil), "etcdserverpb.DefragmentRequest")
proto.RegisterType((*DefragmentResponse)(nil), "etcdserverpb.DefragmentResponse")
proto.RegisterType((*MoveLeaderRequest)(nil), "etcdserverpb.MoveLeaderRequest")
@@ -3938,6 +4017,8 @@
MemberUpdate(ctx context.Context, in *MemberUpdateRequest, opts ...grpc.CallOption) (*MemberUpdateResponse, error)
// MemberList lists all the members in the cluster.
MemberList(ctx context.Context, in *MemberListRequest, opts ...grpc.CallOption) (*MemberListResponse, error)
+ // MemberPromote promotes a member from raft learner (non-voting) to raft voting member.
+ MemberPromote(ctx context.Context, in *MemberPromoteRequest, opts ...grpc.CallOption) (*MemberPromoteResponse, error)
}
type clusterClient struct {
@@ -3984,6 +4065,15 @@
return out, nil
}
+func (c *clusterClient) MemberPromote(ctx context.Context, in *MemberPromoteRequest, opts ...grpc.CallOption) (*MemberPromoteResponse, error) {
+ out := new(MemberPromoteResponse)
+ err := grpc.Invoke(ctx, "/etcdserverpb.Cluster/MemberPromote", in, out, c.cc, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
// Server API for Cluster service
type ClusterServer interface {
@@ -3995,6 +4085,8 @@
MemberUpdate(context.Context, *MemberUpdateRequest) (*MemberUpdateResponse, error)
// MemberList lists all the members in the cluster.
MemberList(context.Context, *MemberListRequest) (*MemberListResponse, error)
+ // MemberPromote promotes a member from raft learner (non-voting) to raft voting member.
+ MemberPromote(context.Context, *MemberPromoteRequest) (*MemberPromoteResponse, error)
}
func RegisterClusterServer(s *grpc.Server, srv ClusterServer) {
@@ -4073,6 +4165,24 @@
return interceptor(ctx, in, info, handler)
}
+func _Cluster_MemberPromote_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+ in := new(MemberPromoteRequest)
+ if err := dec(in); err != nil {
+ return nil, err
+ }
+ if interceptor == nil {
+ return srv.(ClusterServer).MemberPromote(ctx, in)
+ }
+ info := &grpc.UnaryServerInfo{
+ Server: srv,
+ FullMethod: "/etcdserverpb.Cluster/MemberPromote",
+ }
+ handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+ return srv.(ClusterServer).MemberPromote(ctx, req.(*MemberPromoteRequest))
+ }
+ return interceptor(ctx, in, info, handler)
+}
+
var _Cluster_serviceDesc = grpc.ServiceDesc{
ServiceName: "etcdserverpb.Cluster",
HandlerType: (*ClusterServer)(nil),
@@ -4093,6 +4203,10 @@
MethodName: "MemberList",
Handler: _Cluster_MemberList_Handler,
},
+ {
+ MethodName: "MemberPromote",
+ Handler: _Cluster_MemberPromote_Handler,
+ },
},
Streams: []grpc.StreamDesc{},
Metadata: "rpc.proto",
@@ -4423,7 +4537,7 @@
AuthDisable(ctx context.Context, in *AuthDisableRequest, opts ...grpc.CallOption) (*AuthDisableResponse, error)
// Authenticate processes an authenticate request.
Authenticate(ctx context.Context, in *AuthenticateRequest, opts ...grpc.CallOption) (*AuthenticateResponse, error)
- // UserAdd adds a new user.
+ // UserAdd adds a new user. User name cannot be empty.
UserAdd(ctx context.Context, in *AuthUserAddRequest, opts ...grpc.CallOption) (*AuthUserAddResponse, error)
// UserGet gets detailed user information.
UserGet(ctx context.Context, in *AuthUserGetRequest, opts ...grpc.CallOption) (*AuthUserGetResponse, error)
@@ -4437,7 +4551,7 @@
UserGrantRole(ctx context.Context, in *AuthUserGrantRoleRequest, opts ...grpc.CallOption) (*AuthUserGrantRoleResponse, error)
// UserRevokeRole revokes a role of specified user.
UserRevokeRole(ctx context.Context, in *AuthUserRevokeRoleRequest, opts ...grpc.CallOption) (*AuthUserRevokeRoleResponse, error)
- // RoleAdd adds a new role.
+ // RoleAdd adds a new role. Role name cannot be empty.
RoleAdd(ctx context.Context, in *AuthRoleAddRequest, opts ...grpc.CallOption) (*AuthRoleAddResponse, error)
// RoleGet gets detailed role information.
RoleGet(ctx context.Context, in *AuthRoleGetRequest, opts ...grpc.CallOption) (*AuthRoleGetResponse, error)
@@ -4612,7 +4726,7 @@
AuthDisable(context.Context, *AuthDisableRequest) (*AuthDisableResponse, error)
// Authenticate processes an authenticate request.
Authenticate(context.Context, *AuthenticateRequest) (*AuthenticateResponse, error)
- // UserAdd adds a new user.
+ // UserAdd adds a new user. User name cannot be empty.
UserAdd(context.Context, *AuthUserAddRequest) (*AuthUserAddResponse, error)
// UserGet gets detailed user information.
UserGet(context.Context, *AuthUserGetRequest) (*AuthUserGetResponse, error)
@@ -4626,7 +4740,7 @@
UserGrantRole(context.Context, *AuthUserGrantRoleRequest) (*AuthUserGrantRoleResponse, error)
// UserRevokeRole revokes a role of specified user.
UserRevokeRole(context.Context, *AuthUserRevokeRoleRequest) (*AuthUserRevokeRoleResponse, error)
- // RoleAdd adds a new role.
+ // RoleAdd adds a new role. Role name cannot be empty.
RoleAdd(context.Context, *AuthRoleAddRequest) (*AuthRoleAddResponse, error)
// RoleGet gets detailed role information.
RoleGet(context.Context, *AuthRoleGetRequest) (*AuthRoleGetResponse, error)
@@ -6741,6 +6855,16 @@
i += copy(dAtA[i:], s)
}
}
+ if m.IsLearner {
+ dAtA[i] = 0x28
+ i++
+ if m.IsLearner {
+ dAtA[i] = 1
+ } else {
+ dAtA[i] = 0
+ }
+ i++
+ }
return i, nil
}
@@ -6774,6 +6898,16 @@
i += copy(dAtA[i:], s)
}
}
+ if m.IsLearner {
+ dAtA[i] = 0x10
+ i++
+ if m.IsLearner {
+ dAtA[i] = 1
+ } else {
+ dAtA[i] = 0
+ }
+ i++
+ }
return i, nil
}
@@ -7026,6 +7160,69 @@
return i, nil
}
+func (m *MemberPromoteRequest) Marshal() (dAtA []byte, err error) {
+ size := m.Size()
+ dAtA = make([]byte, size)
+ n, err := m.MarshalTo(dAtA)
+ if err != nil {
+ return nil, err
+ }
+ return dAtA[:n], nil
+}
+
+func (m *MemberPromoteRequest) MarshalTo(dAtA []byte) (int, error) {
+ var i int
+ _ = i
+ var l int
+ _ = l
+ if m.ID != 0 {
+ dAtA[i] = 0x8
+ i++
+ i = encodeVarintRpc(dAtA, i, uint64(m.ID))
+ }
+ return i, nil
+}
+
+func (m *MemberPromoteResponse) Marshal() (dAtA []byte, err error) {
+ size := m.Size()
+ dAtA = make([]byte, size)
+ n, err := m.MarshalTo(dAtA)
+ if err != nil {
+ return nil, err
+ }
+ return dAtA[:n], nil
+}
+
+func (m *MemberPromoteResponse) MarshalTo(dAtA []byte) (int, error) {
+ var i int
+ _ = i
+ var l int
+ _ = l
+ if m.Header != nil {
+ dAtA[i] = 0xa
+ i++
+ i = encodeVarintRpc(dAtA, i, uint64(m.Header.Size()))
+ n39, err := m.Header.MarshalTo(dAtA[i:])
+ if err != nil {
+ return 0, err
+ }
+ i += n39
+ }
+ if len(m.Members) > 0 {
+ for _, msg := range m.Members {
+ dAtA[i] = 0x12
+ i++
+ i = encodeVarintRpc(dAtA, i, uint64(msg.Size()))
+ n, err := msg.MarshalTo(dAtA[i:])
+ if err != nil {
+ return 0, err
+ }
+ i += n
+ }
+ }
+ return i, nil
+}
+
func (m *DefragmentRequest) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
@@ -7063,11 +7260,11 @@
dAtA[i] = 0xa
i++
i = encodeVarintRpc(dAtA, i, uint64(m.Header.Size()))
- n39, err := m.Header.MarshalTo(dAtA[i:])
+ n40, err := m.Header.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
- i += n39
+ i += n40
}
return i, nil
}
@@ -7114,11 +7311,11 @@
dAtA[i] = 0xa
i++
i = encodeVarintRpc(dAtA, i, uint64(m.Header.Size()))
- n40, err := m.Header.MarshalTo(dAtA[i:])
+ n41, err := m.Header.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
- i += n40
+ i += n41
}
return i, nil
}
@@ -7203,11 +7400,11 @@
dAtA[i] = 0xa
i++
i = encodeVarintRpc(dAtA, i, uint64(m.Header.Size()))
- n41, err := m.Header.MarshalTo(dAtA[i:])
+ n42, err := m.Header.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
- i += n41
+ i += n42
}
if len(m.Alarms) > 0 {
for _, msg := range m.Alarms {
@@ -7261,11 +7458,11 @@
dAtA[i] = 0xa
i++
i = encodeVarintRpc(dAtA, i, uint64(m.Header.Size()))
- n42, err := m.Header.MarshalTo(dAtA[i:])
+ n43, err := m.Header.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
- i += n42
+ i += n43
}
if len(m.Version) > 0 {
dAtA[i] = 0x12
@@ -7318,6 +7515,16 @@
i++
i = encodeVarintRpc(dAtA, i, uint64(m.DbSizeInUse))
}
+ if m.IsLearner {
+ dAtA[i] = 0x50
+ i++
+ if m.IsLearner {
+ dAtA[i] = 1
+ } else {
+ dAtA[i] = 0
+ }
+ i++
+ }
return i, nil
}
@@ -7414,6 +7621,16 @@
i = encodeVarintRpc(dAtA, i, uint64(len(m.Password)))
i += copy(dAtA[i:], m.Password)
}
+ if m.Options != nil {
+ dAtA[i] = 0x1a
+ i++
+ i = encodeVarintRpc(dAtA, i, uint64(m.Options.Size()))
+ n44, err := m.Options.MarshalTo(dAtA[i:])
+ if err != nil {
+ return 0, err
+ }
+ i += n44
+ }
return i, nil
}
@@ -7688,11 +7905,11 @@
dAtA[i] = 0x12
i++
i = encodeVarintRpc(dAtA, i, uint64(m.Perm.Size()))
- n43, err := m.Perm.MarshalTo(dAtA[i:])
+ n45, err := m.Perm.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
- i += n43
+ i += n45
}
return i, nil
}
@@ -7752,11 +7969,11 @@
dAtA[i] = 0xa
i++
i = encodeVarintRpc(dAtA, i, uint64(m.Header.Size()))
- n44, err := m.Header.MarshalTo(dAtA[i:])
+ n46, err := m.Header.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
- i += n44
+ i += n46
}
return i, nil
}
@@ -7780,11 +7997,11 @@
dAtA[i] = 0xa
i++
i = encodeVarintRpc(dAtA, i, uint64(m.Header.Size()))
- n45, err := m.Header.MarshalTo(dAtA[i:])
+ n47, err := m.Header.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
- i += n45
+ i += n47
}
return i, nil
}
@@ -7808,11 +8025,11 @@
dAtA[i] = 0xa
i++
i = encodeVarintRpc(dAtA, i, uint64(m.Header.Size()))
- n46, err := m.Header.MarshalTo(dAtA[i:])
+ n48, err := m.Header.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
- i += n46
+ i += n48
}
if len(m.Token) > 0 {
dAtA[i] = 0x12
@@ -7842,11 +8059,11 @@
dAtA[i] = 0xa
i++
i = encodeVarintRpc(dAtA, i, uint64(m.Header.Size()))
- n47, err := m.Header.MarshalTo(dAtA[i:])
+ n49, err := m.Header.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
- i += n47
+ i += n49
}
return i, nil
}
@@ -7870,11 +8087,11 @@
dAtA[i] = 0xa
i++
i = encodeVarintRpc(dAtA, i, uint64(m.Header.Size()))
- n48, err := m.Header.MarshalTo(dAtA[i:])
+ n50, err := m.Header.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
- i += n48
+ i += n50
}
if len(m.Roles) > 0 {
for _, s := range m.Roles {
@@ -7913,11 +8130,11 @@
dAtA[i] = 0xa
i++
i = encodeVarintRpc(dAtA, i, uint64(m.Header.Size()))
- n49, err := m.Header.MarshalTo(dAtA[i:])
+ n51, err := m.Header.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
- i += n49
+ i += n51
}
return i, nil
}
@@ -7941,11 +8158,11 @@
dAtA[i] = 0xa
i++
i = encodeVarintRpc(dAtA, i, uint64(m.Header.Size()))
- n50, err := m.Header.MarshalTo(dAtA[i:])
+ n52, err := m.Header.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
- i += n50
+ i += n52
}
return i, nil
}
@@ -7969,11 +8186,11 @@
dAtA[i] = 0xa
i++
i = encodeVarintRpc(dAtA, i, uint64(m.Header.Size()))
- n51, err := m.Header.MarshalTo(dAtA[i:])
+ n53, err := m.Header.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
- i += n51
+ i += n53
}
return i, nil
}
@@ -7997,11 +8214,11 @@
dAtA[i] = 0xa
i++
i = encodeVarintRpc(dAtA, i, uint64(m.Header.Size()))
- n52, err := m.Header.MarshalTo(dAtA[i:])
+ n54, err := m.Header.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
- i += n52
+ i += n54
}
return i, nil
}
@@ -8025,11 +8242,11 @@
dAtA[i] = 0xa
i++
i = encodeVarintRpc(dAtA, i, uint64(m.Header.Size()))
- n53, err := m.Header.MarshalTo(dAtA[i:])
+ n55, err := m.Header.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
- i += n53
+ i += n55
}
return i, nil
}
@@ -8053,11 +8270,11 @@
dAtA[i] = 0xa
i++
i = encodeVarintRpc(dAtA, i, uint64(m.Header.Size()))
- n54, err := m.Header.MarshalTo(dAtA[i:])
+ n56, err := m.Header.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
- i += n54
+ i += n56
}
if len(m.Perm) > 0 {
for _, msg := range m.Perm {
@@ -8093,11 +8310,11 @@
dAtA[i] = 0xa
i++
i = encodeVarintRpc(dAtA, i, uint64(m.Header.Size()))
- n55, err := m.Header.MarshalTo(dAtA[i:])
+ n57, err := m.Header.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
- i += n55
+ i += n57
}
if len(m.Roles) > 0 {
for _, s := range m.Roles {
@@ -8136,11 +8353,11 @@
dAtA[i] = 0xa
i++
i = encodeVarintRpc(dAtA, i, uint64(m.Header.Size()))
- n56, err := m.Header.MarshalTo(dAtA[i:])
+ n58, err := m.Header.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
- i += n56
+ i += n58
}
if len(m.Users) > 0 {
for _, s := range m.Users {
@@ -8179,11 +8396,11 @@
dAtA[i] = 0xa
i++
i = encodeVarintRpc(dAtA, i, uint64(m.Header.Size()))
- n57, err := m.Header.MarshalTo(dAtA[i:])
+ n59, err := m.Header.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
- i += n57
+ i += n59
}
return i, nil
}
@@ -8207,11 +8424,11 @@
dAtA[i] = 0xa
i++
i = encodeVarintRpc(dAtA, i, uint64(m.Header.Size()))
- n58, err := m.Header.MarshalTo(dAtA[i:])
+ n60, err := m.Header.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
- i += n58
+ i += n60
}
return i, nil
}
@@ -8235,11 +8452,11 @@
dAtA[i] = 0xa
i++
i = encodeVarintRpc(dAtA, i, uint64(m.Header.Size()))
- n59, err := m.Header.MarshalTo(dAtA[i:])
+ n61, err := m.Header.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
- i += n59
+ i += n61
}
return i, nil
}
@@ -9016,6 +9233,9 @@
n += 1 + l + sovRpc(uint64(l))
}
}
+ if m.IsLearner {
+ n += 2
+ }
return n
}
@@ -9028,6 +9248,9 @@
n += 1 + l + sovRpc(uint64(l))
}
}
+ if m.IsLearner {
+ n += 2
+ }
return n
}
@@ -9129,6 +9352,31 @@
return n
}
+func (m *MemberPromoteRequest) Size() (n int) {
+ var l int
+ _ = l
+ if m.ID != 0 {
+ n += 1 + sovRpc(uint64(m.ID))
+ }
+ return n
+}
+
+func (m *MemberPromoteResponse) Size() (n int) {
+ var l int
+ _ = l
+ if m.Header != nil {
+ l = m.Header.Size()
+ n += 1 + l + sovRpc(uint64(l))
+ }
+ if len(m.Members) > 0 {
+ for _, e := range m.Members {
+ l = e.Size()
+ n += 1 + l + sovRpc(uint64(l))
+ }
+ }
+ return n
+}
+
func (m *DefragmentRequest) Size() (n int) {
var l int
_ = l
@@ -9248,6 +9496,9 @@
if m.DbSizeInUse != 0 {
n += 1 + sovRpc(uint64(m.DbSizeInUse))
}
+ if m.IsLearner {
+ n += 2
+ }
return n
}
@@ -9288,6 +9539,10 @@
if l > 0 {
n += 1 + l + sovRpc(uint64(l))
}
+ if m.Options != nil {
+ l = m.Options.Size()
+ n += 1 + l + sovRpc(uint64(l))
+ }
return n
}
@@ -14629,6 +14884,26 @@
}
m.ClientURLs = append(m.ClientURLs, string(dAtA[iNdEx:postIndex]))
iNdEx = postIndex
+ case 5:
+ if wireType != 0 {
+ return fmt.Errorf("proto: wrong wireType = %d for field IsLearner", wireType)
+ }
+ var v int
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowRpc
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ v |= (int(b) & 0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ m.IsLearner = bool(v != 0)
default:
iNdEx = preIndex
skippy, err := skipRpc(dAtA[iNdEx:])
@@ -14708,6 +14983,26 @@
}
m.PeerURLs = append(m.PeerURLs, string(dAtA[iNdEx:postIndex]))
iNdEx = postIndex
+ case 2:
+ if wireType != 0 {
+ return fmt.Errorf("proto: wrong wireType = %d for field IsLearner", wireType)
+ }
+ var v int
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowRpc
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ v |= (int(b) & 0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ m.IsLearner = bool(v != 0)
default:
iNdEx = preIndex
skippy, err := skipRpc(dAtA[iNdEx:])
@@ -15435,6 +15730,189 @@
}
return nil
}
+func (m *MemberPromoteRequest) Unmarshal(dAtA []byte) error {
+ l := len(dAtA)
+ iNdEx := 0
+ for iNdEx < l {
+ preIndex := iNdEx
+ var wire uint64
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowRpc
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ wire |= (uint64(b) & 0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ fieldNum := int32(wire >> 3)
+ wireType := int(wire & 0x7)
+ if wireType == 4 {
+ return fmt.Errorf("proto: MemberPromoteRequest: wiretype end group for non-group")
+ }
+ if fieldNum <= 0 {
+ return fmt.Errorf("proto: MemberPromoteRequest: illegal tag %d (wire type %d)", fieldNum, wire)
+ }
+ switch fieldNum {
+ case 1:
+ if wireType != 0 {
+ return fmt.Errorf("proto: wrong wireType = %d for field ID", wireType)
+ }
+ m.ID = 0
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowRpc
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ m.ID |= (uint64(b) & 0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ default:
+ iNdEx = preIndex
+ skippy, err := skipRpc(dAtA[iNdEx:])
+ if err != nil {
+ return err
+ }
+ if skippy < 0 {
+ return ErrInvalidLengthRpc
+ }
+ if (iNdEx + skippy) > l {
+ return io.ErrUnexpectedEOF
+ }
+ iNdEx += skippy
+ }
+ }
+
+ if iNdEx > l {
+ return io.ErrUnexpectedEOF
+ }
+ return nil
+}
+func (m *MemberPromoteResponse) Unmarshal(dAtA []byte) error {
+ l := len(dAtA)
+ iNdEx := 0
+ for iNdEx < l {
+ preIndex := iNdEx
+ var wire uint64
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowRpc
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ wire |= (uint64(b) & 0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ fieldNum := int32(wire >> 3)
+ wireType := int(wire & 0x7)
+ if wireType == 4 {
+ return fmt.Errorf("proto: MemberPromoteResponse: wiretype end group for non-group")
+ }
+ if fieldNum <= 0 {
+ return fmt.Errorf("proto: MemberPromoteResponse: illegal tag %d (wire type %d)", fieldNum, wire)
+ }
+ switch fieldNum {
+ case 1:
+ if wireType != 2 {
+ return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType)
+ }
+ var msglen int
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowRpc
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ msglen |= (int(b) & 0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ if msglen < 0 {
+ return ErrInvalidLengthRpc
+ }
+ postIndex := iNdEx + msglen
+ if postIndex > l {
+ return io.ErrUnexpectedEOF
+ }
+ if m.Header == nil {
+ m.Header = &ResponseHeader{}
+ }
+ if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+ return err
+ }
+ iNdEx = postIndex
+ case 2:
+ if wireType != 2 {
+ return fmt.Errorf("proto: wrong wireType = %d for field Members", wireType)
+ }
+ var msglen int
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowRpc
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ msglen |= (int(b) & 0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ if msglen < 0 {
+ return ErrInvalidLengthRpc
+ }
+ postIndex := iNdEx + msglen
+ if postIndex > l {
+ return io.ErrUnexpectedEOF
+ }
+ m.Members = append(m.Members, &Member{})
+ if err := m.Members[len(m.Members)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+ return err
+ }
+ iNdEx = postIndex
+ default:
+ iNdEx = preIndex
+ skippy, err := skipRpc(dAtA[iNdEx:])
+ if err != nil {
+ return err
+ }
+ if skippy < 0 {
+ return ErrInvalidLengthRpc
+ }
+ if (iNdEx + skippy) > l {
+ return io.ErrUnexpectedEOF
+ }
+ iNdEx += skippy
+ }
+ }
+
+ if iNdEx > l {
+ return io.ErrUnexpectedEOF
+ }
+ return nil
+}
func (m *DefragmentRequest) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
@@ -16313,6 +16791,26 @@
break
}
}
+ case 10:
+ if wireType != 0 {
+ return fmt.Errorf("proto: wrong wireType = %d for field IsLearner", wireType)
+ }
+ var v int
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowRpc
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ v |= (int(b) & 0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ m.IsLearner = bool(v != 0)
default:
iNdEx = preIndex
skippy, err := skipRpc(dAtA[iNdEx:])
@@ -16629,6 +17127,39 @@
}
m.Password = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
+ case 3:
+ if wireType != 2 {
+ return fmt.Errorf("proto: wrong wireType = %d for field Options", wireType)
+ }
+ var msglen int
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowRpc
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ msglen |= (int(b) & 0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ if msglen < 0 {
+ return ErrInvalidLengthRpc
+ }
+ postIndex := iNdEx + msglen
+ if postIndex > l {
+ return io.ErrUnexpectedEOF
+ }
+ if m.Options == nil {
+ m.Options = &authpb.UserAddOptions{}
+ }
+ if err := m.Options.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+ return err
+ }
+ iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipRpc(dAtA[iNdEx:])
@@ -19305,245 +19836,251 @@
func init() { proto.RegisterFile("rpc.proto", fileDescriptorRpc) }
var fileDescriptorRpc = []byte{
- // 3836 bytes of a gzipped FileDescriptorProto
- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x5b, 0xdd, 0x6f, 0x23, 0xc9,
- 0x71, 0xd7, 0x90, 0xe2, 0x57, 0xf1, 0x43, 0x54, 0xeb, 0x63, 0x29, 0xee, 0xae, 0x56, 0xd7, 0xbb,
- 0x7b, 0xab, 0xdb, 0xbd, 0x13, 0x6d, 0xd9, 0x4e, 0x80, 0x4d, 0xe2, 0x58, 0x2b, 0xf1, 0x56, 0x3a,
- 0x69, 0x45, 0xdd, 0x88, 0xda, 0xfb, 0x80, 0x11, 0x61, 0x44, 0xf6, 0x4a, 0x13, 0x91, 0x33, 0xf4,
- 0xcc, 0x90, 0x2b, 0x5d, 0x82, 0x38, 0x30, 0x9c, 0x00, 0xc9, 0xa3, 0x0d, 0x04, 0xc9, 0x43, 0x9e,
- 0x82, 0x20, 0xf0, 0x43, 0x80, 0xbc, 0x05, 0xc8, 0x5f, 0x90, 0xb7, 0x24, 0xc8, 0x3f, 0x10, 0x5c,
- 0xfc, 0x92, 0xff, 0x22, 0xe8, 0xaf, 0x99, 0x9e, 0x2f, 0x69, 0x6d, 0xfa, 0xfc, 0x22, 0x4d, 0x57,
- 0x57, 0x57, 0x55, 0x57, 0x77, 0x57, 0x55, 0xff, 0x66, 0x08, 0x25, 0x67, 0xd4, 0xdb, 0x18, 0x39,
- 0xb6, 0x67, 0xa3, 0x0a, 0xf1, 0x7a, 0x7d, 0x97, 0x38, 0x13, 0xe2, 0x8c, 0xce, 0x9a, 0x8b, 0xe7,
- 0xf6, 0xb9, 0xcd, 0x3a, 0x5a, 0xf4, 0x89, 0xf3, 0x34, 0x57, 0x28, 0x4f, 0x6b, 0x38, 0xe9, 0xf5,
- 0xd8, 0x9f, 0xd1, 0x59, 0xeb, 0x72, 0x22, 0xba, 0xee, 0xb2, 0x2e, 0x63, 0xec, 0x5d, 0xb0, 0x3f,
- 0xa3, 0x33, 0xf6, 0x4f, 0x74, 0xde, 0x3b, 0xb7, 0xed, 0xf3, 0x01, 0x69, 0x19, 0x23, 0xb3, 0x65,
- 0x58, 0x96, 0xed, 0x19, 0x9e, 0x69, 0x5b, 0x2e, 0xef, 0xc5, 0x7f, 0xa1, 0x41, 0x4d, 0x27, 0xee,
- 0xc8, 0xb6, 0x5c, 0xb2, 0x4b, 0x8c, 0x3e, 0x71, 0xd0, 0x7d, 0x80, 0xde, 0x60, 0xec, 0x7a, 0xc4,
- 0x39, 0x35, 0xfb, 0x0d, 0x6d, 0x4d, 0x5b, 0x9f, 0xd5, 0x4b, 0x82, 0xb2, 0xd7, 0x47, 0x77, 0xa1,
- 0x34, 0x24, 0xc3, 0x33, 0xde, 0x9b, 0x61, 0xbd, 0x45, 0x4e, 0xd8, 0xeb, 0xa3, 0x26, 0x14, 0x1d,
- 0x32, 0x31, 0x5d, 0xd3, 0xb6, 0x1a, 0xd9, 0x35, 0x6d, 0x3d, 0xab, 0xfb, 0x6d, 0x3a, 0xd0, 0x31,
- 0xde, 0x78, 0xa7, 0x1e, 0x71, 0x86, 0x8d, 0x59, 0x3e, 0x90, 0x12, 0xba, 0xc4, 0x19, 0xe2, 0x9f,
- 0xe6, 0xa0, 0xa2, 0x1b, 0xd6, 0x39, 0xd1, 0xc9, 0x8f, 0xc6, 0xc4, 0xf5, 0x50, 0x1d, 0xb2, 0x97,
- 0xe4, 0x9a, 0xa9, 0xaf, 0xe8, 0xf4, 0x91, 0x8f, 0xb7, 0xce, 0xc9, 0x29, 0xb1, 0xb8, 0xe2, 0x0a,
- 0x1d, 0x6f, 0x9d, 0x93, 0xb6, 0xd5, 0x47, 0x8b, 0x90, 0x1b, 0x98, 0x43, 0xd3, 0x13, 0x5a, 0x79,
- 0x23, 0x64, 0xce, 0x6c, 0xc4, 0x9c, 0x6d, 0x00, 0xd7, 0x76, 0xbc, 0x53, 0xdb, 0xe9, 0x13, 0xa7,
- 0x91, 0x5b, 0xd3, 0xd6, 0x6b, 0x9b, 0x8f, 0x36, 0xd4, 0x85, 0xd8, 0x50, 0x0d, 0xda, 0x38, 0xb6,
- 0x1d, 0xaf, 0x43, 0x79, 0xf5, 0x92, 0x2b, 0x1f, 0xd1, 0xc7, 0x50, 0x66, 0x42, 0x3c, 0xc3, 0x39,
- 0x27, 0x5e, 0x23, 0xcf, 0xa4, 0x3c, 0xbe, 0x45, 0x4a, 0x97, 0x31, 0xeb, 0x4c, 0x3d, 0x7f, 0x46,
- 0x18, 0x2a, 0x2e, 0x71, 0x4c, 0x63, 0x60, 0x7e, 0x65, 0x9c, 0x0d, 0x48, 0xa3, 0xb0, 0xa6, 0xad,
- 0x17, 0xf5, 0x10, 0x8d, 0xce, 0xff, 0x92, 0x5c, 0xbb, 0xa7, 0xb6, 0x35, 0xb8, 0x6e, 0x14, 0x19,
- 0x43, 0x91, 0x12, 0x3a, 0xd6, 0xe0, 0x9a, 0x2d, 0x9a, 0x3d, 0xb6, 0x3c, 0xde, 0x5b, 0x62, 0xbd,
- 0x25, 0x46, 0x61, 0xdd, 0xeb, 0x50, 0x1f, 0x9a, 0xd6, 0xe9, 0xd0, 0xee, 0x9f, 0xfa, 0x0e, 0x01,
- 0xe6, 0x90, 0xda, 0xd0, 0xb4, 0x5e, 0xd9, 0x7d, 0x5d, 0xba, 0x85, 0x72, 0x1a, 0x57, 0x61, 0xce,
- 0xb2, 0xe0, 0x34, 0xae, 0x54, 0xce, 0x0d, 0x58, 0xa0, 0x32, 0x7b, 0x0e, 0x31, 0x3c, 0x12, 0x30,
- 0x57, 0x18, 0xf3, 0xfc, 0xd0, 0xb4, 0xb6, 0x59, 0x4f, 0x88, 0xdf, 0xb8, 0x8a, 0xf1, 0x57, 0x05,
- 0xbf, 0x71, 0x15, 0xe6, 0xc7, 0x1b, 0x50, 0xf2, 0x7d, 0x8e, 0x8a, 0x30, 0x7b, 0xd8, 0x39, 0x6c,
- 0xd7, 0x67, 0x10, 0x40, 0x7e, 0xeb, 0x78, 0xbb, 0x7d, 0xb8, 0x53, 0xd7, 0x50, 0x19, 0x0a, 0x3b,
- 0x6d, 0xde, 0xc8, 0xe0, 0x17, 0x00, 0x81, 0x77, 0x51, 0x01, 0xb2, 0xfb, 0xed, 0x2f, 0xea, 0x33,
- 0x94, 0xe7, 0x75, 0x5b, 0x3f, 0xde, 0xeb, 0x1c, 0xd6, 0x35, 0x3a, 0x78, 0x5b, 0x6f, 0x6f, 0x75,
- 0xdb, 0xf5, 0x0c, 0xe5, 0x78, 0xd5, 0xd9, 0xa9, 0x67, 0x51, 0x09, 0x72, 0xaf, 0xb7, 0x0e, 0x4e,
- 0xda, 0xf5, 0x59, 0xfc, 0x73, 0x0d, 0xaa, 0x62, 0xbd, 0xf8, 0x99, 0x40, 0xdf, 0x85, 0xfc, 0x05,
- 0x3b, 0x17, 0x6c, 0x2b, 0x96, 0x37, 0xef, 0x45, 0x16, 0x37, 0x74, 0x76, 0x74, 0xc1, 0x8b, 0x30,
- 0x64, 0x2f, 0x27, 0x6e, 0x23, 0xb3, 0x96, 0x5d, 0x2f, 0x6f, 0xd6, 0x37, 0xf8, 0x81, 0xdd, 0xd8,
- 0x27, 0xd7, 0xaf, 0x8d, 0xc1, 0x98, 0xe8, 0xb4, 0x13, 0x21, 0x98, 0x1d, 0xda, 0x0e, 0x61, 0x3b,
+ // 3928 bytes of a gzipped FileDescriptorProto
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x5b, 0x5b, 0x6f, 0x23, 0xc9,
+ 0x75, 0x56, 0x93, 0xe2, 0xed, 0xf0, 0x22, 0xaa, 0x74, 0x19, 0x0e, 0x67, 0x46, 0xa3, 0xad, 0xd9,
+ 0xd9, 0xd5, 0xce, 0xec, 0x8a, 0x6b, 0xd9, 0x4e, 0x80, 0x49, 0xe2, 0x58, 0x23, 0x71, 0x67, 0xb4,
+ 0xd2, 0x88, 0xda, 0x16, 0x67, 0xf6, 0x02, 0x23, 0x42, 0x8b, 0x2c, 0x49, 0x1d, 0x91, 0xdd, 0x74,
+ 0x77, 0x93, 0x23, 0x6d, 0x2e, 0x0e, 0x0c, 0xc7, 0x40, 0xf2, 0x68, 0x03, 0x41, 0xf2, 0x90, 0xa7,
+ 0x20, 0x08, 0xfc, 0x90, 0xe7, 0x00, 0xf9, 0x05, 0x79, 0xca, 0x05, 0xf9, 0x03, 0xc1, 0xc6, 0x2f,
+ 0xc9, 0xaf, 0x30, 0xea, 0xd6, 0x5d, 0x7d, 0xa3, 0xc6, 0xa6, 0x77, 0x5f, 0xa4, 0xae, 0x53, 0xa7,
+ 0xce, 0x39, 0x75, 0xaa, 0xea, 0x9c, 0xd3, 0x5f, 0x17, 0xa1, 0xe4, 0x8c, 0x7a, 0x9b, 0x23, 0xc7,
+ 0xf6, 0x6c, 0x54, 0x21, 0x5e, 0xaf, 0xef, 0x12, 0x67, 0x42, 0x9c, 0xd1, 0x69, 0x73, 0xf9, 0xdc,
+ 0x3e, 0xb7, 0x59, 0x47, 0x8b, 0x3e, 0x71, 0x9e, 0xe6, 0x6d, 0xca, 0xd3, 0x1a, 0x4e, 0x7a, 0x3d,
+ 0xf6, 0x67, 0x74, 0xda, 0xba, 0x9c, 0x88, 0xae, 0x3b, 0xac, 0xcb, 0x18, 0x7b, 0x17, 0xec, 0xcf,
+ 0xe8, 0x94, 0xfd, 0x13, 0x9d, 0x77, 0xcf, 0x6d, 0xfb, 0x7c, 0x40, 0x5a, 0xc6, 0xc8, 0x6c, 0x19,
+ 0x96, 0x65, 0x7b, 0x86, 0x67, 0xda, 0x96, 0xcb, 0x7b, 0xf1, 0x5f, 0x6a, 0x50, 0xd3, 0x89, 0x3b,
+ 0xb2, 0x2d, 0x97, 0x3c, 0x27, 0x46, 0x9f, 0x38, 0xe8, 0x1e, 0x40, 0x6f, 0x30, 0x76, 0x3d, 0xe2,
+ 0x9c, 0x98, 0xfd, 0x86, 0xb6, 0xae, 0x6d, 0xcc, 0xeb, 0x25, 0x41, 0xd9, 0xeb, 0xa3, 0x3b, 0x50,
+ 0x1a, 0x92, 0xe1, 0x29, 0xef, 0xcd, 0xb0, 0xde, 0x22, 0x27, 0xec, 0xf5, 0x51, 0x13, 0x8a, 0x0e,
+ 0x99, 0x98, 0xae, 0x69, 0x5b, 0x8d, 0xec, 0xba, 0xb6, 0x91, 0xd5, 0xfd, 0x36, 0x1d, 0xe8, 0x18,
+ 0x67, 0xde, 0x89, 0x47, 0x9c, 0x61, 0x63, 0x9e, 0x0f, 0xa4, 0x84, 0x2e, 0x71, 0x86, 0xf8, 0x27,
+ 0x39, 0xa8, 0xe8, 0x86, 0x75, 0x4e, 0x74, 0xf2, 0xc3, 0x31, 0x71, 0x3d, 0x54, 0x87, 0xec, 0x25,
+ 0xb9, 0x66, 0xea, 0x2b, 0x3a, 0x7d, 0xe4, 0xe3, 0xad, 0x73, 0x72, 0x42, 0x2c, 0xae, 0xb8, 0x42,
+ 0xc7, 0x5b, 0xe7, 0xa4, 0x6d, 0xf5, 0xd1, 0x32, 0xe4, 0x06, 0xe6, 0xd0, 0xf4, 0x84, 0x56, 0xde,
+ 0x08, 0x99, 0x33, 0x1f, 0x31, 0x67, 0x07, 0xc0, 0xb5, 0x1d, 0xef, 0xc4, 0x76, 0xfa, 0xc4, 0x69,
+ 0xe4, 0xd6, 0xb5, 0x8d, 0xda, 0xd6, 0xdb, 0x9b, 0xea, 0x42, 0x6c, 0xaa, 0x06, 0x6d, 0x1e, 0xdb,
+ 0x8e, 0xd7, 0xa1, 0xbc, 0x7a, 0xc9, 0x95, 0x8f, 0xe8, 0x23, 0x28, 0x33, 0x21, 0x9e, 0xe1, 0x9c,
+ 0x13, 0xaf, 0x91, 0x67, 0x52, 0x1e, 0xde, 0x20, 0xa5, 0xcb, 0x98, 0x75, 0xa6, 0x9e, 0x3f, 0x23,
+ 0x0c, 0x15, 0x97, 0x38, 0xa6, 0x31, 0x30, 0xbf, 0x34, 0x4e, 0x07, 0xa4, 0x51, 0x58, 0xd7, 0x36,
+ 0x8a, 0x7a, 0x88, 0x46, 0xe7, 0x7f, 0x49, 0xae, 0xdd, 0x13, 0xdb, 0x1a, 0x5c, 0x37, 0x8a, 0x8c,
+ 0xa1, 0x48, 0x09, 0x1d, 0x6b, 0x70, 0xcd, 0x16, 0xcd, 0x1e, 0x5b, 0x1e, 0xef, 0x2d, 0xb1, 0xde,
+ 0x12, 0xa3, 0xb0, 0xee, 0x0d, 0xa8, 0x0f, 0x4d, 0xeb, 0x64, 0x68, 0xf7, 0x4f, 0x7c, 0x87, 0x00,
+ 0x73, 0x48, 0x6d, 0x68, 0x5a, 0x2f, 0xec, 0xbe, 0x2e, 0xdd, 0x42, 0x39, 0x8d, 0xab, 0x30, 0x67,
+ 0x59, 0x70, 0x1a, 0x57, 0x2a, 0xe7, 0x26, 0x2c, 0x51, 0x99, 0x3d, 0x87, 0x18, 0x1e, 0x09, 0x98,
+ 0x2b, 0x8c, 0x79, 0x71, 0x68, 0x5a, 0x3b, 0xac, 0x27, 0xc4, 0x6f, 0x5c, 0xc5, 0xf8, 0xab, 0x82,
+ 0xdf, 0xb8, 0x0a, 0xf3, 0xe3, 0x4d, 0x28, 0xf9, 0x3e, 0x47, 0x45, 0x98, 0x3f, 0xec, 0x1c, 0xb6,
+ 0xeb, 0x73, 0x08, 0x20, 0xbf, 0x7d, 0xbc, 0xd3, 0x3e, 0xdc, 0xad, 0x6b, 0xa8, 0x0c, 0x85, 0xdd,
+ 0x36, 0x6f, 0x64, 0xf0, 0x53, 0x80, 0xc0, 0xbb, 0xa8, 0x00, 0xd9, 0xfd, 0xf6, 0xe7, 0xf5, 0x39,
+ 0xca, 0xf3, 0xaa, 0xad, 0x1f, 0xef, 0x75, 0x0e, 0xeb, 0x1a, 0x1d, 0xbc, 0xa3, 0xb7, 0xb7, 0xbb,
+ 0xed, 0x7a, 0x86, 0x72, 0xbc, 0xe8, 0xec, 0xd6, 0xb3, 0xa8, 0x04, 0xb9, 0x57, 0xdb, 0x07, 0x2f,
+ 0xdb, 0xf5, 0x79, 0xfc, 0x73, 0x0d, 0xaa, 0x62, 0xbd, 0xf8, 0x99, 0x40, 0xdf, 0x81, 0xfc, 0x05,
+ 0x3b, 0x17, 0x6c, 0x2b, 0x96, 0xb7, 0xee, 0x46, 0x16, 0x37, 0x74, 0x76, 0x74, 0xc1, 0x8b, 0x30,
+ 0x64, 0x2f, 0x27, 0x6e, 0x23, 0xb3, 0x9e, 0xdd, 0x28, 0x6f, 0xd5, 0x37, 0xf9, 0x81, 0xdd, 0xdc,
+ 0x27, 0xd7, 0xaf, 0x8c, 0xc1, 0x98, 0xe8, 0xb4, 0x13, 0x21, 0x98, 0x1f, 0xda, 0x0e, 0x61, 0x3b,
0xb6, 0xa8, 0xb3, 0x67, 0xba, 0x8d, 0xd9, 0xa2, 0x89, 0xdd, 0xca, 0x1b, 0xf8, 0x17, 0x1a, 0xc0,
- 0xd1, 0xd8, 0x4b, 0x3f, 0x1a, 0x8b, 0x90, 0x9b, 0x50, 0xc1, 0xe2, 0x58, 0xf0, 0x06, 0x3b, 0x13,
- 0xc4, 0x70, 0x89, 0x7f, 0x26, 0x68, 0x03, 0xdd, 0x81, 0xc2, 0xc8, 0x21, 0x93, 0xd3, 0xcb, 0x09,
- 0x53, 0x52, 0xd4, 0xf3, 0xb4, 0xb9, 0x3f, 0x41, 0xef, 0x41, 0xc5, 0x3c, 0xb7, 0x6c, 0x87, 0x9c,
- 0x72, 0x59, 0x39, 0xd6, 0x5b, 0xe6, 0x34, 0x66, 0xb7, 0xc2, 0xc2, 0x05, 0xe7, 0x55, 0x96, 0x03,
- 0x4a, 0xc2, 0x16, 0x94, 0x99, 0xa9, 0x53, 0xb9, 0xef, 0x83, 0xc0, 0xc6, 0x0c, 0x1b, 0x16, 0x77,
- 0xa1, 0xb0, 0x1a, 0xff, 0x10, 0xd0, 0x0e, 0x19, 0x10, 0x8f, 0x4c, 0x13, 0x3d, 0x14, 0x9f, 0x64,
- 0x55, 0x9f, 0xe0, 0x9f, 0x69, 0xb0, 0x10, 0x12, 0x3f, 0xd5, 0xb4, 0x1a, 0x50, 0xe8, 0x33, 0x61,
- 0xdc, 0x82, 0xac, 0x2e, 0x9b, 0xe8, 0x19, 0x14, 0x85, 0x01, 0x6e, 0x23, 0x9b, 0xb2, 0x69, 0x0a,
- 0xdc, 0x26, 0x17, 0xff, 0x22, 0x03, 0x25, 0x31, 0xd1, 0xce, 0x08, 0x6d, 0x41, 0xd5, 0xe1, 0x8d,
- 0x53, 0x36, 0x1f, 0x61, 0x51, 0x33, 0x3d, 0x08, 0xed, 0xce, 0xe8, 0x15, 0x31, 0x84, 0x91, 0xd1,
- 0xef, 0x41, 0x59, 0x8a, 0x18, 0x8d, 0x3d, 0xe1, 0xf2, 0x46, 0x58, 0x40, 0xb0, 0xff, 0x76, 0x67,
- 0x74, 0x10, 0xec, 0x47, 0x63, 0x0f, 0x75, 0x61, 0x51, 0x0e, 0xe6, 0xb3, 0x11, 0x66, 0x64, 0x99,
- 0x94, 0xb5, 0xb0, 0x94, 0xf8, 0x52, 0xed, 0xce, 0xe8, 0x48, 0x8c, 0x57, 0x3a, 0x55, 0x93, 0xbc,
- 0x2b, 0x1e, 0xbc, 0x63, 0x26, 0x75, 0xaf, 0xac, 0xb8, 0x49, 0xdd, 0x2b, 0xeb, 0x45, 0x09, 0x0a,
- 0xa2, 0x85, 0xff, 0x35, 0x03, 0x20, 0x57, 0xa3, 0x33, 0x42, 0x3b, 0x50, 0x73, 0x44, 0x2b, 0xe4,
- 0xad, 0xbb, 0x89, 0xde, 0x12, 0x8b, 0x38, 0xa3, 0x57, 0xe5, 0x20, 0x6e, 0xdc, 0xf7, 0xa1, 0xe2,
- 0x4b, 0x09, 0x1c, 0xb6, 0x92, 0xe0, 0x30, 0x5f, 0x42, 0x59, 0x0e, 0xa0, 0x2e, 0xfb, 0x0c, 0x96,
- 0xfc, 0xf1, 0x09, 0x3e, 0x7b, 0xef, 0x06, 0x9f, 0xf9, 0x02, 0x17, 0xa4, 0x04, 0xd5, 0x6b, 0xaa,
- 0x61, 0x81, 0xdb, 0x56, 0x12, 0xdc, 0x16, 0x37, 0x8c, 0x3a, 0x0e, 0x68, 0xbe, 0xe4, 0x4d, 0xfc,
- 0x7f, 0x59, 0x28, 0x6c, 0xdb, 0xc3, 0x91, 0xe1, 0xd0, 0xd5, 0xc8, 0x3b, 0xc4, 0x1d, 0x0f, 0x3c,
- 0xe6, 0xae, 0xda, 0xe6, 0xc3, 0xb0, 0x44, 0xc1, 0x26, 0xff, 0xeb, 0x8c, 0x55, 0x17, 0x43, 0xe8,
- 0x60, 0x91, 0x1e, 0x33, 0xef, 0x30, 0x58, 0x24, 0x47, 0x31, 0x44, 0x1e, 0xe4, 0x6c, 0x70, 0x90,
- 0x9b, 0x50, 0x98, 0x10, 0x27, 0x48, 0xe9, 0xbb, 0x33, 0xba, 0x24, 0xa0, 0x0f, 0x60, 0x2e, 0x9a,
- 0x5e, 0x72, 0x82, 0xa7, 0xd6, 0x0b, 0x67, 0xa3, 0x87, 0x50, 0x09, 0xe5, 0xb8, 0xbc, 0xe0, 0x2b,
- 0x0f, 0x95, 0x14, 0xb7, 0x2c, 0xe3, 0x2a, 0xcd, 0xc7, 0x95, 0xdd, 0x19, 0x19, 0x59, 0x97, 0x65,
- 0x64, 0x2d, 0x8a, 0x51, 0x22, 0xb6, 0x86, 0x82, 0xcc, 0x0f, 0xc2, 0x41, 0x06, 0xff, 0x00, 0xaa,
- 0x21, 0x07, 0xd1, 0xbc, 0xd3, 0xfe, 0xf4, 0x64, 0xeb, 0x80, 0x27, 0xa9, 0x97, 0x2c, 0x2f, 0xe9,
- 0x75, 0x8d, 0xe6, 0xba, 0x83, 0xf6, 0xf1, 0x71, 0x3d, 0x83, 0xaa, 0x50, 0x3a, 0xec, 0x74, 0x4f,
- 0x39, 0x57, 0x16, 0xbf, 0xf4, 0x25, 0x88, 0x24, 0xa7, 0xe4, 0xb6, 0x19, 0x25, 0xb7, 0x69, 0x32,
- 0xb7, 0x65, 0x82, 0xdc, 0xc6, 0xd2, 0xdc, 0x41, 0x7b, 0xeb, 0xb8, 0x5d, 0x9f, 0x7d, 0x51, 0x83,
- 0x0a, 0xf7, 0xef, 0xe9, 0xd8, 0xa2, 0xa9, 0xf6, 0x1f, 0x34, 0x80, 0xe0, 0x34, 0xa1, 0x16, 0x14,
- 0x7a, 0x5c, 0x4f, 0x43, 0x63, 0xc1, 0x68, 0x29, 0x71, 0xc9, 0x74, 0xc9, 0x85, 0xbe, 0x0d, 0x05,
- 0x77, 0xdc, 0xeb, 0x11, 0x57, 0xa6, 0xbc, 0x3b, 0xd1, 0x78, 0x28, 0xa2, 0x95, 0x2e, 0xf9, 0xe8,
- 0x90, 0x37, 0x86, 0x39, 0x18, 0xb3, 0x04, 0x78, 0xf3, 0x10, 0xc1, 0x87, 0xff, 0x4e, 0x83, 0xb2,
- 0xb2, 0x79, 0x7f, 0xcd, 0x20, 0x7c, 0x0f, 0x4a, 0xcc, 0x06, 0xd2, 0x17, 0x61, 0xb8, 0xa8, 0x07,
- 0x04, 0xf4, 0x3b, 0x50, 0x92, 0x27, 0x40, 0x46, 0xe2, 0x46, 0xb2, 0xd8, 0xce, 0x48, 0x0f, 0x58,
- 0xf1, 0x3e, 0xcc, 0x33, 0xaf, 0xf4, 0x68, 0x71, 0x2d, 0xfd, 0xa8, 0x96, 0x9f, 0x5a, 0xa4, 0xfc,
- 0x6c, 0x42, 0x71, 0x74, 0x71, 0xed, 0x9a, 0x3d, 0x63, 0x20, 0xac, 0xf0, 0xdb, 0xf8, 0x13, 0x40,
- 0xaa, 0xb0, 0x69, 0xa6, 0x8b, 0xab, 0x50, 0xde, 0x35, 0xdc, 0x0b, 0x61, 0x12, 0x7e, 0x06, 0x55,
- 0xda, 0xdc, 0x7f, 0xfd, 0x0e, 0x36, 0xb2, 0xcb, 0x81, 0xe4, 0x9e, 0xca, 0xe7, 0x08, 0x66, 0x2f,
- 0x0c, 0xf7, 0x82, 0x4d, 0xb4, 0xaa, 0xb3, 0x67, 0xf4, 0x01, 0xd4, 0x7b, 0x7c, 0x92, 0xa7, 0x91,
- 0x2b, 0xc3, 0x9c, 0xa0, 0xfb, 0x95, 0xe0, 0xe7, 0x50, 0xe1, 0x73, 0xf8, 0x4d, 0x1b, 0x81, 0xe7,
- 0x61, 0xee, 0xd8, 0x32, 0x46, 0xee, 0x85, 0x2d, 0xb3, 0x1b, 0x9d, 0x74, 0x3d, 0xa0, 0x4d, 0xa5,
- 0xf1, 0x09, 0xcc, 0x39, 0x64, 0x68, 0x98, 0x96, 0x69, 0x9d, 0x9f, 0x9e, 0x5d, 0x7b, 0xc4, 0x15,
- 0x17, 0xa6, 0x9a, 0x4f, 0x7e, 0x41, 0xa9, 0xd4, 0xb4, 0xb3, 0x81, 0x7d, 0x26, 0xc2, 0x1c, 0x7b,
- 0xc6, 0x7f, 0x99, 0x81, 0xca, 0x67, 0x86, 0xd7, 0x93, 0x4b, 0x87, 0xf6, 0xa0, 0xe6, 0x07, 0x37,
- 0x46, 0x11, 0xb6, 0x44, 0x52, 0x2c, 0x1b, 0x23, 0x4b, 0x69, 0x99, 0x1d, 0xab, 0x3d, 0x95, 0xc0,
- 0x44, 0x19, 0x56, 0x8f, 0x0c, 0x7c, 0x51, 0x99, 0x74, 0x51, 0x8c, 0x51, 0x15, 0xa5, 0x12, 0x50,
- 0x07, 0xea, 0x23, 0xc7, 0x3e, 0x77, 0x88, 0xeb, 0xfa, 0xc2, 0x78, 0x1a, 0xc3, 0x09, 0xc2, 0x8e,
- 0x04, 0x6b, 0x20, 0x6e, 0x6e, 0x14, 0x26, 0xbd, 0x98, 0x0b, 0xea, 0x19, 0x1e, 0x9c, 0xfe, 0x2b,
- 0x03, 0x28, 0x3e, 0xa9, 0x5f, 0xb5, 0xc4, 0x7b, 0x0c, 0x35, 0xd7, 0x33, 0x9c, 0xd8, 0x66, 0xab,
- 0x32, 0xaa, 0x1f, 0xf1, 0x9f, 0x80, 0x6f, 0xd0, 0xa9, 0x65, 0x7b, 0xe6, 0x9b, 0x6b, 0x51, 0x25,
- 0xd7, 0x24, 0xf9, 0x90, 0x51, 0x51, 0x1b, 0x0a, 0x6f, 0xcc, 0x81, 0x47, 0x1c, 0xb7, 0x91, 0x5b,
- 0xcb, 0xae, 0xd7, 0x36, 0x9f, 0xdd, 0xb6, 0x0c, 0x1b, 0x1f, 0x33, 0xfe, 0xee, 0xf5, 0x88, 0xe8,
- 0x72, 0xac, 0x5a, 0x79, 0xe6, 0x43, 0xd5, 0xf8, 0x0a, 0x14, 0xdf, 0x52, 0x11, 0xf4, 0x96, 0x5d,
- 0xe0, 0xc5, 0x22, 0x6b, 0xf3, 0x4b, 0xf6, 0x1b, 0xc7, 0x38, 0x1f, 0x12, 0xcb, 0x93, 0xf7, 0x40,
- 0xd9, 0xc6, 0x8f, 0x01, 0x02, 0x35, 0x34, 0xe4, 0x1f, 0x76, 0x8e, 0x4e, 0xba, 0xf5, 0x19, 0x54,
- 0x81, 0xe2, 0x61, 0x67, 0xa7, 0x7d, 0xd0, 0xa6, 0xf9, 0x01, 0xb7, 0xa4, 0x4b, 0x43, 0x6b, 0xa9,
- 0xea, 0xd4, 0x42, 0x3a, 0xf1, 0x32, 0x2c, 0x26, 0x2d, 0x20, 0xad, 0x45, 0xab, 0x62, 0x97, 0x4e,
- 0x75, 0x54, 0x54, 0xd5, 0x99, 0xf0, 0x74, 0x1b, 0x50, 0xe0, 0xbb, 0xb7, 0x2f, 0x8a, 0x73, 0xd9,
- 0xa4, 0x8e, 0xe0, 0x9b, 0x91, 0xf4, 0xc5, 0x2a, 0xf9, 0xed, 0xc4, 0xf0, 0x92, 0x4b, 0x0c, 0x2f,
- 0xe8, 0x21, 0x54, 0xfd, 0xd3, 0x60, 0xb8, 0xa2, 0x16, 0x28, 0xe9, 0x15, 0xb9, 0xd1, 0x29, 0x2d,
- 0xe4, 0xf4, 0x42, 0xd8, 0xe9, 0xe8, 0x31, 0xe4, 0xc9, 0x84, 0x58, 0x9e, 0xdb, 0x28, 0xb3, 0x8c,
- 0x51, 0x95, 0xb5, 0x7b, 0x9b, 0x52, 0x75, 0xd1, 0x89, 0xbf, 0x07, 0xf3, 0xec, 0x8e, 0xf4, 0xd2,
- 0x31, 0x2c, 0xf5, 0x32, 0xd7, 0xed, 0x1e, 0x08, 0x77, 0xd3, 0x47, 0x54, 0x83, 0xcc, 0xde, 0x8e,
- 0x70, 0x42, 0x66, 0x6f, 0x07, 0xff, 0x44, 0x03, 0xa4, 0x8e, 0x9b, 0xca, 0xcf, 0x11, 0xe1, 0x52,
- 0x7d, 0x36, 0x50, 0xbf, 0x08, 0x39, 0xe2, 0x38, 0xb6, 0xc3, 0x3c, 0x5a, 0xd2, 0x79, 0x03, 0x3f,
- 0x12, 0x36, 0xe8, 0x64, 0x62, 0x5f, 0xfa, 0x67, 0x90, 0x4b, 0xd3, 0x7c, 0x53, 0xf7, 0x61, 0x21,
- 0xc4, 0x35, 0x55, 0xe6, 0xfa, 0x18, 0xe6, 0x98, 0xb0, 0xed, 0x0b, 0xd2, 0xbb, 0x1c, 0xd9, 0xa6,
- 0x15, 0xd3, 0x47, 0x57, 0x2e, 0x08, 0xb0, 0x74, 0x1e, 0x7c, 0x62, 0x15, 0x9f, 0xd8, 0xed, 0x1e,
- 0xe0, 0x2f, 0x60, 0x39, 0x22, 0x47, 0x9a, 0xff, 0x87, 0x50, 0xee, 0xf9, 0x44, 0x57, 0xd4, 0x3a,
- 0xf7, 0xc3, 0xc6, 0x45, 0x87, 0xaa, 0x23, 0x70, 0x07, 0xee, 0xc4, 0x44, 0x4f, 0x35, 0xe7, 0x27,
- 0xb0, 0xc4, 0x04, 0xee, 0x13, 0x32, 0xda, 0x1a, 0x98, 0x93, 0x54, 0x4f, 0x8f, 0xc4, 0xa4, 0x14,
- 0xc6, 0x6f, 0x76, 0x5f, 0xe0, 0xdf, 0x17, 0x1a, 0xbb, 0xe6, 0x90, 0x74, 0xed, 0x83, 0x74, 0xdb,
- 0x68, 0x36, 0xbb, 0x24, 0xd7, 0xae, 0x28, 0x6b, 0xd8, 0x33, 0xfe, 0x47, 0x4d, 0xb8, 0x4a, 0x1d,
- 0xfe, 0x0d, 0xef, 0xe4, 0x55, 0x80, 0x73, 0x7a, 0x64, 0x48, 0x9f, 0x76, 0x70, 0x44, 0x45, 0xa1,
- 0xf8, 0x76, 0xd2, 0xf8, 0x5d, 0x11, 0x76, 0x2e, 0x8a, 0x7d, 0xce, 0xfe, 0xf8, 0x51, 0xee, 0x3e,
- 0x94, 0x19, 0xe1, 0xd8, 0x33, 0xbc, 0xb1, 0x1b, 0x5b, 0x8c, 0x3f, 0x13, 0xdb, 0x5e, 0x0e, 0x9a,
- 0x6a, 0x5e, 0xdf, 0x86, 0x3c, 0xbb, 0x4c, 0xc8, 0x52, 0x7a, 0x25, 0x61, 0x3f, 0x72, 0x3b, 0x74,
- 0xc1, 0x88, 0x2f, 0x20, 0xff, 0x8a, 0x21, 0xb0, 0x8a, 0x65, 0xb3, 0x72, 0x29, 0x2c, 0x63, 0xc8,
- 0x71, 0xa1, 0x92, 0xce, 0x9e, 0x59, 0xe5, 0x49, 0x88, 0x73, 0xa2, 0x1f, 0xf0, 0x0a, 0xb7, 0xa4,
- 0xfb, 0x6d, 0xea, 0xb2, 0xde, 0xc0, 0x24, 0x96, 0xc7, 0x7a, 0x67, 0x59, 0xaf, 0x42, 0xc1, 0x1b,
- 0x50, 0xe7, 0x9a, 0xb6, 0xfa, 0x7d, 0xa5, 0x82, 0xf4, 0xe5, 0x69, 0x61, 0x79, 0xf8, 0x9f, 0x34,
- 0x98, 0x57, 0x06, 0x4c, 0xe5, 0x98, 0x0f, 0x21, 0xcf, 0x71, 0x66, 0x51, 0xac, 0x2c, 0x86, 0x47,
- 0x71, 0x35, 0xba, 0xe0, 0x41, 0x1b, 0x50, 0xe0, 0x4f, 0xb2, 0x8c, 0x4f, 0x66, 0x97, 0x4c, 0xf8,
- 0x31, 0x2c, 0x08, 0x12, 0x19, 0xda, 0x49, 0x7b, 0x9b, 0x39, 0x14, 0xff, 0x29, 0x2c, 0x86, 0xd9,
- 0xa6, 0x9a, 0x92, 0x62, 0x64, 0xe6, 0x5d, 0x8c, 0xdc, 0x92, 0x46, 0x9e, 0x8c, 0xfa, 0x4a, 0x29,
- 0x14, 0x5d, 0x75, 0x75, 0x45, 0x32, 0x91, 0x15, 0xf1, 0x27, 0x20, 0x45, 0xfc, 0x56, 0x27, 0xb0,
- 0x20, 0xb7, 0xc3, 0x81, 0xe9, 0xfa, 0x15, 0xf7, 0x57, 0x80, 0x54, 0xe2, 0x6f, 0xdb, 0xa0, 0x1d,
- 0x22, 0x13, 0xb9, 0x34, 0xe8, 0x13, 0x40, 0x2a, 0x71, 0xaa, 0x88, 0xde, 0x82, 0xf9, 0x57, 0xf6,
- 0x84, 0x86, 0x06, 0x4a, 0x0d, 0x8e, 0x0c, 0xbf, 0x7f, 0xfb, 0xcb, 0xe6, 0xb7, 0xa9, 0x72, 0x75,
- 0xc0, 0x54, 0xca, 0xff, 0x43, 0x83, 0xca, 0xd6, 0xc0, 0x70, 0x86, 0x52, 0xf1, 0xf7, 0x21, 0xcf,
- 0x6f, 0x95, 0x02, 0xc8, 0x79, 0x3f, 0x2c, 0x46, 0xe5, 0xe5, 0x8d, 0x2d, 0x7e, 0x07, 0x15, 0xa3,
- 0xa8, 0xe1, 0xe2, 0x5d, 0xcf, 0x4e, 0xe4, 0xdd, 0xcf, 0x0e, 0xfa, 0x08, 0x72, 0x06, 0x1d, 0xc2,
- 0x42, 0x70, 0x2d, 0x7a, 0x9f, 0x67, 0xd2, 0x58, 0xed, 0xcb, 0xb9, 0xf0, 0x77, 0xa1, 0xac, 0x68,
- 0x40, 0x05, 0xc8, 0xbe, 0x6c, 0x8b, 0x42, 0x75, 0x6b, 0xbb, 0xbb, 0xf7, 0x9a, 0x03, 0x19, 0x35,
- 0x80, 0x9d, 0xb6, 0xdf, 0xce, 0xe0, 0xcf, 0xc5, 0x28, 0x11, 0xef, 0x54, 0x7b, 0xb4, 0x34, 0x7b,
- 0x32, 0xef, 0x64, 0xcf, 0x15, 0x54, 0xc5, 0xf4, 0xa7, 0x0d, 0xdf, 0x4c, 0x5e, 0x4a, 0xf8, 0x56,
- 0x8c, 0xd7, 0x05, 0x23, 0x9e, 0x83, 0xaa, 0x08, 0xe8, 0x62, 0xff, 0xfd, 0x4b, 0x06, 0x6a, 0x92,
- 0x32, 0x2d, 0xe0, 0x2c, 0xb1, 0x32, 0x9e, 0x01, 0x7c, 0xa4, 0x6c, 0x19, 0xf2, 0xfd, 0xb3, 0x63,
- 0xf3, 0x2b, 0xf9, 0x72, 0x40, 0xb4, 0x28, 0x7d, 0xc0, 0xf5, 0xf0, 0x37, 0x74, 0xa2, 0x85, 0xee,
- 0xf1, 0x97, 0x77, 0x7b, 0x56, 0x9f, 0x5c, 0xb1, 0x3a, 0x7a, 0x56, 0x0f, 0x08, 0x0c, 0x44, 0x10,
- 0x6f, 0xf2, 0x58, 0xf1, 0xac, 0xbc, 0xd9, 0x43, 0x4f, 0xa1, 0x4e, 0x9f, 0xb7, 0x46, 0xa3, 0x81,
- 0x49, 0xfa, 0x5c, 0x40, 0x81, 0xf1, 0xc4, 0xe8, 0x54, 0x3b, 0x2b, 0x37, 0xdd, 0x46, 0x91, 0x85,
- 0x2d, 0xd1, 0x42, 0x6b, 0x50, 0xe6, 0xf6, 0xed, 0x59, 0x27, 0x2e, 0x61, 0xaf, 0xb7, 0xb2, 0xba,
- 0x4a, 0xa2, 0xe7, 0x78, 0x6b, 0xec, 0x5d, 0xb4, 0x2d, 0xe3, 0x6c, 0x20, 0xe3, 0x22, 0x4d, 0xe6,
- 0x94, 0xb8, 0x63, 0xba, 0x2a, 0xb5, 0x0d, 0x0b, 0x94, 0x4a, 0x2c, 0xcf, 0xec, 0x29, 0x41, 0x54,
- 0xa6, 0x4a, 0x2d, 0x92, 0x2a, 0x0d, 0xd7, 0x7d, 0x6b, 0x3b, 0x7d, 0xe1, 0x40, 0xbf, 0x8d, 0x77,
- 0xb8, 0xf0, 0x13, 0x37, 0x94, 0x0c, 0x7f, 0x55, 0x29, 0xeb, 0x81, 0x94, 0x97, 0xc4, 0xbb, 0x41,
- 0x0a, 0x7e, 0x06, 0x4b, 0x92, 0x53, 0x40, 0xbe, 0x37, 0x30, 0x77, 0xe0, 0xbe, 0x64, 0xde, 0xbe,
- 0xa0, 0x57, 0xe0, 0x23, 0xa1, 0xf0, 0xd7, 0xb5, 0xf3, 0x05, 0x34, 0x7c, 0x3b, 0xd9, 0x35, 0xc4,
- 0x1e, 0xa8, 0x06, 0x8c, 0x5d, 0xb1, 0x33, 0x4b, 0x3a, 0x7b, 0xa6, 0x34, 0xc7, 0x1e, 0xf8, 0x85,
- 0x07, 0x7d, 0xc6, 0xdb, 0xb0, 0x22, 0x65, 0x88, 0x0b, 0x42, 0x58, 0x48, 0xcc, 0xa0, 0x24, 0x21,
- 0xc2, 0x61, 0x74, 0xe8, 0xcd, 0x6e, 0x57, 0x39, 0xc3, 0xae, 0x65, 0x32, 0x35, 0x45, 0xe6, 0x12,
- 0xdf, 0x11, 0xd4, 0x30, 0x35, 0x2f, 0x09, 0x32, 0x15, 0xa0, 0x92, 0xc5, 0x42, 0x50, 0x72, 0x6c,
- 0x21, 0x62, 0xa2, 0x7f, 0x08, 0xab, 0xbe, 0x11, 0xd4, 0x6f, 0x47, 0xc4, 0x19, 0x9a, 0xae, 0xab,
- 0x80, 0x84, 0x49, 0x13, 0x7f, 0x1f, 0x66, 0x47, 0x44, 0x44, 0xae, 0xf2, 0x26, 0xda, 0xe0, 0x6f,
- 0xf5, 0x37, 0x94, 0xc1, 0xac, 0x1f, 0xf7, 0xe1, 0x81, 0x94, 0xce, 0x3d, 0x9a, 0x28, 0x3e, 0x6a,
- 0x94, 0x84, 0x4e, 0x32, 0x29, 0xd0, 0x49, 0x36, 0x02, 0x5c, 0x7f, 0xc2, 0x1d, 0x29, 0xcf, 0xd6,
- 0x54, 0x19, 0x69, 0x9f, 0xfb, 0xd4, 0x3f, 0x92, 0x53, 0x09, 0x3b, 0x83, 0xc5, 0xf0, 0x49, 0x9e,
- 0x2a, 0x58, 0x2e, 0x42, 0xce, 0xb3, 0x2f, 0x89, 0x0c, 0x95, 0xbc, 0x21, 0x0d, 0xf6, 0x8f, 0xf9,
- 0x54, 0x06, 0x1b, 0x81, 0x30, 0xb6, 0x25, 0xa7, 0xb5, 0x97, 0xae, 0xa6, 0x2c, 0xf1, 0x78, 0x03,
- 0x1f, 0xc2, 0x72, 0x34, 0x4c, 0x4c, 0x65, 0xf2, 0x6b, 0xbe, 0x81, 0x93, 0x22, 0xc9, 0x54, 0x72,
- 0x3f, 0x0d, 0x82, 0x81, 0x12, 0x50, 0xa6, 0x12, 0xa9, 0x43, 0x33, 0x29, 0xbe, 0xfc, 0x26, 0xf6,
- 0xab, 0x1f, 0x6e, 0xa6, 0x12, 0xe6, 0x06, 0xc2, 0xa6, 0x5f, 0xfe, 0x20, 0x46, 0x64, 0x6f, 0x8c,
- 0x11, 0xe2, 0x90, 0x04, 0x51, 0xec, 0x1b, 0xd8, 0x74, 0x42, 0x47, 0x10, 0x40, 0xa7, 0xd5, 0x41,
- 0x73, 0x88, 0xaf, 0x83, 0x35, 0xe4, 0xc6, 0x56, 0xc3, 0xee, 0x54, 0x8b, 0xf1, 0x59, 0x10, 0x3b,
- 0x63, 0x91, 0x79, 0x2a, 0xc1, 0x9f, 0xc3, 0x5a, 0x7a, 0x50, 0x9e, 0x46, 0xf2, 0xd3, 0x16, 0x94,
- 0xfc, 0xb2, 0x55, 0xf9, 0x22, 0xa6, 0x0c, 0x85, 0xc3, 0xce, 0xf1, 0xd1, 0xd6, 0x76, 0x9b, 0x7f,
- 0x12, 0xb3, 0xdd, 0xd1, 0xf5, 0x93, 0xa3, 0x6e, 0x3d, 0xb3, 0xf9, 0xcb, 0x2c, 0x64, 0xf6, 0x5f,
- 0xa3, 0x2f, 0x20, 0xc7, 0xdf, 0x0f, 0xdf, 0xf0, 0x51, 0x40, 0xf3, 0xa6, 0x57, 0xe0, 0xf8, 0xce,
- 0x4f, 0xfe, 0xfb, 0x97, 0x3f, 0xcf, 0xcc, 0xe3, 0x4a, 0x6b, 0xf2, 0x9d, 0xd6, 0xe5, 0xa4, 0xc5,
- 0x72, 0xc3, 0x73, 0xed, 0x29, 0xfa, 0x14, 0xb2, 0x47, 0x63, 0x0f, 0xa5, 0x7e, 0x2c, 0xd0, 0x4c,
- 0x7f, 0x2b, 0x8e, 0x97, 0x98, 0xd0, 0x39, 0x0c, 0x42, 0xe8, 0x68, 0xec, 0x51, 0x91, 0x3f, 0x82,
- 0xb2, 0xfa, 0x4e, 0xfb, 0xd6, 0x2f, 0x08, 0x9a, 0xb7, 0xbf, 0x2f, 0xc7, 0xf7, 0x99, 0xaa, 0x3b,
- 0x18, 0x09, 0x55, 0xfc, 0xad, 0xbb, 0x3a, 0x8b, 0xee, 0x95, 0x85, 0x52, 0xbf, 0x2f, 0x68, 0xa6,
- 0xbf, 0x42, 0x8f, 0xcd, 0xc2, 0xbb, 0xb2, 0xa8, 0xc8, 0x3f, 0x16, 0x6f, 0xcf, 0x7b, 0x1e, 0x7a,
- 0x90, 0xf0, 0xf6, 0x54, 0x7d, 0x4f, 0xd8, 0x5c, 0x4b, 0x67, 0x10, 0x4a, 0xee, 0x31, 0x25, 0xcb,
- 0x78, 0x5e, 0x28, 0xe9, 0xf9, 0x2c, 0xcf, 0xb5, 0xa7, 0x9b, 0x3d, 0xc8, 0x31, 0x0c, 0x1e, 0x7d,
- 0x29, 0x1f, 0x9a, 0x09, 0x2f, 0x23, 0x52, 0x16, 0x3a, 0x84, 0xde, 0xe3, 0x45, 0xa6, 0xa8, 0x86,
- 0x4b, 0x54, 0x11, 0x43, 0xe0, 0x9f, 0x6b, 0x4f, 0xd7, 0xb5, 0x6f, 0x69, 0x9b, 0xff, 0x9c, 0x83,
- 0x1c, 0x03, 0x9f, 0xd0, 0x25, 0x40, 0x80, 0x47, 0x47, 0x67, 0x17, 0x43, 0xb8, 0xa3, 0xb3, 0x8b,
- 0x43, 0xd9, 0xb8, 0xc9, 0x94, 0x2e, 0xe2, 0x39, 0xaa, 0x94, 0x61, 0x5a, 0x2d, 0x06, 0xd3, 0x51,
- 0x3f, 0xfe, 0x95, 0x26, 0xb0, 0x37, 0x7e, 0x96, 0x50, 0x92, 0xb4, 0x10, 0x28, 0x1d, 0xdd, 0x0e,
- 0x09, 0x80, 0x34, 0xfe, 0x1e, 0x53, 0xd8, 0xc2, 0xf5, 0x40, 0xa1, 0xc3, 0x38, 0x9e, 0x6b, 0x4f,
- 0xbf, 0x6c, 0xe0, 0x05, 0xe1, 0xe5, 0x48, 0x0f, 0xfa, 0x31, 0xd4, 0xc2, 0xa0, 0x2b, 0x7a, 0x98,
- 0xa0, 0x2b, 0x8a, 0xdd, 0x36, 0x1f, 0xdd, 0xcc, 0x24, 0x6c, 0x5a, 0x65, 0x36, 0x09, 0xe5, 0x5c,
- 0xf3, 0x25, 0x21, 0x23, 0x83, 0x32, 0x89, 0x35, 0x40, 0x7f, 0xaf, 0x09, 0x4c, 0x3c, 0x40, 0x51,
- 0x51, 0x92, 0xf4, 0x18, 0x46, 0xdb, 0x7c, 0x7c, 0x0b, 0x97, 0x30, 0xe2, 0x0f, 0x98, 0x11, 0xbf,
- 0x8b, 0x17, 0x03, 0x23, 0x3c, 0x73, 0x48, 0x3c, 0x5b, 0x58, 0xf1, 0xe5, 0x3d, 0x7c, 0x27, 0xe4,
- 0x9c, 0x50, 0x6f, 0xb0, 0x58, 0x1c, 0x09, 0x4d, 0x5c, 0xac, 0x10, 0xb2, 0x9a, 0xb8, 0x58, 0x61,
- 0x18, 0x35, 0x69, 0xb1, 0x38, 0xee, 0x99, 0xb4, 0x58, 0x7e, 0xcf, 0x26, 0xfb, 0x7e, 0x85, 0x7f,
- 0xb5, 0x8a, 0x6c, 0x28, 0xf9, 0x28, 0x24, 0x5a, 0x4d, 0x42, 0x84, 0x82, 0xbb, 0x44, 0xf3, 0x41,
- 0x6a, 0xbf, 0x30, 0xe8, 0x3d, 0x66, 0xd0, 0x5d, 0xbc, 0x4c, 0x35, 0x8b, 0x0f, 0x63, 0x5b, 0x1c,
- 0x76, 0x68, 0x19, 0xfd, 0x3e, 0x75, 0xc4, 0x9f, 0x40, 0x45, 0x85, 0x09, 0xd1, 0x7b, 0x89, 0x28,
- 0x94, 0x8a, 0x34, 0x36, 0xf1, 0x4d, 0x2c, 0x42, 0xf3, 0x23, 0xa6, 0x79, 0x15, 0xaf, 0x24, 0x68,
- 0x76, 0x18, 0x6b, 0x48, 0x39, 0x87, 0xf8, 0x92, 0x95, 0x87, 0x10, 0xc4, 0x64, 0xe5, 0x61, 0x84,
- 0xf0, 0x46, 0xe5, 0x63, 0xc6, 0x4a, 0x95, 0xbb, 0x00, 0x01, 0x98, 0x87, 0x12, 0x7d, 0xa9, 0x5c,
- 0xa6, 0xa2, 0xc1, 0x21, 0x8e, 0x03, 0x62, 0xcc, 0xd4, 0x8a, 0x7d, 0x17, 0x51, 0x3b, 0x30, 0x5d,
- 0x1a, 0x24, 0x36, 0xff, 0x3a, 0x0f, 0xe5, 0x57, 0x86, 0x69, 0x79, 0xc4, 0x32, 0xac, 0x1e, 0x41,
- 0x67, 0x90, 0x63, 0x89, 0x32, 0x1a, 0x07, 0x55, 0x7c, 0x2b, 0x1a, 0x07, 0x43, 0xe0, 0x0f, 0x5e,
- 0x63, 0x5a, 0x9b, 0x78, 0x89, 0x6a, 0x1d, 0x06, 0xa2, 0x5b, 0x0c, 0xb3, 0xa1, 0x13, 0x7d, 0x03,
- 0x79, 0xf1, 0x3a, 0x20, 0x22, 0x28, 0x84, 0xe5, 0x34, 0xef, 0x25, 0x77, 0x26, 0x6d, 0x25, 0x55,
- 0x8d, 0xcb, 0xf8, 0xa8, 0x9e, 0x09, 0x40, 0x00, 0x46, 0x46, 0x1d, 0x1a, 0xc3, 0x2e, 0x9b, 0x6b,
- 0xe9, 0x0c, 0x42, 0xe7, 0x63, 0xa6, 0xf3, 0x01, 0x6e, 0x46, 0x75, 0xf6, 0x7d, 0x5e, 0xaa, 0xf7,
- 0x8f, 0x60, 0x76, 0xd7, 0x70, 0x2f, 0x50, 0x24, 0xf5, 0x29, 0x1f, 0x93, 0x34, 0x9b, 0x49, 0x5d,
- 0x42, 0xcb, 0x03, 0xa6, 0x65, 0x85, 0x47, 0x12, 0x55, 0xcb, 0x85, 0xe1, 0xd2, 0x9c, 0x82, 0xfa,
- 0x90, 0xe7, 0xdf, 0x96, 0x44, 0xfd, 0x17, 0xfa, 0x3e, 0x25, 0xea, 0xbf, 0xf0, 0xe7, 0x28, 0xb7,
- 0x6b, 0x19, 0x41, 0x51, 0x7e, 0xcc, 0x81, 0x22, 0x6f, 0xf6, 0x22, 0x1f, 0x7e, 0x34, 0x57, 0xd3,
- 0xba, 0x85, 0xae, 0x87, 0x4c, 0xd7, 0x7d, 0xdc, 0x88, 0xad, 0x95, 0xe0, 0x7c, 0xae, 0x3d, 0xfd,
- 0x96, 0x86, 0x7e, 0x0c, 0x10, 0xe0, 0xb7, 0xb1, 0x03, 0x10, 0x85, 0x82, 0x63, 0x07, 0x20, 0x06,
- 0xfd, 0xe2, 0x0d, 0xa6, 0x77, 0x1d, 0x3f, 0x8c, 0xea, 0xf5, 0x1c, 0xc3, 0x72, 0xdf, 0x10, 0xe7,
- 0x23, 0x8e, 0xd1, 0xb9, 0x17, 0xe6, 0x88, 0x1e, 0x86, 0x7f, 0x9b, 0x83, 0x59, 0x5a, 0x80, 0xd2,
- 0x3c, 0x1d, 0xdc, 0xdb, 0xa3, 0x96, 0xc4, 0xd0, 0xb2, 0xa8, 0x25, 0xf1, 0x2b, 0x7f, 0x38, 0x4f,
- 0xb3, 0x9f, 0x1b, 0x10, 0xc6, 0x40, 0x1d, 0x6d, 0x43, 0x59, 0xb9, 0xd8, 0xa3, 0x04, 0x61, 0x61,
- 0x18, 0x2e, 0x1a, 0xf9, 0x13, 0x50, 0x01, 0x7c, 0x97, 0xe9, 0x5b, 0xe2, 0x91, 0x9f, 0xe9, 0xeb,
- 0x73, 0x0e, 0xaa, 0xf0, 0x2d, 0x54, 0xd4, 0xcb, 0x3f, 0x4a, 0x90, 0x17, 0x81, 0xf8, 0xa2, 0x51,
- 0x2e, 0x09, 0x3b, 0x08, 0x1f, 0x7c, 0xff, 0x27, 0x15, 0x92, 0x8d, 0x2a, 0x1e, 0x40, 0x41, 0xa0,
- 0x01, 0x49, 0xb3, 0x0c, 0xe3, 0x81, 0x49, 0xb3, 0x8c, 0x40, 0x09, 0xe1, 0xda, 0x8e, 0x69, 0xa4,
- 0x17, 0x1e, 0x99, 0x49, 0x84, 0xb6, 0x97, 0xc4, 0x4b, 0xd3, 0x16, 0x80, 0x5b, 0x69, 0xda, 0x94,
- 0xcb, 0x66, 0x9a, 0xb6, 0x73, 0xe2, 0x89, 0xe3, 0x22, 0x2f, 0x71, 0x28, 0x45, 0x98, 0x1a, 0xbd,
- 0xf1, 0x4d, 0x2c, 0x49, 0xa5, 0x77, 0xa0, 0x50, 0x84, 0x6e, 0x74, 0x05, 0x10, 0x60, 0x15, 0xd1,
- 0x7a, 0x2a, 0x11, 0xf0, 0x8c, 0xd6, 0x53, 0xc9, 0x70, 0x47, 0x38, 0x34, 0x04, 0x7a, 0x79, 0xe5,
- 0x4f, 0x35, 0xff, 0x4c, 0x03, 0x14, 0x87, 0x35, 0xd0, 0xb3, 0x64, 0xe9, 0x89, 0x30, 0x6a, 0xf3,
- 0xc3, 0x77, 0x63, 0x4e, 0x8a, 0xf6, 0x81, 0x49, 0x3d, 0xc6, 0x3d, 0x7a, 0x4b, 0x8d, 0xfa, 0x73,
- 0x0d, 0xaa, 0x21, 0x4c, 0x04, 0xbd, 0x9f, 0xb2, 0xa6, 0x11, 0x14, 0xb6, 0xf9, 0xe4, 0x56, 0xbe,
- 0xa4, 0x42, 0x53, 0xd9, 0x01, 0xb2, 0xe2, 0xfe, 0xa9, 0x06, 0xb5, 0x30, 0x86, 0x82, 0x52, 0x64,
- 0xc7, 0x50, 0xdc, 0xe6, 0xfa, 0xed, 0x8c, 0x37, 0x2f, 0x4f, 0x50, 0x6c, 0x0f, 0xa0, 0x20, 0x50,
- 0x97, 0xa4, 0x8d, 0x1f, 0xc6, 0x7f, 0x93, 0x36, 0x7e, 0x04, 0xb2, 0x49, 0xd8, 0xf8, 0x8e, 0x3d,
- 0x20, 0xca, 0x31, 0x13, 0xb0, 0x4c, 0x9a, 0xb6, 0x9b, 0x8f, 0x59, 0x04, 0xd3, 0x49, 0xd3, 0x16,
- 0x1c, 0x33, 0x89, 0xc7, 0xa0, 0x14, 0x61, 0xb7, 0x1c, 0xb3, 0x28, 0x9c, 0x93, 0x70, 0xcc, 0x98,
- 0x42, 0xe5, 0x98, 0x05, 0xc8, 0x49, 0xd2, 0x31, 0x8b, 0xc1, 0xd9, 0x49, 0xc7, 0x2c, 0x0e, 0xbe,
- 0x24, 0xac, 0x23, 0xd3, 0x1b, 0x3a, 0x66, 0x0b, 0x09, 0x20, 0x0b, 0xfa, 0x30, 0xc5, 0x89, 0x89,
- 0x28, 0x79, 0xf3, 0xa3, 0x77, 0xe4, 0x4e, 0xdd, 0xe3, 0xdc, 0xfd, 0x72, 0x8f, 0xff, 0x8d, 0x06,
- 0x8b, 0x49, 0x00, 0x0d, 0x4a, 0xd1, 0x93, 0x82, 0xae, 0x37, 0x37, 0xde, 0x95, 0xfd, 0x66, 0x6f,
- 0xf9, 0xbb, 0xfe, 0x45, 0xfd, 0xdf, 0xbf, 0x5e, 0xd5, 0xfe, 0xf3, 0xeb, 0x55, 0xed, 0x7f, 0xbe,
- 0x5e, 0xd5, 0xfe, 0xf6, 0x7f, 0x57, 0x67, 0xce, 0xf2, 0xec, 0x87, 0x7a, 0xdf, 0xf9, 0xff, 0x00,
- 0x00, 0x00, 0xff, 0xff, 0xc6, 0xc3, 0xa2, 0xb2, 0x2f, 0x38, 0x00, 0x00,
+ 0xd1, 0xd8, 0x4b, 0x3f, 0x1a, 0xcb, 0x90, 0x9b, 0x50, 0xc1, 0xe2, 0x58, 0xf0, 0x06, 0x3b, 0x13,
+ 0xc4, 0x70, 0x89, 0x7f, 0x26, 0x68, 0x03, 0xdd, 0x82, 0xc2, 0xc8, 0x21, 0x93, 0x93, 0xcb, 0x09,
+ 0x53, 0x52, 0xd4, 0xf3, 0xb4, 0xb9, 0x3f, 0x41, 0x6f, 0x41, 0xc5, 0x3c, 0xb7, 0x6c, 0x87, 0x9c,
+ 0x70, 0x59, 0x39, 0xd6, 0x5b, 0xe6, 0x34, 0x66, 0xb7, 0xc2, 0xc2, 0x05, 0xe7, 0x55, 0x96, 0x03,
+ 0x4a, 0xc2, 0x16, 0x94, 0x99, 0xa9, 0x33, 0xb9, 0xef, 0xbd, 0xc0, 0xc6, 0x0c, 0x1b, 0x16, 0x77,
+ 0xa1, 0xb0, 0x1a, 0xff, 0x00, 0xd0, 0x2e, 0x19, 0x10, 0x8f, 0xcc, 0x12, 0x3d, 0x14, 0x9f, 0x64,
+ 0x55, 0x9f, 0xe0, 0x9f, 0x69, 0xb0, 0x14, 0x12, 0x3f, 0xd3, 0xb4, 0x1a, 0x50, 0xe8, 0x33, 0x61,
+ 0xdc, 0x82, 0xac, 0x2e, 0x9b, 0xe8, 0x31, 0x14, 0x85, 0x01, 0x6e, 0x23, 0x9b, 0xb2, 0x69, 0x0a,
+ 0xdc, 0x26, 0x17, 0xff, 0x22, 0x03, 0x25, 0x31, 0xd1, 0xce, 0x08, 0x6d, 0x43, 0xd5, 0xe1, 0x8d,
+ 0x13, 0x36, 0x1f, 0x61, 0x51, 0x33, 0x3d, 0x08, 0x3d, 0x9f, 0xd3, 0x2b, 0x62, 0x08, 0x23, 0xa3,
+ 0xdf, 0x83, 0xb2, 0x14, 0x31, 0x1a, 0x7b, 0xc2, 0xe5, 0x8d, 0xb0, 0x80, 0x60, 0xff, 0x3d, 0x9f,
+ 0xd3, 0x41, 0xb0, 0x1f, 0x8d, 0x3d, 0xd4, 0x85, 0x65, 0x39, 0x98, 0xcf, 0x46, 0x98, 0x91, 0x65,
+ 0x52, 0xd6, 0xc3, 0x52, 0xe2, 0x4b, 0xf5, 0x7c, 0x4e, 0x47, 0x62, 0xbc, 0xd2, 0xa9, 0x9a, 0xe4,
+ 0x5d, 0xf1, 0xe0, 0x1d, 0x33, 0xa9, 0x7b, 0x65, 0xc5, 0x4d, 0xea, 0x5e, 0x59, 0x4f, 0x4b, 0x50,
+ 0x10, 0x2d, 0xfc, 0x2f, 0x19, 0x00, 0xb9, 0x1a, 0x9d, 0x11, 0xda, 0x85, 0x9a, 0x23, 0x5a, 0x21,
+ 0x6f, 0xdd, 0x49, 0xf4, 0x96, 0x58, 0xc4, 0x39, 0xbd, 0x2a, 0x07, 0x71, 0xe3, 0xbe, 0x07, 0x15,
+ 0x5f, 0x4a, 0xe0, 0xb0, 0xdb, 0x09, 0x0e, 0xf3, 0x25, 0x94, 0xe5, 0x00, 0xea, 0xb2, 0x4f, 0x61,
+ 0xc5, 0x1f, 0x9f, 0xe0, 0xb3, 0xb7, 0xa6, 0xf8, 0xcc, 0x17, 0xb8, 0x24, 0x25, 0xa8, 0x5e, 0x53,
+ 0x0d, 0x0b, 0xdc, 0x76, 0x3b, 0xc1, 0x6d, 0x71, 0xc3, 0xa8, 0xe3, 0x80, 0xe6, 0x4b, 0xde, 0xc4,
+ 0xff, 0x97, 0x85, 0xc2, 0x8e, 0x3d, 0x1c, 0x19, 0x0e, 0x5d, 0x8d, 0xbc, 0x43, 0xdc, 0xf1, 0xc0,
+ 0x63, 0xee, 0xaa, 0x6d, 0x3d, 0x08, 0x4b, 0x14, 0x6c, 0xf2, 0xbf, 0xce, 0x58, 0x75, 0x31, 0x84,
+ 0x0e, 0x16, 0xe9, 0x31, 0xf3, 0x06, 0x83, 0x45, 0x72, 0x14, 0x43, 0xe4, 0x41, 0xce, 0x06, 0x07,
+ 0xb9, 0x09, 0x85, 0x09, 0x71, 0x82, 0x94, 0xfe, 0x7c, 0x4e, 0x97, 0x04, 0xf4, 0x1e, 0x2c, 0x44,
+ 0xd3, 0x4b, 0x4e, 0xf0, 0xd4, 0x7a, 0xe1, 0x6c, 0xf4, 0x00, 0x2a, 0xa1, 0x1c, 0x97, 0x17, 0x7c,
+ 0xe5, 0xa1, 0x92, 0xe2, 0x56, 0x65, 0x5c, 0xa5, 0xf9, 0xb8, 0xf2, 0x7c, 0x4e, 0x46, 0xd6, 0x55,
+ 0x19, 0x59, 0x8b, 0x62, 0x94, 0x88, 0xad, 0xa1, 0x20, 0xf3, 0xfd, 0x70, 0x90, 0xc1, 0xdf, 0x87,
+ 0x6a, 0xc8, 0x41, 0x34, 0xef, 0xb4, 0x3f, 0x79, 0xb9, 0x7d, 0xc0, 0x93, 0xd4, 0x33, 0x96, 0x97,
+ 0xf4, 0xba, 0x46, 0x73, 0xdd, 0x41, 0xfb, 0xf8, 0xb8, 0x9e, 0x41, 0x55, 0x28, 0x1d, 0x76, 0xba,
+ 0x27, 0x9c, 0x2b, 0x8b, 0x9f, 0xf9, 0x12, 0x44, 0x92, 0x53, 0x72, 0xdb, 0x9c, 0x92, 0xdb, 0x34,
+ 0x99, 0xdb, 0x32, 0x41, 0x6e, 0x63, 0x69, 0xee, 0xa0, 0xbd, 0x7d, 0xdc, 0xae, 0xcf, 0x3f, 0xad,
+ 0x41, 0x85, 0xfb, 0xf7, 0x64, 0x6c, 0xd1, 0x54, 0xfb, 0x0f, 0x1a, 0x40, 0x70, 0x9a, 0x50, 0x0b,
+ 0x0a, 0x3d, 0xae, 0xa7, 0xa1, 0xb1, 0x60, 0xb4, 0x92, 0xb8, 0x64, 0xba, 0xe4, 0x42, 0xdf, 0x82,
+ 0x82, 0x3b, 0xee, 0xf5, 0x88, 0x2b, 0x53, 0xde, 0xad, 0x68, 0x3c, 0x14, 0xd1, 0x4a, 0x97, 0x7c,
+ 0x74, 0xc8, 0x99, 0x61, 0x0e, 0xc6, 0x2c, 0x01, 0x4e, 0x1f, 0x22, 0xf8, 0xf0, 0xdf, 0x69, 0x50,
+ 0x56, 0x36, 0xef, 0x6f, 0x18, 0x84, 0xef, 0x42, 0x89, 0xd9, 0x40, 0xfa, 0x22, 0x0c, 0x17, 0xf5,
+ 0x80, 0x80, 0x7e, 0x07, 0x4a, 0xf2, 0x04, 0xc8, 0x48, 0xdc, 0x48, 0x16, 0xdb, 0x19, 0xe9, 0x01,
+ 0x2b, 0xde, 0x87, 0x45, 0xe6, 0x95, 0x1e, 0x2d, 0xae, 0xa5, 0x1f, 0xd5, 0xf2, 0x53, 0x8b, 0x94,
+ 0x9f, 0x4d, 0x28, 0x8e, 0x2e, 0xae, 0x5d, 0xb3, 0x67, 0x0c, 0x84, 0x15, 0x7e, 0x1b, 0x7f, 0x0c,
+ 0x48, 0x15, 0x36, 0xcb, 0x74, 0x71, 0x15, 0xca, 0xcf, 0x0d, 0xf7, 0x42, 0x98, 0x84, 0x1f, 0x43,
+ 0x95, 0x36, 0xf7, 0x5f, 0xbd, 0x81, 0x8d, 0xec, 0xe5, 0x40, 0x72, 0xcf, 0xe4, 0x73, 0x04, 0xf3,
+ 0x17, 0x86, 0x7b, 0xc1, 0x26, 0x5a, 0xd5, 0xd9, 0x33, 0x7a, 0x0f, 0xea, 0x3d, 0x3e, 0xc9, 0x93,
+ 0xc8, 0x2b, 0xc3, 0x82, 0xa0, 0xfb, 0x95, 0xe0, 0x67, 0x50, 0xe1, 0x73, 0xf8, 0x6d, 0x1b, 0x81,
+ 0x17, 0x61, 0xe1, 0xd8, 0x32, 0x46, 0xee, 0x85, 0x2d, 0xb3, 0x1b, 0x9d, 0x74, 0x3d, 0xa0, 0xcd,
+ 0xa4, 0xf1, 0x5d, 0x58, 0x70, 0xc8, 0xd0, 0x30, 0x2d, 0xd3, 0x3a, 0x3f, 0x39, 0xbd, 0xf6, 0x88,
+ 0x2b, 0x5e, 0x98, 0x6a, 0x3e, 0xf9, 0x29, 0xa5, 0x52, 0xd3, 0x4e, 0x07, 0xf6, 0xa9, 0x08, 0x73,
+ 0xec, 0x19, 0xff, 0x34, 0x03, 0x95, 0x4f, 0x0d, 0xaf, 0x27, 0x97, 0x0e, 0xed, 0x41, 0xcd, 0x0f,
+ 0x6e, 0x8c, 0x22, 0x6c, 0x89, 0xa4, 0x58, 0x36, 0x46, 0x96, 0xd2, 0x32, 0x3b, 0x56, 0x7b, 0x2a,
+ 0x81, 0x89, 0x32, 0xac, 0x1e, 0x19, 0xf8, 0xa2, 0x32, 0xe9, 0xa2, 0x18, 0xa3, 0x2a, 0x4a, 0x25,
+ 0xa0, 0x0e, 0xd4, 0x47, 0x8e, 0x7d, 0xee, 0x10, 0xd7, 0xf5, 0x85, 0xf1, 0x34, 0x86, 0x13, 0x84,
+ 0x1d, 0x09, 0xd6, 0x40, 0xdc, 0xc2, 0x28, 0x4c, 0x7a, 0xba, 0x10, 0xd4, 0x33, 0x3c, 0x38, 0xfd,
+ 0x57, 0x06, 0x50, 0x7c, 0x52, 0xbf, 0x6e, 0x89, 0xf7, 0x10, 0x6a, 0xae, 0x67, 0x38, 0xb1, 0xcd,
+ 0x56, 0x65, 0x54, 0x3f, 0xe2, 0xbf, 0x0b, 0xbe, 0x41, 0x27, 0x96, 0xed, 0x99, 0x67, 0xd7, 0xa2,
+ 0x4a, 0xae, 0x49, 0xf2, 0x21, 0xa3, 0xa2, 0x36, 0x14, 0xce, 0xcc, 0x81, 0x47, 0x1c, 0xb7, 0x91,
+ 0x5b, 0xcf, 0x6e, 0xd4, 0xb6, 0x1e, 0xdf, 0xb4, 0x0c, 0x9b, 0x1f, 0x31, 0xfe, 0xee, 0xf5, 0x88,
+ 0xe8, 0x72, 0xac, 0x5a, 0x79, 0xe6, 0x43, 0xd5, 0xf8, 0x6d, 0x28, 0xbe, 0xa6, 0x22, 0xe8, 0x5b,
+ 0x76, 0x81, 0x17, 0x8b, 0xac, 0xcd, 0x5f, 0xb2, 0xcf, 0x1c, 0xe3, 0x7c, 0x48, 0x2c, 0x4f, 0xbe,
+ 0x07, 0xca, 0x36, 0x7e, 0x08, 0x10, 0xa8, 0xa1, 0x21, 0xff, 0xb0, 0x73, 0xf4, 0xb2, 0x5b, 0x9f,
+ 0x43, 0x15, 0x28, 0x1e, 0x76, 0x76, 0xdb, 0x07, 0x6d, 0x9a, 0x1f, 0x70, 0x4b, 0xba, 0x34, 0xb4,
+ 0x96, 0xaa, 0x4e, 0x2d, 0xa4, 0x13, 0xaf, 0xc2, 0x72, 0xd2, 0x02, 0xd2, 0x5a, 0xb4, 0x2a, 0x76,
+ 0xe9, 0x4c, 0x47, 0x45, 0x55, 0x9d, 0x09, 0x4f, 0xb7, 0x01, 0x05, 0xbe, 0x7b, 0xfb, 0xa2, 0x38,
+ 0x97, 0x4d, 0xea, 0x08, 0xbe, 0x19, 0x49, 0x5f, 0xac, 0x92, 0xdf, 0x4e, 0x0c, 0x2f, 0xb9, 0xc4,
+ 0xf0, 0x82, 0x1e, 0x40, 0xd5, 0x3f, 0x0d, 0x86, 0x2b, 0x6a, 0x81, 0x92, 0x5e, 0x91, 0x1b, 0x9d,
+ 0xd2, 0x42, 0x4e, 0x2f, 0x84, 0x9d, 0x8e, 0x1e, 0x42, 0x9e, 0x4c, 0x88, 0xe5, 0xb9, 0x8d, 0x32,
+ 0xcb, 0x18, 0x55, 0x59, 0xbb, 0xb7, 0x29, 0x55, 0x17, 0x9d, 0xf8, 0xbb, 0xb0, 0xc8, 0xde, 0x91,
+ 0x9e, 0x39, 0x86, 0xa5, 0xbe, 0xcc, 0x75, 0xbb, 0x07, 0xc2, 0xdd, 0xf4, 0x11, 0xd5, 0x20, 0xb3,
+ 0xb7, 0x2b, 0x9c, 0x90, 0xd9, 0xdb, 0xc5, 0x3f, 0xd6, 0x00, 0xa9, 0xe3, 0x66, 0xf2, 0x73, 0x44,
+ 0xb8, 0x54, 0x9f, 0x0d, 0xd4, 0x2f, 0x43, 0x8e, 0x38, 0x8e, 0xed, 0x30, 0x8f, 0x96, 0x74, 0xde,
+ 0xc0, 0x6f, 0x0b, 0x1b, 0x74, 0x32, 0xb1, 0x2f, 0xfd, 0x33, 0xc8, 0xa5, 0x69, 0xbe, 0xa9, 0xfb,
+ 0xb0, 0x14, 0xe2, 0x9a, 0x29, 0x73, 0x7d, 0x04, 0x0b, 0x4c, 0xd8, 0xce, 0x05, 0xe9, 0x5d, 0x8e,
+ 0x6c, 0xd3, 0x8a, 0xe9, 0xa3, 0x2b, 0x17, 0x04, 0x58, 0x3a, 0x0f, 0x3e, 0xb1, 0x8a, 0x4f, 0xec,
+ 0x76, 0x0f, 0xf0, 0xe7, 0xb0, 0x1a, 0x91, 0x23, 0xcd, 0xff, 0x43, 0x28, 0xf7, 0x7c, 0xa2, 0x2b,
+ 0x6a, 0x9d, 0x7b, 0x61, 0xe3, 0xa2, 0x43, 0xd5, 0x11, 0xb8, 0x03, 0xb7, 0x62, 0xa2, 0x67, 0x9a,
+ 0xf3, 0xbb, 0xb0, 0xc2, 0x04, 0xee, 0x13, 0x32, 0xda, 0x1e, 0x98, 0x93, 0x54, 0x4f, 0x8f, 0xc4,
+ 0xa4, 0x14, 0xc6, 0xaf, 0x77, 0x5f, 0xe0, 0xdf, 0x17, 0x1a, 0xbb, 0xe6, 0x90, 0x74, 0xed, 0x83,
+ 0x74, 0xdb, 0x68, 0x36, 0xbb, 0x24, 0xd7, 0xae, 0x28, 0x6b, 0xd8, 0x33, 0xfe, 0x47, 0x4d, 0xb8,
+ 0x4a, 0x1d, 0xfe, 0x35, 0xef, 0xe4, 0x35, 0x80, 0x73, 0x7a, 0x64, 0x48, 0x9f, 0x76, 0x70, 0x44,
+ 0x45, 0xa1, 0xf8, 0x76, 0xd2, 0xf8, 0x5d, 0x11, 0x76, 0x2e, 0x8b, 0x7d, 0xce, 0xfe, 0xf8, 0x51,
+ 0xee, 0x1e, 0x94, 0x19, 0xe1, 0xd8, 0x33, 0xbc, 0xb1, 0x1b, 0x5b, 0x8c, 0x3f, 0x17, 0xdb, 0x5e,
+ 0x0e, 0x9a, 0x69, 0x5e, 0xdf, 0x82, 0x3c, 0x7b, 0x99, 0x90, 0xa5, 0xf4, 0xed, 0x84, 0xfd, 0xc8,
+ 0xed, 0xd0, 0x05, 0x23, 0xfe, 0xa9, 0x06, 0xf9, 0x17, 0x0c, 0x82, 0x55, 0x4c, 0x9b, 0x97, 0x6b,
+ 0x61, 0x19, 0x43, 0x0e, 0x0c, 0x95, 0x74, 0xf6, 0xcc, 0x4a, 0x4f, 0x42, 0x9c, 0x97, 0xfa, 0x01,
+ 0x2f, 0x71, 0x4b, 0xba, 0xdf, 0xa6, 0x3e, 0xeb, 0x0d, 0x4c, 0x62, 0x79, 0xac, 0x77, 0x9e, 0xf5,
+ 0x2a, 0x14, 0x5a, 0x3d, 0x9b, 0xee, 0x01, 0x31, 0x1c, 0x4b, 0x80, 0xa6, 0x45, 0x3d, 0x20, 0xe0,
+ 0x03, 0xa8, 0x73, 0x3b, 0xb6, 0xfb, 0x7d, 0xa5, 0xc0, 0xf4, 0xb5, 0x69, 0x11, 0x6d, 0x21, 0x69,
+ 0x99, 0xa8, 0xb4, 0x7f, 0xd2, 0x60, 0x51, 0x11, 0x37, 0x93, 0x57, 0xdf, 0x87, 0x3c, 0x07, 0xa9,
+ 0x45, 0xa5, 0xb3, 0x1c, 0x1e, 0xc5, 0xd5, 0xe8, 0x82, 0x07, 0x6d, 0x42, 0x81, 0x3f, 0xc9, 0x77,
+ 0x80, 0x64, 0x76, 0xc9, 0x84, 0x1f, 0xc2, 0x92, 0x20, 0x91, 0xa1, 0x9d, 0x74, 0x30, 0xd8, 0x62,
+ 0xe0, 0x3f, 0x85, 0xe5, 0x30, 0xdb, 0x4c, 0x53, 0x52, 0x8c, 0xcc, 0xbc, 0x89, 0x91, 0xdb, 0xd2,
+ 0xc8, 0x97, 0xa3, 0xbe, 0x52, 0x47, 0x45, 0x77, 0x8c, 0xba, 0x5e, 0x99, 0xf0, 0x7a, 0x05, 0x13,
+ 0x90, 0x22, 0xbe, 0xd1, 0x09, 0x2c, 0xc9, 0xed, 0x70, 0x60, 0xba, 0x7e, 0xb9, 0xfe, 0x25, 0x20,
+ 0x95, 0xf8, 0x8d, 0x1a, 0xf4, 0x8e, 0x74, 0xc7, 0x91, 0x63, 0x0f, 0xed, 0x54, 0x97, 0xe2, 0x3f,
+ 0x83, 0x95, 0x08, 0xdf, 0x37, 0xed, 0xb7, 0x5d, 0x22, 0x8b, 0x15, 0xe9, 0xb7, 0x8f, 0x01, 0xa9,
+ 0xc4, 0x99, 0xb2, 0x56, 0x0b, 0x16, 0x5f, 0xd8, 0x13, 0x1a, 0xfe, 0x28, 0x35, 0x38, 0xf7, 0x1c,
+ 0x63, 0xf0, 0x5d, 0xe1, 0xb7, 0xa9, 0x72, 0x75, 0xc0, 0x4c, 0xca, 0xff, 0x43, 0x83, 0xca, 0xf6,
+ 0xc0, 0x70, 0x86, 0x52, 0xf1, 0xf7, 0x20, 0xcf, 0xdf, 0x9c, 0x05, 0x58, 0xf5, 0x4e, 0x58, 0x8c,
+ 0xca, 0xcb, 0x1b, 0xdb, 0xfc, 0x3d, 0x5b, 0x8c, 0xa2, 0x86, 0x8b, 0xef, 0x59, 0xbb, 0x91, 0xef,
+ 0x5b, 0xbb, 0xe8, 0x03, 0xc8, 0x19, 0x74, 0x08, 0x4b, 0x33, 0xb5, 0x28, 0x66, 0xc1, 0xa4, 0xb1,
+ 0xfa, 0x9e, 0x73, 0xe1, 0xef, 0x40, 0x59, 0xd1, 0x80, 0x0a, 0x90, 0x7d, 0xd6, 0x16, 0xc5, 0xf8,
+ 0xf6, 0x4e, 0x77, 0xef, 0x15, 0x07, 0x6b, 0x6a, 0x00, 0xbb, 0x6d, 0xbf, 0x9d, 0xc1, 0x9f, 0x89,
+ 0x51, 0x22, 0xa4, 0xab, 0xf6, 0x68, 0x69, 0xf6, 0x64, 0xde, 0xc8, 0x9e, 0x2b, 0xa8, 0x8a, 0xe9,
+ 0xcf, 0x9a, 0xa2, 0x98, 0xbc, 0x94, 0x14, 0xa5, 0x18, 0xaf, 0x0b, 0x46, 0xbc, 0x00, 0x55, 0x91,
+ 0xb4, 0xc4, 0xfe, 0xfb, 0xf7, 0x0c, 0xd4, 0x24, 0x65, 0x56, 0x50, 0x5d, 0xe2, 0x81, 0x3c, 0xc9,
+ 0xf9, 0x68, 0xe0, 0x2a, 0xe4, 0xfb, 0xa7, 0xc7, 0xe6, 0x97, 0xf2, 0x03, 0x88, 0x68, 0x51, 0xfa,
+ 0x80, 0xeb, 0xe1, 0x5f, 0x21, 0x45, 0x8b, 0x66, 0x23, 0xc7, 0x38, 0xf3, 0xf6, 0xac, 0x3e, 0xb9,
+ 0x62, 0xb9, 0x6d, 0x5e, 0x0f, 0x08, 0x0c, 0x28, 0x11, 0x5f, 0x2b, 0xd9, 0x0b, 0x82, 0xf2, 0xf5,
+ 0x12, 0x3d, 0x82, 0x3a, 0x7d, 0xde, 0x1e, 0x8d, 0x06, 0x26, 0xe9, 0x73, 0x01, 0x05, 0xc6, 0x13,
+ 0xa3, 0x53, 0xed, 0xac, 0xa4, 0x76, 0x1b, 0x45, 0x16, 0x5d, 0x45, 0x0b, 0xad, 0x43, 0x99, 0xdb,
+ 0xb7, 0x67, 0xbd, 0x74, 0x09, 0xfb, 0x84, 0x97, 0xd5, 0x55, 0x52, 0x38, 0x5b, 0x42, 0x34, 0x5b,
+ 0x2e, 0xc1, 0xe2, 0xf6, 0xd8, 0xbb, 0x68, 0x5b, 0xc6, 0xe9, 0x40, 0x46, 0x22, 0x5a, 0xce, 0x50,
+ 0xe2, 0xae, 0xe9, 0xaa, 0xd4, 0x36, 0x2c, 0x51, 0x2a, 0xb1, 0x3c, 0xb3, 0xa7, 0x64, 0x02, 0x59,
+ 0x2b, 0x68, 0x91, 0x5a, 0xc1, 0x70, 0xdd, 0xd7, 0xb6, 0xd3, 0x17, 0xee, 0xf5, 0xdb, 0x78, 0xc2,
+ 0x85, 0xbf, 0x74, 0x43, 0xf9, 0xfe, 0xd7, 0x94, 0x82, 0x3e, 0x84, 0x82, 0x3d, 0x62, 0x9f, 0xa4,
+ 0x05, 0x6e, 0xb0, 0xba, 0xc9, 0x3f, 0x62, 0x6f, 0x0a, 0xc1, 0x1d, 0xde, 0xab, 0x4b, 0x36, 0xbc,
+ 0x11, 0xe8, 0x7d, 0x46, 0xbc, 0x29, 0x7a, 0xf1, 0x63, 0x58, 0x91, 0x9c, 0x02, 0x26, 0x9f, 0xc2,
+ 0xdc, 0x81, 0x7b, 0x92, 0x79, 0xe7, 0xc2, 0xb0, 0xce, 0xc9, 0x91, 0x30, 0xf1, 0x37, 0xf5, 0xcf,
+ 0x53, 0x68, 0xf8, 0x76, 0xb2, 0x57, 0x37, 0x7b, 0xa0, 0x1a, 0x30, 0x76, 0xc5, 0x4e, 0x2f, 0xe9,
+ 0xec, 0x99, 0xd2, 0x1c, 0x7b, 0xe0, 0xd7, 0x6a, 0xf4, 0x19, 0xef, 0xc0, 0x6d, 0x29, 0x43, 0xbc,
+ 0x54, 0x85, 0x85, 0xc4, 0x0c, 0x4a, 0x12, 0x22, 0x1c, 0x46, 0x87, 0x4e, 0x5f, 0x28, 0x95, 0x33,
+ 0xec, 0x5a, 0x26, 0x53, 0x53, 0x64, 0xae, 0xf0, 0x3d, 0x44, 0x0d, 0x53, 0xd3, 0xb1, 0x20, 0x53,
+ 0x01, 0x2a, 0x59, 0x2c, 0x04, 0x25, 0xc7, 0x16, 0x22, 0x26, 0xfa, 0x07, 0xb0, 0xe6, 0x1b, 0x41,
+ 0xfd, 0x76, 0x44, 0x9c, 0xa1, 0xe9, 0xba, 0x0a, 0xb0, 0x9a, 0x34, 0xf1, 0x77, 0x60, 0x7e, 0x44,
+ 0x44, 0x24, 0x2c, 0x6f, 0x21, 0xb9, 0x89, 0x94, 0xc1, 0xac, 0x1f, 0xf7, 0xe1, 0xbe, 0x94, 0xce,
+ 0x3d, 0x9a, 0x28, 0x3e, 0x6a, 0x94, 0x84, 0x9b, 0x32, 0x29, 0x70, 0x53, 0x36, 0x02, 0xf6, 0x7f,
+ 0xcc, 0x1d, 0x29, 0x4f, 0xe3, 0x4c, 0x19, 0x6e, 0x9f, 0xfb, 0xd4, 0x3f, 0xc4, 0x33, 0x09, 0x3b,
+ 0x85, 0xe5, 0xf0, 0xd9, 0x9f, 0x29, 0xf8, 0x2e, 0x43, 0xce, 0xb3, 0x2f, 0x89, 0x0c, 0xbd, 0xbc,
+ 0x21, 0x0d, 0xf6, 0x03, 0xc3, 0x4c, 0x06, 0x1b, 0x81, 0x30, 0xb6, 0x25, 0x67, 0xb5, 0x97, 0xae,
+ 0xa6, 0xac, 0x6c, 0x79, 0x03, 0x1f, 0xc2, 0x6a, 0x34, 0x4c, 0xcc, 0x64, 0xf2, 0x2b, 0xbe, 0x81,
+ 0x93, 0x22, 0xc9, 0x4c, 0x72, 0x3f, 0x09, 0x82, 0x81, 0x12, 0x50, 0x66, 0x12, 0xa9, 0x43, 0x33,
+ 0x29, 0xbe, 0xfc, 0x36, 0xf6, 0xab, 0x1f, 0x6e, 0x66, 0x12, 0xe6, 0x06, 0xc2, 0x66, 0x5f, 0xfe,
+ 0x20, 0x46, 0x64, 0xa7, 0xc6, 0x08, 0x71, 0x48, 0x82, 0x28, 0xf6, 0x35, 0x6c, 0x3a, 0xa1, 0x23,
+ 0x08, 0xa0, 0xb3, 0xea, 0xa0, 0x39, 0xc4, 0xd7, 0xc1, 0x1a, 0x72, 0x63, 0xab, 0x61, 0x77, 0xa6,
+ 0xc5, 0xf8, 0x34, 0x88, 0x9d, 0xb1, 0xc8, 0x3c, 0x93, 0xe0, 0xcf, 0x60, 0x3d, 0x3d, 0x28, 0xcf,
+ 0x22, 0xf9, 0x51, 0x0b, 0x4a, 0x7e, 0x19, 0xac, 0xdc, 0x22, 0x2a, 0x43, 0xe1, 0xb0, 0x73, 0x7c,
+ 0xb4, 0xbd, 0xd3, 0xe6, 0xd7, 0x88, 0x76, 0x3a, 0xba, 0xfe, 0xf2, 0xa8, 0x5b, 0xcf, 0x6c, 0xfd,
+ 0x32, 0x0b, 0x99, 0xfd, 0x57, 0xe8, 0x73, 0xc8, 0xf1, 0x6f, 0xea, 0x53, 0x2e, 0x52, 0x34, 0xa7,
+ 0x5d, 0x1b, 0xc0, 0xb7, 0x7e, 0xfc, 0xdf, 0xbf, 0xfc, 0x79, 0x66, 0x11, 0x57, 0x5a, 0x93, 0x6f,
+ 0xb7, 0x2e, 0x27, 0x2d, 0x96, 0x1b, 0x9e, 0x68, 0x8f, 0xd0, 0x27, 0x90, 0x3d, 0x1a, 0x7b, 0x28,
+ 0xf5, 0x82, 0x45, 0x33, 0xfd, 0x26, 0x01, 0x5e, 0x61, 0x42, 0x17, 0x30, 0x08, 0xa1, 0xa3, 0xb1,
+ 0x47, 0x45, 0xfe, 0x10, 0xca, 0xea, 0x3d, 0x80, 0x1b, 0x6f, 0x5d, 0x34, 0x6f, 0xbe, 0x63, 0x80,
+ 0xef, 0x31, 0x55, 0xb7, 0x30, 0x12, 0xaa, 0xf8, 0x4d, 0x05, 0x75, 0x16, 0xdd, 0x2b, 0x0b, 0xa5,
+ 0xde, 0xc9, 0x68, 0xa6, 0x5f, 0x3b, 0x88, 0xcd, 0xc2, 0xbb, 0xb2, 0xa8, 0xc8, 0x3f, 0x16, 0x37,
+ 0x0e, 0x7a, 0x1e, 0xba, 0x9f, 0xf0, 0xc5, 0x59, 0xfd, 0xb6, 0xda, 0x5c, 0x4f, 0x67, 0x10, 0x4a,
+ 0xee, 0x32, 0x25, 0xab, 0x78, 0x51, 0x28, 0xe9, 0xf9, 0x2c, 0x4f, 0xb4, 0x47, 0x5b, 0x3d, 0xc8,
+ 0xb1, 0xef, 0x16, 0xe8, 0x0b, 0xf9, 0xd0, 0x4c, 0xf8, 0x80, 0x93, 0xb2, 0xd0, 0xa1, 0x2f, 0x1e,
+ 0x78, 0x99, 0x29, 0xaa, 0xe1, 0x12, 0x55, 0xc4, 0xbe, 0x5a, 0x3c, 0xd1, 0x1e, 0x6d, 0x68, 0x1f,
+ 0x6a, 0x5b, 0xff, 0x9c, 0x83, 0x1c, 0x03, 0xec, 0xd0, 0x25, 0x40, 0x80, 0xe1, 0x47, 0x67, 0x17,
+ 0xfb, 0x2a, 0x10, 0x9d, 0x5d, 0x1c, 0xfe, 0xc7, 0x4d, 0xa6, 0x74, 0x19, 0x2f, 0x50, 0xa5, 0x0c,
+ 0x07, 0x6c, 0x31, 0x68, 0x93, 0xfa, 0xf1, 0xaf, 0x34, 0x81, 0x57, 0xf2, 0xb3, 0x84, 0x92, 0xa4,
+ 0x85, 0x80, 0xfc, 0xe8, 0x76, 0x48, 0x00, 0xf1, 0xf1, 0x77, 0x99, 0xc2, 0x16, 0xae, 0x07, 0x0a,
+ 0x1d, 0xc6, 0xf1, 0x44, 0x7b, 0xf4, 0x45, 0x03, 0x2f, 0x09, 0x2f, 0x47, 0x7a, 0xd0, 0x8f, 0xa0,
+ 0x16, 0x06, 0xaa, 0xd1, 0x83, 0x04, 0x5d, 0x51, 0xbc, 0xbb, 0xf9, 0xf6, 0x74, 0x26, 0x61, 0xd3,
+ 0x1a, 0xb3, 0x49, 0x28, 0xe7, 0x9a, 0x2f, 0x09, 0x19, 0x19, 0x94, 0x49, 0xac, 0x01, 0xfa, 0x7b,
+ 0x4d, 0x7c, 0x47, 0x08, 0x90, 0x67, 0x94, 0x24, 0x3d, 0x86, 0x6b, 0x37, 0x1f, 0xde, 0xc0, 0x25,
+ 0x8c, 0xf8, 0x03, 0x66, 0xc4, 0xef, 0xe2, 0xe5, 0xc0, 0x08, 0xcf, 0x1c, 0x12, 0xcf, 0x16, 0x56,
+ 0x7c, 0x71, 0x17, 0xdf, 0x0a, 0x39, 0x27, 0xd4, 0x1b, 0x2c, 0x16, 0x47, 0x8f, 0x13, 0x17, 0x2b,
+ 0x84, 0x46, 0x27, 0x2e, 0x56, 0x18, 0x7a, 0x4e, 0x5a, 0x2c, 0x8e, 0x15, 0x27, 0x2d, 0x96, 0xdf,
+ 0xb3, 0xf5, 0xff, 0xf3, 0x50, 0xd8, 0xe1, 0x37, 0x7d, 0x91, 0x0d, 0x25, 0x1f, 0x7c, 0x45, 0x6b,
+ 0x49, 0x08, 0x53, 0xf0, 0x2e, 0xd1, 0xbc, 0x9f, 0xda, 0x2f, 0x0c, 0x7a, 0x8b, 0x19, 0x74, 0x07,
+ 0xaf, 0x52, 0xcd, 0xe2, 0x32, 0x71, 0x8b, 0xc3, 0x18, 0x2d, 0xa3, 0xdf, 0xa7, 0x8e, 0xf8, 0x13,
+ 0xa8, 0xa8, 0xe8, 0x28, 0x7a, 0x2b, 0x11, 0xd5, 0x52, 0x01, 0xd6, 0x26, 0x9e, 0xc6, 0x22, 0x34,
+ 0xbf, 0xcd, 0x34, 0xaf, 0xe1, 0xdb, 0x09, 0x9a, 0x1d, 0xc6, 0x1a, 0x52, 0xce, 0x91, 0xcd, 0x64,
+ 0xe5, 0x21, 0xe0, 0x34, 0x59, 0x79, 0x18, 0x18, 0x9d, 0xaa, 0x7c, 0xcc, 0x58, 0xa9, 0x72, 0x17,
+ 0x20, 0xc0, 0x30, 0x51, 0xa2, 0x2f, 0x95, 0x97, 0xa9, 0x68, 0x70, 0x88, 0xc3, 0x9f, 0x18, 0x33,
+ 0xb5, 0x62, 0xdf, 0x45, 0xd4, 0x0e, 0x4c, 0xd7, 0xe3, 0x07, 0xb3, 0x1a, 0x02, 0x25, 0x51, 0xe2,
+ 0x7c, 0xc2, 0xc8, 0x66, 0xf3, 0xc1, 0x54, 0x1e, 0xa1, 0xfd, 0x21, 0xd3, 0x7e, 0x1f, 0x37, 0x13,
+ 0xb4, 0x8f, 0x38, 0x2f, 0xdd, 0x6c, 0x7f, 0x9d, 0x87, 0xf2, 0x0b, 0xc3, 0xb4, 0x3c, 0x62, 0x19,
+ 0x56, 0x8f, 0xa0, 0x53, 0xc8, 0xb1, 0x4c, 0x1d, 0x0d, 0xc4, 0x2a, 0x60, 0x17, 0x0d, 0xc4, 0x21,
+ 0x34, 0x0b, 0xaf, 0x33, 0xc5, 0x4d, 0xbc, 0x42, 0x15, 0x0f, 0x03, 0xd1, 0x2d, 0x06, 0x42, 0xd1,
+ 0x49, 0x9f, 0x41, 0x5e, 0x7c, 0xc3, 0x89, 0x08, 0x0a, 0x81, 0x53, 0xcd, 0xbb, 0xc9, 0x9d, 0x49,
+ 0x7b, 0x59, 0x55, 0xe3, 0x32, 0x3e, 0xaa, 0x67, 0x02, 0x10, 0xa0, 0xab, 0xd1, 0x15, 0x8d, 0x81,
+ 0xb1, 0xcd, 0xf5, 0x74, 0x86, 0x24, 0x9f, 0xaa, 0x3a, 0xfb, 0x3e, 0x2f, 0xd5, 0xfb, 0x47, 0x30,
+ 0xff, 0xdc, 0x70, 0x2f, 0x50, 0x24, 0xf7, 0x2a, 0x37, 0x80, 0x9a, 0xcd, 0xa4, 0x2e, 0xa1, 0xe5,
+ 0x3e, 0xd3, 0x72, 0x9b, 0x87, 0x32, 0x55, 0xcb, 0x85, 0xe1, 0xd2, 0xa4, 0x86, 0xfa, 0x90, 0xe7,
+ 0x17, 0x82, 0xa2, 0xfe, 0x0b, 0x5d, 0x2a, 0x8a, 0xfa, 0x2f, 0x7c, 0x87, 0xe8, 0x66, 0x2d, 0x23,
+ 0x28, 0xca, 0x1b, 0x38, 0x28, 0xf2, 0x39, 0x36, 0x72, 0x5b, 0xa7, 0xb9, 0x96, 0xd6, 0x2d, 0x74,
+ 0x3d, 0x60, 0xba, 0xee, 0xe1, 0x46, 0x6c, 0xad, 0x04, 0xe7, 0x13, 0xed, 0xd1, 0x87, 0x1a, 0xfa,
+ 0x11, 0x40, 0x00, 0x48, 0xc7, 0x4e, 0x60, 0x14, 0xdb, 0x8e, 0x9d, 0xc0, 0x18, 0x96, 0x8d, 0x37,
+ 0x99, 0xde, 0x0d, 0xfc, 0x20, 0xaa, 0xd7, 0x73, 0x0c, 0xcb, 0x3d, 0x23, 0xce, 0x07, 0x1c, 0x74,
+ 0x74, 0x2f, 0xcc, 0x11, 0x3d, 0x0c, 0xff, 0xba, 0x00, 0xf3, 0xb4, 0x02, 0xa6, 0x85, 0x42, 0x00,
+ 0x1c, 0x44, 0x2d, 0x89, 0x01, 0x7c, 0x51, 0x4b, 0xe2, 0x98, 0x43, 0xb8, 0x50, 0x60, 0xbf, 0x11,
+ 0x21, 0x8c, 0x81, 0x3a, 0xda, 0x86, 0xb2, 0x82, 0x2c, 0xa0, 0x04, 0x61, 0x61, 0xe4, 0x30, 0x9a,
+ 0x7a, 0x12, 0x60, 0x09, 0x7c, 0x87, 0xe9, 0x5b, 0xe1, 0xa9, 0x87, 0xe9, 0xeb, 0x73, 0x0e, 0xaa,
+ 0xf0, 0x35, 0x54, 0x54, 0xf4, 0x01, 0x25, 0xc8, 0x8b, 0xa0, 0x92, 0xd1, 0x30, 0x9b, 0x04, 0x5e,
+ 0x84, 0x0f, 0xbe, 0xff, 0x3b, 0x18, 0xc9, 0x46, 0x15, 0x0f, 0xa0, 0x20, 0xe0, 0x88, 0xa4, 0x59,
+ 0x86, 0x21, 0xcc, 0xa4, 0x59, 0x46, 0xb0, 0x8c, 0x70, 0x71, 0xc9, 0x34, 0xd2, 0x37, 0x2e, 0x99,
+ 0xca, 0x84, 0xb6, 0x67, 0xc4, 0x4b, 0xd3, 0x16, 0xa0, 0x6b, 0x69, 0xda, 0x94, 0xb7, 0xdd, 0x34,
+ 0x6d, 0xe7, 0xc4, 0x13, 0xc7, 0x45, 0xbe, 0x45, 0xa2, 0x14, 0x61, 0x6a, 0xfa, 0xc0, 0xd3, 0x58,
+ 0x92, 0x6a, 0xff, 0x40, 0xa1, 0xcc, 0x1d, 0x57, 0x00, 0x01, 0x58, 0x12, 0x2d, 0xe8, 0x12, 0x11,
+ 0xd7, 0x68, 0x41, 0x97, 0x8c, 0xb7, 0x84, 0x43, 0x43, 0xa0, 0x97, 0xbf, 0x7a, 0x50, 0xcd, 0x3f,
+ 0xd3, 0x00, 0xc5, 0x71, 0x15, 0xf4, 0x38, 0x59, 0x7a, 0x22, 0x8e, 0xdb, 0x7c, 0xff, 0xcd, 0x98,
+ 0x93, 0xa2, 0x7d, 0x60, 0x52, 0x8f, 0x71, 0x8f, 0x5e, 0x53, 0xa3, 0xfe, 0x42, 0x83, 0x6a, 0x08,
+ 0x94, 0x41, 0xef, 0xa4, 0xac, 0x69, 0x04, 0x06, 0x6e, 0xbe, 0x7b, 0x23, 0x5f, 0x52, 0xa5, 0xab,
+ 0xec, 0x00, 0x59, 0xf2, 0xff, 0x44, 0x83, 0x5a, 0x18, 0xc4, 0x41, 0x29, 0xb2, 0x63, 0x30, 0x72,
+ 0x73, 0xe3, 0x66, 0xc6, 0xe9, 0xcb, 0x13, 0x54, 0xfb, 0x03, 0x28, 0x08, 0xd8, 0x27, 0x69, 0xe3,
+ 0x87, 0x01, 0xe8, 0xa4, 0x8d, 0x1f, 0xc1, 0x8c, 0x12, 0x36, 0xbe, 0x63, 0x0f, 0x88, 0x72, 0xcc,
+ 0x04, 0x2e, 0x94, 0xa6, 0x6d, 0xfa, 0x31, 0x8b, 0x80, 0x4a, 0x69, 0xda, 0x82, 0x63, 0x26, 0x01,
+ 0x21, 0x94, 0x22, 0xec, 0x86, 0x63, 0x16, 0xc5, 0x93, 0x12, 0x8e, 0x19, 0x53, 0xa8, 0x1c, 0xb3,
+ 0x00, 0xba, 0x49, 0x3a, 0x66, 0x31, 0x3c, 0x3d, 0xe9, 0x98, 0xc5, 0xd1, 0x9f, 0x84, 0x75, 0x64,
+ 0x7a, 0x43, 0xc7, 0x6c, 0x29, 0x01, 0xe5, 0x41, 0xef, 0xa7, 0x38, 0x31, 0x11, 0xa6, 0x6f, 0x7e,
+ 0xf0, 0x86, 0xdc, 0xa9, 0x7b, 0x9c, 0xbb, 0x5f, 0xee, 0xf1, 0xbf, 0xd1, 0x60, 0x39, 0x09, 0x21,
+ 0x42, 0x29, 0x7a, 0x52, 0xe0, 0xfd, 0xe6, 0xe6, 0x9b, 0xb2, 0x4f, 0xf7, 0x96, 0xbf, 0xeb, 0x9f,
+ 0xd6, 0xff, 0xed, 0xab, 0x35, 0xed, 0x3f, 0xbf, 0x5a, 0xd3, 0xfe, 0xe7, 0xab, 0x35, 0xed, 0x6f,
+ 0xff, 0x77, 0x6d, 0xee, 0x34, 0xcf, 0x7e, 0x5d, 0xf9, 0xed, 0x5f, 0x05, 0x00, 0x00, 0xff, 0xff,
+ 0x52, 0x4e, 0xd7, 0x33, 0xe4, 0x39, 0x00, 0x00,
}
diff --git a/vendor/go.etcd.io/etcd/etcdserver/etcdserverpb/rpc.proto b/vendor/go.etcd.io/etcd/etcdserver/etcdserverpb/rpc.proto
index 8060ca0..423eaba 100644
--- a/vendor/go.etcd.io/etcd/etcdserver/etcdserverpb/rpc.proto
+++ b/vendor/go.etcd.io/etcd/etcdserver/etcdserverpb/rpc.proto
@@ -165,6 +165,14 @@
body: "*"
};
}
+
+ // MemberPromote promotes a member from raft learner (non-voting) to raft voting member.
+ rpc MemberPromote(MemberPromoteRequest) returns (MemberPromoteResponse) {
+ option (google.api.http) = {
+ post: "/v3/cluster/member/promote"
+ body: "*"
+ };
+ }
}
service Maintenance {
@@ -256,7 +264,7 @@
};
}
- // UserAdd adds a new user.
+ // UserAdd adds a new user. User name cannot be empty.
rpc UserAdd(AuthUserAddRequest) returns (AuthUserAddResponse) {
option (google.api.http) = {
post: "/v3/auth/user/add"
@@ -312,7 +320,7 @@
};
}
- // RoleAdd adds a new role.
+ // RoleAdd adds a new role. Role name cannot be empty.
rpc RoleAdd(AuthRoleAddRequest) returns (AuthRoleAddResponse) {
option (google.api.http) = {
post: "/v3/auth/role/add"
@@ -846,11 +854,15 @@
repeated string peerURLs = 3;
// clientURLs is the list of URLs the member exposes to clients for communication. If the member is not started, clientURLs will be empty.
repeated string clientURLs = 4;
+ // isLearner indicates if the member is raft learner.
+ bool isLearner = 5;
}
message MemberAddRequest {
// peerURLs is the list of URLs the added member will use to communicate with the cluster.
repeated string peerURLs = 1;
+ // isLearner indicates if the added member is raft learner.
+ bool isLearner = 2;
}
message MemberAddResponse {
@@ -894,6 +906,17 @@
repeated Member members = 2;
}
+message MemberPromoteRequest {
+ // ID is the member ID of the member to promote.
+ uint64 ID = 1;
+}
+
+message MemberPromoteResponse {
+ ResponseHeader header = 1;
+ // members is a list of all members after promoting the member.
+ repeated Member members = 2;
+}
+
message DefragmentRequest {
}
@@ -967,6 +990,8 @@
repeated string errors = 8;
// dbSizeInUse is the size of the backend database logically in use, in bytes, of the responding member.
int64 dbSizeInUse = 9;
+ // isLearner indicates if the member is raft learner.
+ bool isLearner = 10;
}
message AuthEnableRequest {
@@ -983,6 +1008,7 @@
message AuthUserAddRequest {
string name = 1;
string password = 2;
+ authpb.UserAddOptions options = 3;
}
message AuthUserGetRequest {
diff --git a/vendor/go.etcd.io/etcd/pkg/logutil/log_level.go b/vendor/go.etcd.io/etcd/pkg/logutil/log_level.go
new file mode 100644
index 0000000..d57e173
--- /dev/null
+++ b/vendor/go.etcd.io/etcd/pkg/logutil/log_level.go
@@ -0,0 +1,70 @@
+// Copyright 2019 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package logutil
+
+import (
+ "fmt"
+
+ "github.com/coreos/pkg/capnslog"
+ "go.uber.org/zap"
+ "go.uber.org/zap/zapcore"
+)
+
+var DefaultLogLevel = "info"
+
+// ConvertToZapLevel converts log level string to zapcore.Level.
+func ConvertToZapLevel(lvl string) zapcore.Level {
+ switch lvl {
+ case "debug":
+ return zap.DebugLevel
+ case "info":
+ return zap.InfoLevel
+ case "warn":
+ return zap.WarnLevel
+ case "error":
+ return zap.ErrorLevel
+ case "dpanic":
+ return zap.DPanicLevel
+ case "panic":
+ return zap.PanicLevel
+ case "fatal":
+ return zap.FatalLevel
+ default:
+ panic(fmt.Sprintf("unknown level %q", lvl))
+ }
+}
+
+// ConvertToCapnslogLogLevel convert log level string to capnslog.LogLevel.
+// TODO: deprecate this in 3.5
+func ConvertToCapnslogLogLevel(lvl string) capnslog.LogLevel {
+ switch lvl {
+ case "debug":
+ return capnslog.DEBUG
+ case "info":
+ return capnslog.INFO
+ case "warn":
+ return capnslog.WARNING
+ case "error":
+ return capnslog.ERROR
+ case "dpanic":
+ return capnslog.CRITICAL
+ case "panic":
+ return capnslog.CRITICAL
+ case "fatal":
+ return capnslog.CRITICAL
+ default:
+ panic(fmt.Sprintf("unknown level %q", lvl))
+ }
+}
diff --git a/vendor/go.etcd.io/etcd/pkg/logutil/zap.go b/vendor/go.etcd.io/etcd/pkg/logutil/zap.go
index 313d914..8fc6e03 100644
--- a/vendor/go.etcd.io/etcd/pkg/logutil/zap.go
+++ b/vendor/go.etcd.io/etcd/pkg/logutil/zap.go
@@ -23,7 +23,7 @@
// DefaultZapLoggerConfig defines default zap logger configuration.
var DefaultZapLoggerConfig = zap.Config{
- Level: zap.NewAtomicLevelAt(zap.InfoLevel),
+ Level: zap.NewAtomicLevelAt(ConvertToZapLevel(DefaultLogLevel)),
Development: false,
Sampling: &zap.SamplingConfig{
@@ -53,15 +53,12 @@
ErrorOutputPaths: []string{"stderr"},
}
-// AddOutputPaths adds output paths to the existing output paths, resolving conflicts.
-func AddOutputPaths(cfg zap.Config, outputPaths, errorOutputPaths []string) zap.Config {
+// MergeOutputPaths merges logging output paths, resolving conflicts.
+func MergeOutputPaths(cfg zap.Config) zap.Config {
outputs := make(map[string]struct{})
for _, v := range cfg.OutputPaths {
outputs[v] = struct{}{}
}
- for _, v := range outputPaths {
- outputs[v] = struct{}{}
- }
outputSlice := make([]string, 0)
if _, ok := outputs["/dev/null"]; ok {
// "/dev/null" to discard all
@@ -78,9 +75,6 @@
for _, v := range cfg.ErrorOutputPaths {
errOutputs[v] = struct{}{}
}
- for _, v := range errorOutputPaths {
- errOutputs[v] = struct{}{}
- }
errOutputSlice := make([]string, 0)
if _, ok := errOutputs["/dev/null"]; ok {
// "/dev/null" to discard all
diff --git a/vendor/go.etcd.io/etcd/pkg/logutil/zap_raft.go b/vendor/go.etcd.io/etcd/pkg/logutil/zap_raft.go
index e92cba0..f016b30 100644
--- a/vendor/go.etcd.io/etcd/pkg/logutil/zap_raft.go
+++ b/vendor/go.etcd.io/etcd/pkg/logutil/zap_raft.go
@@ -23,7 +23,7 @@
"go.uber.org/zap/zapcore"
)
-// NewRaftLogger converts "*zap.Logger" to "raft.Logger".
+// NewRaftLogger builds "raft.Logger" from "*zap.Config".
func NewRaftLogger(lcfg *zap.Config) (raft.Logger, error) {
if lcfg == nil {
return nil, errors.New("nil zap.Config")
@@ -35,6 +35,11 @@
return &zapRaftLogger{lg: lg, sugar: lg.Sugar()}, nil
}
+// NewRaftLoggerZap converts "*zap.Logger" to "raft.Logger".
+func NewRaftLoggerZap(lg *zap.Logger) raft.Logger {
+ return &zapRaftLogger{lg: lg, sugar: lg.Sugar()}
+}
+
// NewRaftLoggerFromZapCore creates "raft.Logger" from "zap.Core"
// and "zapcore.WriteSyncer".
func NewRaftLoggerFromZapCore(cr zapcore.Core, syncer zapcore.WriteSyncer) raft.Logger {
diff --git a/vendor/go.etcd.io/etcd/pkg/types/set.go b/vendor/go.etcd.io/etcd/pkg/types/set.go
index c111b0c..e7a3cdc 100644
--- a/vendor/go.etcd.io/etcd/pkg/types/set.go
+++ b/vendor/go.etcd.io/etcd/pkg/types/set.go
@@ -148,6 +148,14 @@
func (ts *tsafeSet) Equals(other Set) bool {
ts.m.RLock()
defer ts.m.RUnlock()
+
+ // If ts and other represent the same variable, avoid calling
+ // ts.us.Equals(other), to avoid double RLock bug
+ if _other, ok := other.(*tsafeSet); ok {
+ if _other == ts {
+ return true
+ }
+ }
return ts.us.Equals(other)
}
@@ -173,6 +181,15 @@
func (ts *tsafeSet) Sub(other Set) Set {
ts.m.RLock()
defer ts.m.RUnlock()
+
+ // If ts and other represent the same variable, avoid calling
+ // ts.us.Sub(other), to avoid double RLock bug
+ if _other, ok := other.(*tsafeSet); ok {
+ if _other == ts {
+ usResult := NewUnsafeSet()
+ return &tsafeSet{usResult, sync.RWMutex{}}
+ }
+ }
usResult := ts.us.Sub(other).(*unsafeSet)
return &tsafeSet{usResult, sync.RWMutex{}}
}
diff --git a/vendor/go.etcd.io/etcd/raft/README.md b/vendor/go.etcd.io/etcd/raft/README.md
index a78e5f7..83cf040 100644
--- a/vendor/go.etcd.io/etcd/raft/README.md
+++ b/vendor/go.etcd.io/etcd/raft/README.md
@@ -3,7 +3,7 @@
Raft is a protocol with which a cluster of nodes can maintain a replicated state machine.
The state machine is kept in sync through the use of a replicated log.
For more details on Raft, see "In Search of an Understandable Consensus Algorithm"
-(https://ramcloud.stanford.edu/raft.pdf) by Diego Ongaro and John Ousterhout.
+(https://raft.github.io/raft.pdf) by Diego Ongaro and John Ousterhout.
This Raft library is stable and feature complete. As of 2016, it is **the most widely used** Raft library in production, serving tens of thousands clusters each day. It powers distributed systems such as etcd, Kubernetes, Docker Swarm, Cloud Foundry Diego, CockroachDB, TiDB, Project Calico, Flannel, and more.
@@ -190,7 +190,7 @@
## Implementation notes
-This implementation is up to date with the final Raft thesis (https://ramcloud.stanford.edu/~ongaro/thesis.pdf), although this implementation of the membership change protocol differs somewhat from that described in chapter 4. The key invariant that membership changes happen one node at a time is preserved, but in our implementation the membership change takes effect when its entry is applied, not when it is added to the log (so the entry is committed under the old membership instead of the new). This is equivalent in terms of safety, since the old and new configurations are guaranteed to overlap.
+This implementation is up to date with the final Raft thesis (https://github.com/ongardie/dissertation/blob/master/stanford.pdf), although this implementation of the membership change protocol differs somewhat from that described in chapter 4. The key invariant that membership changes happen one node at a time is preserved, but in our implementation the membership change takes effect when its entry is applied, not when it is added to the log (so the entry is committed under the old membership instead of the new). This is equivalent in terms of safety, since the old and new configurations are guaranteed to overlap.
To ensure there is no attempt to commit two membership changes at once by matching log positions (which would be unsafe since they should have different quorum requirements), any proposed membership change is simply disallowed while any uncommitted change appears in the leader's log.
diff --git a/vendor/go.etcd.io/etcd/raft/bootstrap.go b/vendor/go.etcd.io/etcd/raft/bootstrap.go
new file mode 100644
index 0000000..bd82b20
--- /dev/null
+++ b/vendor/go.etcd.io/etcd/raft/bootstrap.go
@@ -0,0 +1,80 @@
+// Copyright 2015 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package raft
+
+import (
+ "errors"
+
+ pb "go.etcd.io/etcd/raft/raftpb"
+)
+
+// Bootstrap initializes the RawNode for first use by appending configuration
+// changes for the supplied peers. This method returns an error if the Storage
+// is nonempty.
+//
+// It is recommended that instead of calling this method, applications bootstrap
+// their state manually by setting up a Storage that has a first index > 1 and
+// which stores the desired ConfState as its InitialState.
+func (rn *RawNode) Bootstrap(peers []Peer) error {
+ if len(peers) == 0 {
+ return errors.New("must provide at least one peer to Bootstrap")
+ }
+ lastIndex, err := rn.raft.raftLog.storage.LastIndex()
+ if err != nil {
+ return err
+ }
+
+ if lastIndex != 0 {
+ return errors.New("can't bootstrap a nonempty Storage")
+ }
+
+ // We've faked out initial entries above, but nothing has been
+ // persisted. Start with an empty HardState (thus the first Ready will
+ // emit a HardState update for the app to persist).
+ rn.prevHardSt = emptyState
+
+ // TODO(tbg): remove StartNode and give the application the right tools to
+ // bootstrap the initial membership in a cleaner way.
+ rn.raft.becomeFollower(1, None)
+ ents := make([]pb.Entry, len(peers))
+ for i, peer := range peers {
+ cc := pb.ConfChange{Type: pb.ConfChangeAddNode, NodeID: peer.ID, Context: peer.Context}
+ data, err := cc.Marshal()
+ if err != nil {
+ return err
+ }
+
+ ents[i] = pb.Entry{Type: pb.EntryConfChange, Term: 1, Index: uint64(i + 1), Data: data}
+ }
+ rn.raft.raftLog.append(ents...)
+
+ // Now apply them, mainly so that the application can call Campaign
+ // immediately after StartNode in tests. Note that these nodes will
+ // be added to raft twice: here and when the application's Ready
+ // loop calls ApplyConfChange. The calls to addNode must come after
+ // all calls to raftLog.append so progress.next is set after these
+ // bootstrapping entries (it is an error if we try to append these
+ // entries since they have already been committed).
+ // We do not set raftLog.applied so the application will be able
+ // to observe all conf changes via Ready.CommittedEntries.
+ //
+ // TODO(bdarnell): These entries are still unstable; do we need to preserve
+ // the invariant that committed < unstable?
+ rn.raft.raftLog.committed = uint64(len(ents))
+ for _, peer := range peers {
+ rn.raft.applyConfChange(pb.ConfChange{NodeID: peer.ID, Type: pb.ConfChangeAddNode}.AsV2())
+ }
+ return nil
+}
diff --git a/vendor/go.etcd.io/etcd/raft/confchange/confchange.go b/vendor/go.etcd.io/etcd/raft/confchange/confchange.go
new file mode 100644
index 0000000..a0dc486
--- /dev/null
+++ b/vendor/go.etcd.io/etcd/raft/confchange/confchange.go
@@ -0,0 +1,425 @@
+// Copyright 2019 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package confchange
+
+import (
+ "errors"
+ "fmt"
+ "strings"
+
+ "go.etcd.io/etcd/raft/quorum"
+ pb "go.etcd.io/etcd/raft/raftpb"
+ "go.etcd.io/etcd/raft/tracker"
+)
+
+// Changer facilitates configuration changes. It exposes methods to handle
+// simple and joint consensus while performing the proper validation that allows
+// refusing invalid configuration changes before they affect the active
+// configuration.
+type Changer struct {
+ Tracker tracker.ProgressTracker
+ LastIndex uint64
+}
+
+// EnterJoint verifies that the outgoing (=right) majority config of the joint
+// config is empty and initializes it with a copy of the incoming (=left)
+// majority config. That is, it transitions from
+//
+// (1 2 3)&&()
+// to
+// (1 2 3)&&(1 2 3).
+//
+// The supplied changes are then applied to the incoming majority config,
+// resulting in a joint configuration that in terms of the Raft thesis[1]
+// (Section 4.3) corresponds to `C_{new,old}`.
+//
+// [1]: https://github.com/ongardie/dissertation/blob/master/online-trim.pdf
+func (c Changer) EnterJoint(autoLeave bool, ccs ...pb.ConfChangeSingle) (tracker.Config, tracker.ProgressMap, error) {
+ cfg, prs, err := c.checkAndCopy()
+ if err != nil {
+ return c.err(err)
+ }
+ if joint(cfg) {
+ err := errors.New("config is already joint")
+ return c.err(err)
+ }
+ if len(incoming(cfg.Voters)) == 0 {
+ // We allow adding nodes to an empty config for convenience (testing and
+ // bootstrap), but you can't enter a joint state.
+ err := errors.New("can't make a zero-voter config joint")
+ return c.err(err)
+ }
+ // Clear the outgoing config.
+ *outgoingPtr(&cfg.Voters) = quorum.MajorityConfig{}
+ // Copy incoming to outgoing.
+ for id := range incoming(cfg.Voters) {
+ outgoing(cfg.Voters)[id] = struct{}{}
+ }
+
+ if err := c.apply(&cfg, prs, ccs...); err != nil {
+ return c.err(err)
+ }
+ cfg.AutoLeave = autoLeave
+ return checkAndReturn(cfg, prs)
+}
+
+// LeaveJoint transitions out of a joint configuration. It is an error to call
+// this method if the configuration is not joint, i.e. if the outgoing majority
+// config Voters[1] is empty.
+//
+// The outgoing majority config of the joint configuration will be removed,
+// that is, the incoming config is promoted as the sole decision maker. In the
+// notation of the Raft thesis[1] (Section 4.3), this method transitions from
+// `C_{new,old}` into `C_new`.
+//
+// At the same time, any staged learners (LearnersNext) the addition of which
+// was held back by an overlapping voter in the former outgoing config will be
+// inserted into Learners.
+//
+// [1]: https://github.com/ongardie/dissertation/blob/master/online-trim.pdf
+func (c Changer) LeaveJoint() (tracker.Config, tracker.ProgressMap, error) {
+ cfg, prs, err := c.checkAndCopy()
+ if err != nil {
+ return c.err(err)
+ }
+ if !joint(cfg) {
+ err := errors.New("can't leave a non-joint config")
+ return c.err(err)
+ }
+ if len(outgoing(cfg.Voters)) == 0 {
+ err := fmt.Errorf("configuration is not joint: %v", cfg)
+ return c.err(err)
+ }
+ for id := range cfg.LearnersNext {
+ nilAwareAdd(&cfg.Learners, id)
+ prs[id].IsLearner = true
+ }
+ cfg.LearnersNext = nil
+
+ for id := range outgoing(cfg.Voters) {
+ _, isVoter := incoming(cfg.Voters)[id]
+ _, isLearner := cfg.Learners[id]
+
+ if !isVoter && !isLearner {
+ delete(prs, id)
+ }
+ }
+ *outgoingPtr(&cfg.Voters) = nil
+ cfg.AutoLeave = false
+
+ return checkAndReturn(cfg, prs)
+}
+
+// Simple carries out a series of configuration changes that (in aggregate)
+// mutates the incoming majority config Voters[0] by at most one. This method
+// will return an error if that is not the case, if the resulting quorum is
+// zero, or if the configuration is in a joint state (i.e. if there is an
+// outgoing configuration).
+func (c Changer) Simple(ccs ...pb.ConfChangeSingle) (tracker.Config, tracker.ProgressMap, error) {
+ cfg, prs, err := c.checkAndCopy()
+ if err != nil {
+ return c.err(err)
+ }
+ if joint(cfg) {
+ err := errors.New("can't apply simple config change in joint config")
+ return c.err(err)
+ }
+ if err := c.apply(&cfg, prs, ccs...); err != nil {
+ return c.err(err)
+ }
+ if n := symdiff(incoming(c.Tracker.Voters), incoming(cfg.Voters)); n > 1 {
+ return tracker.Config{}, nil, errors.New("more than one voter changed without entering joint config")
+ }
+ if err := checkInvariants(cfg, prs); err != nil {
+ return tracker.Config{}, tracker.ProgressMap{}, nil
+ }
+
+ return checkAndReturn(cfg, prs)
+}
+
+// apply a change to the configuration. By convention, changes to voters are
+// always made to the incoming majority config Voters[0]. Voters[1] is either
+// empty or preserves the outgoing majority configuration while in a joint state.
+func (c Changer) apply(cfg *tracker.Config, prs tracker.ProgressMap, ccs ...pb.ConfChangeSingle) error {
+ for _, cc := range ccs {
+ if cc.NodeID == 0 {
+ // etcd replaces the NodeID with zero if it decides (downstream of
+ // raft) to not apply a change, so we have to have explicit code
+ // here to ignore these.
+ continue
+ }
+ switch cc.Type {
+ case pb.ConfChangeAddNode:
+ c.makeVoter(cfg, prs, cc.NodeID)
+ case pb.ConfChangeAddLearnerNode:
+ c.makeLearner(cfg, prs, cc.NodeID)
+ case pb.ConfChangeRemoveNode:
+ c.remove(cfg, prs, cc.NodeID)
+ case pb.ConfChangeUpdateNode:
+ default:
+ return fmt.Errorf("unexpected conf type %d", cc.Type)
+ }
+ }
+ if len(incoming(cfg.Voters)) == 0 {
+ return errors.New("removed all voters")
+ }
+ return nil
+}
+
+// makeVoter adds or promotes the given ID to be a voter in the incoming
+// majority config.
+func (c Changer) makeVoter(cfg *tracker.Config, prs tracker.ProgressMap, id uint64) {
+ pr := prs[id]
+ if pr == nil {
+ c.initProgress(cfg, prs, id, false /* isLearner */)
+ return
+ }
+
+ pr.IsLearner = false
+ nilAwareDelete(&cfg.Learners, id)
+ nilAwareDelete(&cfg.LearnersNext, id)
+ incoming(cfg.Voters)[id] = struct{}{}
+ return
+}
+
+// makeLearner makes the given ID a learner or stages it to be a learner once
+// an active joint configuration is exited.
+//
+// The former happens when the peer is not a part of the outgoing config, in
+// which case we either add a new learner or demote a voter in the incoming
+// config.
+//
+// The latter case occurs when the configuration is joint and the peer is a
+// voter in the outgoing config. In that case, we do not want to add the peer
+// as a learner because then we'd have to track a peer as a voter and learner
+// simultaneously. Instead, we add the learner to LearnersNext, so that it will
+// be added to Learners the moment the outgoing config is removed by
+// LeaveJoint().
+func (c Changer) makeLearner(cfg *tracker.Config, prs tracker.ProgressMap, id uint64) {
+ pr := prs[id]
+ if pr == nil {
+ c.initProgress(cfg, prs, id, true /* isLearner */)
+ return
+ }
+ if pr.IsLearner {
+ return
+ }
+ // Remove any existing voter in the incoming config...
+ c.remove(cfg, prs, id)
+ // ... but save the Progress.
+ prs[id] = pr
+ // Use LearnersNext if we can't add the learner to Learners directly, i.e.
+ // if the peer is still tracked as a voter in the outgoing config. It will
+ // be turned into a learner in LeaveJoint().
+ //
+ // Otherwise, add a regular learner right away.
+ if _, onRight := outgoing(cfg.Voters)[id]; onRight {
+ nilAwareAdd(&cfg.LearnersNext, id)
+ } else {
+ pr.IsLearner = true
+ nilAwareAdd(&cfg.Learners, id)
+ }
+}
+
+// remove this peer as a voter or learner from the incoming config.
+func (c Changer) remove(cfg *tracker.Config, prs tracker.ProgressMap, id uint64) {
+ if _, ok := prs[id]; !ok {
+ return
+ }
+
+ delete(incoming(cfg.Voters), id)
+ nilAwareDelete(&cfg.Learners, id)
+ nilAwareDelete(&cfg.LearnersNext, id)
+
+ // If the peer is still a voter in the outgoing config, keep the Progress.
+ if _, onRight := outgoing(cfg.Voters)[id]; !onRight {
+ delete(prs, id)
+ }
+}
+
+// initProgress initializes a new progress for the given node or learner.
+func (c Changer) initProgress(cfg *tracker.Config, prs tracker.ProgressMap, id uint64, isLearner bool) {
+ if !isLearner {
+ incoming(cfg.Voters)[id] = struct{}{}
+ } else {
+ nilAwareAdd(&cfg.Learners, id)
+ }
+ prs[id] = &tracker.Progress{
+ // Initializing the Progress with the last index means that the follower
+ // can be probed (with the last index).
+ //
+ // TODO(tbg): seems awfully optimistic. Using the first index would be
+ // better. The general expectation here is that the follower has no log
+ // at all (and will thus likely need a snapshot), though the app may
+ // have applied a snapshot out of band before adding the replica (thus
+ // making the first index the better choice).
+ Next: c.LastIndex,
+ Match: 0,
+ Inflights: tracker.NewInflights(c.Tracker.MaxInflight),
+ IsLearner: isLearner,
+ // When a node is first added, we should mark it as recently active.
+ // Otherwise, CheckQuorum may cause us to step down if it is invoked
+ // before the added node has had a chance to communicate with us.
+ RecentActive: true,
+ }
+}
+
+// checkInvariants makes sure that the config and progress are compatible with
+// each other. This is used to check both what the Changer is initialized with,
+// as well as what it returns.
+func checkInvariants(cfg tracker.Config, prs tracker.ProgressMap) error {
+ // NB: intentionally allow the empty config. In production we'll never see a
+ // non-empty config (we prevent it from being created) but we will need to
+ // be able to *create* an initial config, for example during bootstrap (or
+ // during tests). Instead of having to hand-code this, we allow
+ // transitioning from an empty config into any other legal and non-empty
+ // config.
+ for _, ids := range []map[uint64]struct{}{
+ cfg.Voters.IDs(),
+ cfg.Learners,
+ cfg.LearnersNext,
+ } {
+ for id := range ids {
+ if _, ok := prs[id]; !ok {
+ return fmt.Errorf("no progress for %d", id)
+ }
+ }
+ }
+
+ // Any staged learner was staged because it could not be directly added due
+ // to a conflicting voter in the outgoing config.
+ for id := range cfg.LearnersNext {
+ if _, ok := outgoing(cfg.Voters)[id]; !ok {
+ return fmt.Errorf("%d is in LearnersNext, but not Voters[1]", id)
+ }
+ if prs[id].IsLearner {
+ return fmt.Errorf("%d is in LearnersNext, but is already marked as learner", id)
+ }
+ }
+ // Conversely Learners and Voters doesn't intersect at all.
+ for id := range cfg.Learners {
+ if _, ok := outgoing(cfg.Voters)[id]; ok {
+ return fmt.Errorf("%d is in Learners and Voters[1]", id)
+ }
+ if _, ok := incoming(cfg.Voters)[id]; ok {
+ return fmt.Errorf("%d is in Learners and Voters[0]", id)
+ }
+ if !prs[id].IsLearner {
+ return fmt.Errorf("%d is in Learners, but is not marked as learner", id)
+ }
+ }
+
+ if !joint(cfg) {
+ // We enforce that empty maps are nil instead of zero.
+ if outgoing(cfg.Voters) != nil {
+ return fmt.Errorf("Voters[1] must be nil when not joint")
+ }
+ if cfg.LearnersNext != nil {
+ return fmt.Errorf("LearnersNext must be nil when not joint")
+ }
+ if cfg.AutoLeave {
+ return fmt.Errorf("AutoLeave must be false when not joint")
+ }
+ }
+
+ return nil
+}
+
+// checkAndCopy copies the tracker's config and progress map (deeply enough for
+// the purposes of the Changer) and returns those copies. It returns an error
+// if checkInvariants does.
+func (c Changer) checkAndCopy() (tracker.Config, tracker.ProgressMap, error) {
+ cfg := c.Tracker.Config.Clone()
+ prs := tracker.ProgressMap{}
+
+ for id, pr := range c.Tracker.Progress {
+ // A shallow copy is enough because we only mutate the Learner field.
+ ppr := *pr
+ prs[id] = &ppr
+ }
+ return checkAndReturn(cfg, prs)
+}
+
+// checkAndReturn calls checkInvariants on the input and returns either the
+// resulting error or the input.
+func checkAndReturn(cfg tracker.Config, prs tracker.ProgressMap) (tracker.Config, tracker.ProgressMap, error) {
+ if err := checkInvariants(cfg, prs); err != nil {
+ return tracker.Config{}, tracker.ProgressMap{}, err
+ }
+ return cfg, prs, nil
+}
+
+// err returns zero values and an error.
+func (c Changer) err(err error) (tracker.Config, tracker.ProgressMap, error) {
+ return tracker.Config{}, nil, err
+}
+
+// nilAwareAdd populates a map entry, creating the map if necessary.
+func nilAwareAdd(m *map[uint64]struct{}, id uint64) {
+ if *m == nil {
+ *m = map[uint64]struct{}{}
+ }
+ (*m)[id] = struct{}{}
+}
+
+// nilAwareDelete deletes from a map, nil'ing the map itself if it is empty after.
+func nilAwareDelete(m *map[uint64]struct{}, id uint64) {
+ if *m == nil {
+ return
+ }
+ delete(*m, id)
+ if len(*m) == 0 {
+ *m = nil
+ }
+}
+
+// symdiff returns the count of the symmetric difference between the sets of
+// uint64s, i.e. len( (l - r) \union (r - l)).
+func symdiff(l, r map[uint64]struct{}) int {
+ var n int
+ pairs := [][2]quorum.MajorityConfig{
+ {l, r}, // count elems in l but not in r
+ {r, l}, // count elems in r but not in l
+ }
+ for _, p := range pairs {
+ for id := range p[0] {
+ if _, ok := p[1][id]; !ok {
+ n++
+ }
+ }
+ }
+ return n
+}
+
+func joint(cfg tracker.Config) bool {
+ return len(outgoing(cfg.Voters)) > 0
+}
+
+func incoming(voters quorum.JointConfig) quorum.MajorityConfig { return voters[0] }
+func outgoing(voters quorum.JointConfig) quorum.MajorityConfig { return voters[1] }
+func outgoingPtr(voters *quorum.JointConfig) *quorum.MajorityConfig { return &voters[1] }
+
+// Describe prints the type and NodeID of the configuration changes as a
+// space-delimited string.
+func Describe(ccs ...pb.ConfChangeSingle) string {
+ var buf strings.Builder
+ for _, cc := range ccs {
+ if buf.Len() > 0 {
+ buf.WriteByte(' ')
+ }
+ fmt.Fprintf(&buf, "%s(%d)", cc.Type, cc.NodeID)
+ }
+ return buf.String()
+}
diff --git a/vendor/go.etcd.io/etcd/raft/confchange/restore.go b/vendor/go.etcd.io/etcd/raft/confchange/restore.go
new file mode 100644
index 0000000..724068d
--- /dev/null
+++ b/vendor/go.etcd.io/etcd/raft/confchange/restore.go
@@ -0,0 +1,155 @@
+// Copyright 2019 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package confchange
+
+import (
+ pb "go.etcd.io/etcd/raft/raftpb"
+ "go.etcd.io/etcd/raft/tracker"
+)
+
+// toConfChangeSingle translates a conf state into 1) a slice of operations creating
+// first the config that will become the outgoing one, and then the incoming one, and
+// b) another slice that, when applied to the config resulted from 1), represents the
+// ConfState.
+func toConfChangeSingle(cs pb.ConfState) (out []pb.ConfChangeSingle, in []pb.ConfChangeSingle) {
+ // Example to follow along this code:
+ // voters=(1 2 3) learners=(5) outgoing=(1 2 4 6) learners_next=(4)
+ //
+ // This means that before entering the joint config, the configuration
+ // had voters (1 2 4) and perhaps some learners that are already gone.
+ // The new set of voters is (1 2 3), i.e. (1 2) were kept around, and (4 6)
+ // are no longer voters; however 4 is poised to become a learner upon leaving
+ // the joint state.
+ // We can't tell whether 5 was a learner before entering the joint config,
+ // but it doesn't matter (we'll pretend that it wasn't).
+ //
+ // The code below will construct
+ // outgoing = add 1; add 2; add 4; add 6
+ // incoming = remove 1; remove 2; remove 4; remove 6
+ // add 1; add 2; add 3;
+ // add-learner 5;
+ // add-learner 4;
+ //
+ // So, when starting with an empty config, after applying 'outgoing' we have
+ //
+ // quorum=(1 2 4 6)
+ //
+ // From which we enter a joint state via 'incoming'
+ //
+ // quorum=(1 2 3)&&(1 2 4 6) learners=(5) learners_next=(4)
+ //
+ // as desired.
+
+ for _, id := range cs.VotersOutgoing {
+ // If there are outgoing voters, first add them one by one so that the
+ // (non-joint) config has them all.
+ out = append(out, pb.ConfChangeSingle{
+ Type: pb.ConfChangeAddNode,
+ NodeID: id,
+ })
+
+ }
+
+ // We're done constructing the outgoing slice, now on to the incoming one
+ // (which will apply on top of the config created by the outgoing slice).
+
+ // First, we'll remove all of the outgoing voters.
+ for _, id := range cs.VotersOutgoing {
+ in = append(in, pb.ConfChangeSingle{
+ Type: pb.ConfChangeRemoveNode,
+ NodeID: id,
+ })
+ }
+ // Then we'll add the incoming voters and learners.
+ for _, id := range cs.Voters {
+ in = append(in, pb.ConfChangeSingle{
+ Type: pb.ConfChangeAddNode,
+ NodeID: id,
+ })
+ }
+ for _, id := range cs.Learners {
+ in = append(in, pb.ConfChangeSingle{
+ Type: pb.ConfChangeAddLearnerNode,
+ NodeID: id,
+ })
+ }
+ // Same for LearnersNext; these are nodes we want to be learners but which
+ // are currently voters in the outgoing config.
+ for _, id := range cs.LearnersNext {
+ in = append(in, pb.ConfChangeSingle{
+ Type: pb.ConfChangeAddLearnerNode,
+ NodeID: id,
+ })
+ }
+ return out, in
+}
+
+func chain(chg Changer, ops ...func(Changer) (tracker.Config, tracker.ProgressMap, error)) (tracker.Config, tracker.ProgressMap, error) {
+ for _, op := range ops {
+ cfg, prs, err := op(chg)
+ if err != nil {
+ return tracker.Config{}, nil, err
+ }
+ chg.Tracker.Config = cfg
+ chg.Tracker.Progress = prs
+ }
+ return chg.Tracker.Config, chg.Tracker.Progress, nil
+}
+
+// Restore takes a Changer (which must represent an empty configuration), and
+// runs a sequence of changes enacting the configuration described in the
+// ConfState.
+//
+// TODO(tbg) it's silly that this takes a Changer. Unravel this by making sure
+// the Changer only needs a ProgressMap (not a whole Tracker) at which point
+// this can just take LastIndex and MaxInflight directly instead and cook up
+// the results from that alone.
+func Restore(chg Changer, cs pb.ConfState) (tracker.Config, tracker.ProgressMap, error) {
+ outgoing, incoming := toConfChangeSingle(cs)
+
+ var ops []func(Changer) (tracker.Config, tracker.ProgressMap, error)
+
+ if len(outgoing) == 0 {
+ // No outgoing config, so just apply the incoming changes one by one.
+ for _, cc := range incoming {
+ cc := cc // loop-local copy
+ ops = append(ops, func(chg Changer) (tracker.Config, tracker.ProgressMap, error) {
+ return chg.Simple(cc)
+ })
+ }
+ } else {
+ // The ConfState describes a joint configuration.
+ //
+ // First, apply all of the changes of the outgoing config one by one, so
+ // that it temporarily becomes the incoming active config. For example,
+ // if the config is (1 2 3)&(2 3 4), this will establish (2 3 4)&().
+ for _, cc := range outgoing {
+ cc := cc // loop-local copy
+ ops = append(ops, func(chg Changer) (tracker.Config, tracker.ProgressMap, error) {
+ return chg.Simple(cc)
+ })
+ }
+ // Now enter the joint state, which rotates the above additions into the
+ // outgoing config, and adds the incoming config in. Continuing the
+ // example above, we'd get (1 2 3)&(2 3 4), i.e. the incoming operations
+ // would be removing 2,3,4 and then adding in 1,2,3 while transitioning
+ // into a joint state.
+ ops = append(ops, func(chg Changer) (tracker.Config, tracker.ProgressMap, error) {
+ return chg.EnterJoint(cs.AutoLeave, incoming...)
+ })
+ }
+
+ return chain(chg, ops...)
+}
diff --git a/vendor/go.etcd.io/etcd/raft/doc.go b/vendor/go.etcd.io/etcd/raft/doc.go
index c30d884..68fe6f0 100644
--- a/vendor/go.etcd.io/etcd/raft/doc.go
+++ b/vendor/go.etcd.io/etcd/raft/doc.go
@@ -19,7 +19,7 @@
Raft is a protocol with which a cluster of nodes can maintain a replicated state machine.
The state machine is kept in sync through the use of a replicated log.
For more details on Raft, see "In Search of an Understandable Consensus Algorithm"
-(https://ramcloud.stanford.edu/raft.pdf) by Diego Ongaro and John Ousterhout.
+(https://raft.github.io/raft.pdf) by Diego Ongaro and John Ousterhout.
A simple example application, _raftexample_, is also available to help illustrate
how to use this package in practice:
@@ -172,7 +172,7 @@
Implementation notes
This implementation is up to date with the final Raft thesis
-(https://ramcloud.stanford.edu/~ongaro/thesis.pdf), although our
+(https://github.com/ongardie/dissertation/blob/master/stanford.pdf), although our
implementation of the membership change protocol differs somewhat from
that described in chapter 4. The key invariant that membership changes
happen one node at a time is preserved, but in our implementation the
diff --git a/vendor/go.etcd.io/etcd/raft/log.go b/vendor/go.etcd.io/etcd/raft/log.go
index 03f83e6..77eedfc 100644
--- a/vendor/go.etcd.io/etcd/raft/log.go
+++ b/vendor/go.etcd.io/etcd/raft/log.go
@@ -332,8 +332,10 @@
if hi > l.unstable.offset {
unstable := l.unstable.slice(max(lo, l.unstable.offset), hi)
if len(ents) > 0 {
- ents = append([]pb.Entry{}, ents...)
- ents = append(ents, unstable...)
+ combined := make([]pb.Entry, len(ents)+len(unstable))
+ n := copy(combined, ents)
+ copy(combined[n:], unstable)
+ ents = combined
} else {
ents = unstable
}
diff --git a/vendor/go.etcd.io/etcd/raft/log_unstable.go b/vendor/go.etcd.io/etcd/raft/log_unstable.go
index 1005bf6..1bff5a7 100644
--- a/vendor/go.etcd.io/etcd/raft/log_unstable.go
+++ b/vendor/go.etcd.io/etcd/raft/log_unstable.go
@@ -55,10 +55,7 @@
// is any.
func (u *unstable) maybeTerm(i uint64) (uint64, bool) {
if i < u.offset {
- if u.snapshot == nil {
- return 0, false
- }
- if u.snapshot.Metadata.Index == i {
+ if u.snapshot != nil && u.snapshot.Metadata.Index == i {
return u.snapshot.Metadata.Term, true
}
return 0, false
@@ -71,6 +68,7 @@
if i > last {
return 0, false
}
+
return u.entries[i-u.offset].Term, true
}
diff --git a/vendor/go.etcd.io/etcd/raft/logger.go b/vendor/go.etcd.io/etcd/raft/logger.go
index 426a77d..6d89629 100644
--- a/vendor/go.etcd.io/etcd/raft/logger.go
+++ b/vendor/go.etcd.io/etcd/raft/logger.go
@@ -19,6 +19,7 @@
"io/ioutil"
"log"
"os"
+ "sync"
)
type Logger interface {
@@ -41,11 +42,16 @@
Panicf(format string, v ...interface{})
}
-func SetLogger(l Logger) { raftLogger = l }
+func SetLogger(l Logger) {
+ raftLoggerMu.Lock()
+ raftLogger = l
+ raftLoggerMu.Unlock()
+}
var (
defaultLogger = &DefaultLogger{Logger: log.New(os.Stderr, "raft", log.LstdFlags)}
discardLogger = &DefaultLogger{Logger: log.New(ioutil.Discard, "", 0)}
+ raftLoggerMu sync.Mutex
raftLogger = Logger(defaultLogger)
)
diff --git a/vendor/go.etcd.io/etcd/raft/node.go b/vendor/go.etcd.io/etcd/raft/node.go
index 2ec2c3a..ab6185b 100644
--- a/vendor/go.etcd.io/etcd/raft/node.go
+++ b/vendor/go.etcd.io/etcd/raft/node.go
@@ -132,10 +132,20 @@
// Propose proposes that data be appended to the log. Note that proposals can be lost without
// notice, therefore it is user's job to ensure proposal retries.
Propose(ctx context.Context, data []byte) error
- // ProposeConfChange proposes config change.
- // At most one ConfChange can be in the process of going through consensus.
- // Application needs to call ApplyConfChange when applying EntryConfChange type entry.
- ProposeConfChange(ctx context.Context, cc pb.ConfChange) error
+ // ProposeConfChange proposes a configuration change. Like any proposal, the
+ // configuration change may be dropped with or without an error being
+ // returned. In particular, configuration changes are dropped unless the
+ // leader has certainty that there is no prior unapplied configuration
+ // change in its log.
+ //
+ // The method accepts either a pb.ConfChange (deprecated) or pb.ConfChangeV2
+ // message. The latter allows arbitrary configuration changes via joint
+ // consensus, notably including replacing a voter. Passing a ConfChangeV2
+ // message is only allowed if all Nodes participating in the cluster run a
+ // version of this library aware of the V2 API. See pb.ConfChangeV2 for
+ // usage details and semantics.
+ ProposeConfChange(ctx context.Context, cc pb.ConfChangeI) error
+
// Step advances the state machine using the given message. ctx.Err() will be returned, if any.
Step(ctx context.Context, msg pb.Message) error
@@ -156,11 +166,13 @@
// a long time to apply the snapshot data. To continue receiving Ready without blocking raft
// progress, it can call Advance before finishing applying the last ready.
Advance()
- // ApplyConfChange applies config change to the local node.
- // Returns an opaque ConfState protobuf which must be recorded
- // in snapshots. Will never return nil; it returns a pointer only
- // to match MemoryStorage.Compact.
- ApplyConfChange(cc pb.ConfChange) *pb.ConfState
+ // ApplyConfChange applies a config change (previously passed to
+ // ProposeConfChange) to the node. This must be called whenever a config
+ // change is observed in Ready.CommittedEntries.
+ //
+ // Returns an opaque non-nil ConfState protobuf which must be recorded in
+ // snapshots.
+ ApplyConfChange(cc pb.ConfChangeI) *pb.ConfState
// TransferLeadership attempts to transfer leadership to the given transferee.
TransferLeadership(ctx context.Context, lead, transferee uint64)
@@ -197,40 +209,21 @@
// StartNode returns a new Node given configuration and a list of raft peers.
// It appends a ConfChangeAddNode entry for each given peer to the initial log.
+//
+// Peers must not be zero length; call RestartNode in that case.
func StartNode(c *Config, peers []Peer) Node {
- r := newRaft(c)
- // become the follower at term 1 and apply initial configuration
- // entries of term 1
- r.becomeFollower(1, None)
- for _, peer := range peers {
- cc := pb.ConfChange{Type: pb.ConfChangeAddNode, NodeID: peer.ID, Context: peer.Context}
- d, err := cc.Marshal()
- if err != nil {
- panic("unexpected marshal error")
- }
- e := pb.Entry{Type: pb.EntryConfChange, Term: 1, Index: r.raftLog.lastIndex() + 1, Data: d}
- r.raftLog.append(e)
+ if len(peers) == 0 {
+ panic("no peers given; use RestartNode instead")
}
- // Mark these initial entries as committed.
- // TODO(bdarnell): These entries are still unstable; do we need to preserve
- // the invariant that committed < unstable?
- r.raftLog.committed = r.raftLog.lastIndex()
- // Now apply them, mainly so that the application can call Campaign
- // immediately after StartNode in tests. Note that these nodes will
- // be added to raft twice: here and when the application's Ready
- // loop calls ApplyConfChange. The calls to addNode must come after
- // all calls to raftLog.append so progress.next is set after these
- // bootstrapping entries (it is an error if we try to append these
- // entries since they have already been committed).
- // We do not set raftLog.applied so the application will be able
- // to observe all conf changes via Ready.CommittedEntries.
- for _, peer := range peers {
- r.addNode(peer.ID)
+ rn, err := NewRawNode(c)
+ if err != nil {
+ panic(err)
}
+ rn.Bootstrap(peers)
- n := newNode()
- n.logger = c.Logger
- go n.run(r)
+ n := newNode(rn)
+
+ go n.run()
return &n
}
@@ -239,11 +232,12 @@
// If the caller has an existing state machine, pass in the last log index that
// has been applied to it; otherwise use zero.
func RestartNode(c *Config) Node {
- r := newRaft(c)
-
- n := newNode()
- n.logger = c.Logger
- go n.run(r)
+ rn, err := NewRawNode(c)
+ if err != nil {
+ panic(err)
+ }
+ n := newNode(rn)
+ go n.run()
return &n
}
@@ -256,7 +250,7 @@
type node struct {
propc chan msgWithResult
recvc chan pb.Message
- confc chan pb.ConfChange
+ confc chan pb.ConfChangeV2
confstatec chan pb.ConfState
readyc chan Ready
advancec chan struct{}
@@ -265,14 +259,14 @@
stop chan struct{}
status chan chan Status
- logger Logger
+ rn *RawNode
}
-func newNode() node {
+func newNode(rn *RawNode) node {
return node{
propc: make(chan msgWithResult),
recvc: make(chan pb.Message),
- confc: make(chan pb.ConfChange),
+ confc: make(chan pb.ConfChangeV2),
confstatec: make(chan pb.ConfState),
readyc: make(chan Ready),
advancec: make(chan struct{}),
@@ -283,6 +277,7 @@
done: make(chan struct{}),
stop: make(chan struct{}),
status: make(chan chan Status),
+ rn: rn,
}
}
@@ -298,30 +293,30 @@
<-n.done
}
-func (n *node) run(r *raft) {
+func (n *node) run() {
var propc chan msgWithResult
var readyc chan Ready
var advancec chan struct{}
- var prevLastUnstablei, prevLastUnstablet uint64
- var havePrevLastUnstablei bool
- var prevSnapi uint64
- var applyingToI uint64
var rd Ready
+ r := n.rn.raft
+
lead := None
- prevSoftSt := r.softState()
- prevHardSt := emptyState
for {
if advancec != nil {
readyc = nil
- } else {
- rd = newReady(r, prevSoftSt, prevHardSt)
- if rd.containsUpdates() {
- readyc = n.readyc
- } else {
- readyc = nil
- }
+ } else if n.rn.HasReady() {
+ // Populate a Ready. Note that this Ready is not guaranteed to
+ // actually be handled. We will arm readyc, but there's no guarantee
+ // that we will actually send on it. It's possible that we will
+ // service another channel instead, loop around, and then populate
+ // the Ready again. We could instead force the previous Ready to be
+ // handled first, but it's generally good to emit larger Readys plus
+ // it simplifies testing (by emitting less frequently and more
+ // predictably).
+ rd = n.rn.readyWithoutAccept()
+ readyc = n.readyc
}
if lead != r.lead {
@@ -353,76 +348,46 @@
}
case m := <-n.recvc:
// filter out response message from unknown From.
- if pr := r.getProgress(m.From); pr != nil || !IsResponseMsg(m.Type) {
+ if pr := r.prs.Progress[m.From]; pr != nil || !IsResponseMsg(m.Type) {
r.Step(m)
}
case cc := <-n.confc:
- if cc.NodeID == None {
- select {
- case n.confstatec <- pb.ConfState{
- Nodes: r.nodes(),
- Learners: r.learnerNodes()}:
- case <-n.done:
+ _, okBefore := r.prs.Progress[r.id]
+ cs := r.applyConfChange(cc)
+ // If the node was removed, block incoming proposals. Note that we
+ // only do this if the node was in the config before. Nodes may be
+ // a member of the group without knowing this (when they're catching
+ // up on the log and don't have the latest config) and we don't want
+ // to block the proposal channel in that case.
+ //
+ // NB: propc is reset when the leader changes, which, if we learn
+ // about it, sort of implies that we got readded, maybe? This isn't
+ // very sound and likely has bugs.
+ if _, okAfter := r.prs.Progress[r.id]; okBefore && !okAfter {
+ var found bool
+ for _, sl := range [][]uint64{cs.Voters, cs.VotersOutgoing} {
+ for _, id := range sl {
+ if id == r.id {
+ found = true
+ }
+ }
}
- break
- }
- switch cc.Type {
- case pb.ConfChangeAddNode:
- r.addNode(cc.NodeID)
- case pb.ConfChangeAddLearnerNode:
- r.addLearner(cc.NodeID)
- case pb.ConfChangeRemoveNode:
- // block incoming proposal when local node is
- // removed
- if cc.NodeID == r.id {
+ if !found {
propc = nil
}
- r.removeNode(cc.NodeID)
- case pb.ConfChangeUpdateNode:
- default:
- panic("unexpected conf type")
}
select {
- case n.confstatec <- pb.ConfState{
- Nodes: r.nodes(),
- Learners: r.learnerNodes()}:
+ case n.confstatec <- cs:
case <-n.done:
}
case <-n.tickc:
- r.tick()
+ n.rn.Tick()
case readyc <- rd:
- if rd.SoftState != nil {
- prevSoftSt = rd.SoftState
- }
- if len(rd.Entries) > 0 {
- prevLastUnstablei = rd.Entries[len(rd.Entries)-1].Index
- prevLastUnstablet = rd.Entries[len(rd.Entries)-1].Term
- havePrevLastUnstablei = true
- }
- if !IsEmptyHardState(rd.HardState) {
- prevHardSt = rd.HardState
- }
- if !IsEmptySnap(rd.Snapshot) {
- prevSnapi = rd.Snapshot.Metadata.Index
- }
- if index := rd.appliedCursor(); index != 0 {
- applyingToI = index
- }
-
- r.msgs = nil
- r.readStates = nil
- r.reduceUncommittedSize(rd.CommittedEntries)
+ n.rn.acceptReady(rd)
advancec = n.advancec
case <-advancec:
- if applyingToI != 0 {
- r.raftLog.appliedTo(applyingToI)
- applyingToI = 0
- }
- if havePrevLastUnstablei {
- r.raftLog.stableTo(prevLastUnstablei, prevLastUnstablet)
- havePrevLastUnstablei = false
- }
- r.raftLog.stableSnapTo(prevSnapi)
+ n.rn.Advance(rd)
+ rd = Ready{}
advancec = nil
case c := <-n.status:
c <- getStatus(r)
@@ -440,7 +405,7 @@
case n.tickc <- struct{}{}:
case <-n.done:
default:
- n.logger.Warningf("A tick missed to fire. Node blocks too long!")
+ n.rn.raft.logger.Warningf("%x (leader %v) A tick missed to fire. Node blocks too long!", n.rn.raft.id, n.rn.raft.id == n.rn.raft.lead)
}
}
@@ -459,12 +424,20 @@
return n.step(ctx, m)
}
-func (n *node) ProposeConfChange(ctx context.Context, cc pb.ConfChange) error {
- data, err := cc.Marshal()
+func confChangeToMsg(c pb.ConfChangeI) (pb.Message, error) {
+ typ, data, err := pb.MarshalConfChange(c)
+ if err != nil {
+ return pb.Message{}, err
+ }
+ return pb.Message{Type: pb.MsgProp, Entries: []pb.Entry{{Type: typ, Data: data}}}, nil
+}
+
+func (n *node) ProposeConfChange(ctx context.Context, cc pb.ConfChangeI) error {
+ msg, err := confChangeToMsg(cc)
if err != nil {
return err
}
- return n.Step(ctx, pb.Message{Type: pb.MsgProp, Entries: []pb.Entry{{Type: pb.EntryConfChange, Data: data}}})
+ return n.Step(ctx, msg)
}
func (n *node) step(ctx context.Context, m pb.Message) error {
@@ -525,10 +498,10 @@
}
}
-func (n *node) ApplyConfChange(cc pb.ConfChange) *pb.ConfState {
+func (n *node) ApplyConfChange(cc pb.ConfChangeI) *pb.ConfState {
var cs pb.ConfState
select {
- case n.confc <- cc:
+ case n.confc <- cc.AsV2():
case <-n.done:
}
select {
diff --git a/vendor/go.etcd.io/etcd/raft/progress.go b/vendor/go.etcd.io/etcd/raft/progress.go
deleted file mode 100644
index ef3787d..0000000
--- a/vendor/go.etcd.io/etcd/raft/progress.go
+++ /dev/null
@@ -1,284 +0,0 @@
-// Copyright 2015 The etcd Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package raft
-
-import "fmt"
-
-const (
- ProgressStateProbe ProgressStateType = iota
- ProgressStateReplicate
- ProgressStateSnapshot
-)
-
-type ProgressStateType uint64
-
-var prstmap = [...]string{
- "ProgressStateProbe",
- "ProgressStateReplicate",
- "ProgressStateSnapshot",
-}
-
-func (st ProgressStateType) String() string { return prstmap[uint64(st)] }
-
-// Progress represents a follower’s progress in the view of the leader. Leader maintains
-// progresses of all followers, and sends entries to the follower based on its progress.
-type Progress struct {
- Match, Next uint64
- // State defines how the leader should interact with the follower.
- //
- // When in ProgressStateProbe, leader sends at most one replication message
- // per heartbeat interval. It also probes actual progress of the follower.
- //
- // When in ProgressStateReplicate, leader optimistically increases next
- // to the latest entry sent after sending replication message. This is
- // an optimized state for fast replicating log entries to the follower.
- //
- // When in ProgressStateSnapshot, leader should have sent out snapshot
- // before and stops sending any replication message.
- State ProgressStateType
-
- // Paused is used in ProgressStateProbe.
- // When Paused is true, raft should pause sending replication message to this peer.
- Paused bool
- // PendingSnapshot is used in ProgressStateSnapshot.
- // If there is a pending snapshot, the pendingSnapshot will be set to the
- // index of the snapshot. If pendingSnapshot is set, the replication process of
- // this Progress will be paused. raft will not resend snapshot until the pending one
- // is reported to be failed.
- PendingSnapshot uint64
-
- // RecentActive is true if the progress is recently active. Receiving any messages
- // from the corresponding follower indicates the progress is active.
- // RecentActive can be reset to false after an election timeout.
- RecentActive bool
-
- // inflights is a sliding window for the inflight messages.
- // Each inflight message contains one or more log entries.
- // The max number of entries per message is defined in raft config as MaxSizePerMsg.
- // Thus inflight effectively limits both the number of inflight messages
- // and the bandwidth each Progress can use.
- // When inflights is full, no more message should be sent.
- // When a leader sends out a message, the index of the last
- // entry should be added to inflights. The index MUST be added
- // into inflights in order.
- // When a leader receives a reply, the previous inflights should
- // be freed by calling inflights.freeTo with the index of the last
- // received entry.
- ins *inflights
-
- // IsLearner is true if this progress is tracked for a learner.
- IsLearner bool
-}
-
-func (pr *Progress) resetState(state ProgressStateType) {
- pr.Paused = false
- pr.PendingSnapshot = 0
- pr.State = state
- pr.ins.reset()
-}
-
-func (pr *Progress) becomeProbe() {
- // If the original state is ProgressStateSnapshot, progress knows that
- // the pending snapshot has been sent to this peer successfully, then
- // probes from pendingSnapshot + 1.
- if pr.State == ProgressStateSnapshot {
- pendingSnapshot := pr.PendingSnapshot
- pr.resetState(ProgressStateProbe)
- pr.Next = max(pr.Match+1, pendingSnapshot+1)
- } else {
- pr.resetState(ProgressStateProbe)
- pr.Next = pr.Match + 1
- }
-}
-
-func (pr *Progress) becomeReplicate() {
- pr.resetState(ProgressStateReplicate)
- pr.Next = pr.Match + 1
-}
-
-func (pr *Progress) becomeSnapshot(snapshoti uint64) {
- pr.resetState(ProgressStateSnapshot)
- pr.PendingSnapshot = snapshoti
-}
-
-// maybeUpdate returns false if the given n index comes from an outdated message.
-// Otherwise it updates the progress and returns true.
-func (pr *Progress) maybeUpdate(n uint64) bool {
- var updated bool
- if pr.Match < n {
- pr.Match = n
- updated = true
- pr.resume()
- }
- if pr.Next < n+1 {
- pr.Next = n + 1
- }
- return updated
-}
-
-func (pr *Progress) optimisticUpdate(n uint64) { pr.Next = n + 1 }
-
-// maybeDecrTo returns false if the given to index comes from an out of order message.
-// Otherwise it decreases the progress next index to min(rejected, last) and returns true.
-func (pr *Progress) maybeDecrTo(rejected, last uint64) bool {
- if pr.State == ProgressStateReplicate {
- // the rejection must be stale if the progress has matched and "rejected"
- // is smaller than "match".
- if rejected <= pr.Match {
- return false
- }
- // directly decrease next to match + 1
- pr.Next = pr.Match + 1
- return true
- }
-
- // the rejection must be stale if "rejected" does not match next - 1
- if pr.Next-1 != rejected {
- return false
- }
-
- if pr.Next = min(rejected, last+1); pr.Next < 1 {
- pr.Next = 1
- }
- pr.resume()
- return true
-}
-
-func (pr *Progress) pause() { pr.Paused = true }
-func (pr *Progress) resume() { pr.Paused = false }
-
-// IsPaused returns whether sending log entries to this node has been
-// paused. A node may be paused because it has rejected recent
-// MsgApps, is currently waiting for a snapshot, or has reached the
-// MaxInflightMsgs limit.
-func (pr *Progress) IsPaused() bool {
- switch pr.State {
- case ProgressStateProbe:
- return pr.Paused
- case ProgressStateReplicate:
- return pr.ins.full()
- case ProgressStateSnapshot:
- return true
- default:
- panic("unexpected state")
- }
-}
-
-func (pr *Progress) snapshotFailure() { pr.PendingSnapshot = 0 }
-
-// needSnapshotAbort returns true if snapshot progress's Match
-// is equal or higher than the pendingSnapshot.
-func (pr *Progress) needSnapshotAbort() bool {
- return pr.State == ProgressStateSnapshot && pr.Match >= pr.PendingSnapshot
-}
-
-func (pr *Progress) String() string {
- return fmt.Sprintf("next = %d, match = %d, state = %s, waiting = %v, pendingSnapshot = %d", pr.Next, pr.Match, pr.State, pr.IsPaused(), pr.PendingSnapshot)
-}
-
-type inflights struct {
- // the starting index in the buffer
- start int
- // number of inflights in the buffer
- count int
-
- // the size of the buffer
- size int
-
- // buffer contains the index of the last entry
- // inside one message.
- buffer []uint64
-}
-
-func newInflights(size int) *inflights {
- return &inflights{
- size: size,
- }
-}
-
-// add adds an inflight into inflights
-func (in *inflights) add(inflight uint64) {
- if in.full() {
- panic("cannot add into a full inflights")
- }
- next := in.start + in.count
- size := in.size
- if next >= size {
- next -= size
- }
- if next >= len(in.buffer) {
- in.growBuf()
- }
- in.buffer[next] = inflight
- in.count++
-}
-
-// grow the inflight buffer by doubling up to inflights.size. We grow on demand
-// instead of preallocating to inflights.size to handle systems which have
-// thousands of Raft groups per process.
-func (in *inflights) growBuf() {
- newSize := len(in.buffer) * 2
- if newSize == 0 {
- newSize = 1
- } else if newSize > in.size {
- newSize = in.size
- }
- newBuffer := make([]uint64, newSize)
- copy(newBuffer, in.buffer)
- in.buffer = newBuffer
-}
-
-// freeTo frees the inflights smaller or equal to the given `to` flight.
-func (in *inflights) freeTo(to uint64) {
- if in.count == 0 || to < in.buffer[in.start] {
- // out of the left side of the window
- return
- }
-
- idx := in.start
- var i int
- for i = 0; i < in.count; i++ {
- if to < in.buffer[idx] { // found the first large inflight
- break
- }
-
- // increase index and maybe rotate
- size := in.size
- if idx++; idx >= size {
- idx -= size
- }
- }
- // free i inflights and set new start index
- in.count -= i
- in.start = idx
- if in.count == 0 {
- // inflights is empty, reset the start index so that we don't grow the
- // buffer unnecessarily.
- in.start = 0
- }
-}
-
-func (in *inflights) freeFirstOne() { in.freeTo(in.buffer[in.start]) }
-
-// full returns true if the inflights is full.
-func (in *inflights) full() bool {
- return in.count == in.size
-}
-
-// resets frees all inflights.
-func (in *inflights) reset() {
- in.count = 0
- in.start = 0
-}
diff --git a/vendor/go.etcd.io/etcd/raft/quorum/joint.go b/vendor/go.etcd.io/etcd/raft/quorum/joint.go
new file mode 100644
index 0000000..e3741e0
--- /dev/null
+++ b/vendor/go.etcd.io/etcd/raft/quorum/joint.go
@@ -0,0 +1,75 @@
+// Copyright 2019 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package quorum
+
+// JointConfig is a configuration of two groups of (possibly overlapping)
+// majority configurations. Decisions require the support of both majorities.
+type JointConfig [2]MajorityConfig
+
+func (c JointConfig) String() string {
+ if len(c[1]) > 0 {
+ return c[0].String() + "&&" + c[1].String()
+ }
+ return c[0].String()
+}
+
+// IDs returns a newly initialized map representing the set of voters present
+// in the joint configuration.
+func (c JointConfig) IDs() map[uint64]struct{} {
+ m := map[uint64]struct{}{}
+ for _, cc := range c {
+ for id := range cc {
+ m[id] = struct{}{}
+ }
+ }
+ return m
+}
+
+// Describe returns a (multi-line) representation of the commit indexes for the
+// given lookuper.
+func (c JointConfig) Describe(l AckedIndexer) string {
+ return MajorityConfig(c.IDs()).Describe(l)
+}
+
+// CommittedIndex returns the largest committed index for the given joint
+// quorum. An index is jointly committed if it is committed in both constituent
+// majorities.
+func (c JointConfig) CommittedIndex(l AckedIndexer) Index {
+ idx0 := c[0].CommittedIndex(l)
+ idx1 := c[1].CommittedIndex(l)
+ if idx0 < idx1 {
+ return idx0
+ }
+ return idx1
+}
+
+// VoteResult takes a mapping of voters to yes/no (true/false) votes and returns
+// a result indicating whether the vote is pending, lost, or won. A joint quorum
+// requires both majority quorums to vote in favor.
+func (c JointConfig) VoteResult(votes map[uint64]bool) VoteResult {
+ r1 := c[0].VoteResult(votes)
+ r2 := c[1].VoteResult(votes)
+
+ if r1 == r2 {
+ // If they agree, return the agreed state.
+ return r1
+ }
+ if r1 == VoteLost || r2 == VoteLost {
+ // If either config has lost, loss is the only possible outcome.
+ return VoteLost
+ }
+ // One side won, the other one is pending, so the whole outcome is.
+ return VotePending
+}
diff --git a/vendor/go.etcd.io/etcd/raft/quorum/majority.go b/vendor/go.etcd.io/etcd/raft/quorum/majority.go
new file mode 100644
index 0000000..8858a36
--- /dev/null
+++ b/vendor/go.etcd.io/etcd/raft/quorum/majority.go
@@ -0,0 +1,210 @@
+// Copyright 2019 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package quorum
+
+import (
+ "fmt"
+ "math"
+ "sort"
+ "strings"
+)
+
+// MajorityConfig is a set of IDs that uses majority quorums to make decisions.
+type MajorityConfig map[uint64]struct{}
+
+func (c MajorityConfig) String() string {
+ sl := make([]uint64, 0, len(c))
+ for id := range c {
+ sl = append(sl, id)
+ }
+ sort.Slice(sl, func(i, j int) bool { return sl[i] < sl[j] })
+ var buf strings.Builder
+ buf.WriteByte('(')
+ for i := range sl {
+ if i > 0 {
+ buf.WriteByte(' ')
+ }
+ fmt.Fprint(&buf, sl[i])
+ }
+ buf.WriteByte(')')
+ return buf.String()
+}
+
+// Describe returns a (multi-line) representation of the commit indexes for the
+// given lookuper.
+func (c MajorityConfig) Describe(l AckedIndexer) string {
+ if len(c) == 0 {
+ return "<empty majority quorum>"
+ }
+ type tup struct {
+ id uint64
+ idx Index
+ ok bool // idx found?
+ bar int // length of bar displayed for this tup
+ }
+
+ // Below, populate .bar so that the i-th largest commit index has bar i (we
+ // plot this as sort of a progress bar). The actual code is a bit more
+ // complicated and also makes sure that equal index => equal bar.
+
+ n := len(c)
+ info := make([]tup, 0, n)
+ for id := range c {
+ idx, ok := l.AckedIndex(id)
+ info = append(info, tup{id: id, idx: idx, ok: ok})
+ }
+
+ // Sort by index
+ sort.Slice(info, func(i, j int) bool {
+ if info[i].idx == info[j].idx {
+ return info[i].id < info[j].id
+ }
+ return info[i].idx < info[j].idx
+ })
+
+ // Populate .bar.
+ for i := range info {
+ if i > 0 && info[i-1].idx < info[i].idx {
+ info[i].bar = i
+ }
+ }
+
+ // Sort by ID.
+ sort.Slice(info, func(i, j int) bool {
+ return info[i].id < info[j].id
+ })
+
+ var buf strings.Builder
+
+ // Print.
+ fmt.Fprint(&buf, strings.Repeat(" ", n)+" idx\n")
+ for i := range info {
+ bar := info[i].bar
+ if !info[i].ok {
+ fmt.Fprint(&buf, "?"+strings.Repeat(" ", n))
+ } else {
+ fmt.Fprint(&buf, strings.Repeat("x", bar)+">"+strings.Repeat(" ", n-bar))
+ }
+ fmt.Fprintf(&buf, " %5d (id=%d)\n", info[i].idx, info[i].id)
+ }
+ return buf.String()
+}
+
+// Slice returns the MajorityConfig as a sorted slice.
+func (c MajorityConfig) Slice() []uint64 {
+ var sl []uint64
+ for id := range c {
+ sl = append(sl, id)
+ }
+ sort.Slice(sl, func(i, j int) bool { return sl[i] < sl[j] })
+ return sl
+}
+
+func insertionSort(sl []uint64) {
+ a, b := 0, len(sl)
+ for i := a + 1; i < b; i++ {
+ for j := i; j > a && sl[j] < sl[j-1]; j-- {
+ sl[j], sl[j-1] = sl[j-1], sl[j]
+ }
+ }
+}
+
+// CommittedIndex computes the committed index from those supplied via the
+// provided AckedIndexer (for the active config).
+func (c MajorityConfig) CommittedIndex(l AckedIndexer) Index {
+ n := len(c)
+ if n == 0 {
+ // This plays well with joint quorums which, when one half is the zero
+ // MajorityConfig, should behave like the other half.
+ return math.MaxUint64
+ }
+
+ // Use an on-stack slice to collect the committed indexes when n <= 7
+ // (otherwise we alloc). The alternative is to stash a slice on
+ // MajorityConfig, but this impairs usability (as is, MajorityConfig is just
+ // a map, and that's nice). The assumption is that running with a
+ // replication factor of >7 is rare, and in cases in which it happens
+ // performance is a lesser concern (additionally the performance
+ // implications of an allocation here are far from drastic).
+ var stk [7]uint64
+ var srt []uint64
+ if len(stk) >= n {
+ srt = stk[:n]
+ } else {
+ srt = make([]uint64, n)
+ }
+
+ {
+ // Fill the slice with the indexes observed. Any unused slots will be
+ // left as zero; these correspond to voters that may report in, but
+ // haven't yet. We fill from the right (since the zeroes will end up on
+ // the left after sorting below anyway).
+ i := n - 1
+ for id := range c {
+ if idx, ok := l.AckedIndex(id); ok {
+ srt[i] = uint64(idx)
+ i--
+ }
+ }
+ }
+
+ // Sort by index. Use a bespoke algorithm (copied from the stdlib's sort
+ // package) to keep srt on the stack.
+ insertionSort(srt)
+
+ // The smallest index into the array for which the value is acked by a
+ // quorum. In other words, from the end of the slice, move n/2+1 to the
+ // left (accounting for zero-indexing).
+ pos := n - (n/2 + 1)
+ return Index(srt[pos])
+}
+
+// VoteResult takes a mapping of voters to yes/no (true/false) votes and returns
+// a result indicating whether the vote is pending (i.e. neither a quorum of
+// yes/no has been reached), won (a quorum of yes has been reached), or lost (a
+// quorum of no has been reached).
+func (c MajorityConfig) VoteResult(votes map[uint64]bool) VoteResult {
+ if len(c) == 0 {
+ // By convention, the elections on an empty config win. This comes in
+ // handy with joint quorums because it'll make a half-populated joint
+ // quorum behave like a majority quorum.
+ return VoteWon
+ }
+
+ ny := [2]int{} // vote counts for no and yes, respectively
+
+ var missing int
+ for id := range c {
+ v, ok := votes[id]
+ if !ok {
+ missing++
+ continue
+ }
+ if v {
+ ny[1]++
+ } else {
+ ny[0]++
+ }
+ }
+
+ q := len(c)/2 + 1
+ if ny[1] >= q {
+ return VoteWon
+ }
+ if ny[1]+missing >= q {
+ return VotePending
+ }
+ return VoteLost
+}
diff --git a/vendor/go.etcd.io/etcd/raft/quorum/quorum.go b/vendor/go.etcd.io/etcd/raft/quorum/quorum.go
new file mode 100644
index 0000000..2899e46
--- /dev/null
+++ b/vendor/go.etcd.io/etcd/raft/quorum/quorum.go
@@ -0,0 +1,58 @@
+// Copyright 2019 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package quorum
+
+import (
+ "math"
+ "strconv"
+)
+
+// Index is a Raft log position.
+type Index uint64
+
+func (i Index) String() string {
+ if i == math.MaxUint64 {
+ return "∞"
+ }
+ return strconv.FormatUint(uint64(i), 10)
+}
+
+// AckedIndexer allows looking up a commit index for a given ID of a voter
+// from a corresponding MajorityConfig.
+type AckedIndexer interface {
+ AckedIndex(voterID uint64) (idx Index, found bool)
+}
+
+type mapAckIndexer map[uint64]Index
+
+func (m mapAckIndexer) AckedIndex(id uint64) (Index, bool) {
+ idx, ok := m[id]
+ return idx, ok
+}
+
+// VoteResult indicates the outcome of a vote.
+//
+//go:generate stringer -type=VoteResult
+type VoteResult uint8
+
+const (
+ // VotePending indicates that the decision of the vote depends on future
+ // votes, i.e. neither "yes" or "no" has reached quorum yet.
+ VotePending VoteResult = 1 + iota
+ // VoteLost indicates that the quorum has voted "no".
+ VoteLost
+ // VoteWon indicates that the quorum has voted "yes".
+ VoteWon
+)
diff --git a/vendor/go.etcd.io/etcd/raft/quorum/voteresult_string.go b/vendor/go.etcd.io/etcd/raft/quorum/voteresult_string.go
new file mode 100644
index 0000000..9eca8fd
--- /dev/null
+++ b/vendor/go.etcd.io/etcd/raft/quorum/voteresult_string.go
@@ -0,0 +1,26 @@
+// Code generated by "stringer -type=VoteResult"; DO NOT EDIT.
+
+package quorum
+
+import "strconv"
+
+func _() {
+ // An "invalid array index" compiler error signifies that the constant values have changed.
+ // Re-run the stringer command to generate them again.
+ var x [1]struct{}
+ _ = x[VotePending-1]
+ _ = x[VoteLost-2]
+ _ = x[VoteWon-3]
+}
+
+const _VoteResult_name = "VotePendingVoteLostVoteWon"
+
+var _VoteResult_index = [...]uint8{0, 11, 19, 26}
+
+func (i VoteResult) String() string {
+ i -= 1
+ if i >= VoteResult(len(_VoteResult_index)-1) {
+ return "VoteResult(" + strconv.FormatInt(int64(i+1), 10) + ")"
+ }
+ return _VoteResult_name[_VoteResult_index[i]:_VoteResult_index[i+1]]
+}
diff --git a/vendor/go.etcd.io/etcd/raft/raft.go b/vendor/go.etcd.io/etcd/raft/raft.go
index e1e6a16..cdcb43d 100644
--- a/vendor/go.etcd.io/etcd/raft/raft.go
+++ b/vendor/go.etcd.io/etcd/raft/raft.go
@@ -25,7 +25,10 @@
"sync"
"time"
+ "go.etcd.io/etcd/raft/confchange"
+ "go.etcd.io/etcd/raft/quorum"
pb "go.etcd.io/etcd/raft/raftpb"
+ "go.etcd.io/etcd/raft/tracker"
)
// None is a placeholder node ID used when there is no leader.
@@ -261,18 +264,14 @@
maxMsgSize uint64
maxUncommittedSize uint64
- maxInflight int
- prs map[uint64]*Progress
- learnerPrs map[uint64]*Progress
- matchBuf uint64Slice
+ // TODO(tbg): rename to trk.
+ prs tracker.ProgressTracker
state StateType
// isLearner is true if the local raft node is a learner.
isLearner bool
- votes map[uint64]bool
-
msgs []pb.Message
// the leader id
@@ -330,28 +329,26 @@
if err != nil {
panic(err) // TODO(bdarnell)
}
- peers := c.peers
- learners := c.learners
- if len(cs.Nodes) > 0 || len(cs.Learners) > 0 {
- if len(peers) > 0 || len(learners) > 0 {
+
+ if len(c.peers) > 0 || len(c.learners) > 0 {
+ if len(cs.Voters) > 0 || len(cs.Learners) > 0 {
// TODO(bdarnell): the peers argument is always nil except in
// tests; the argument should be removed and these tests should be
// updated to specify their nodes through a snapshot.
- panic("cannot specify both newRaft(peers, learners) and ConfState.(Nodes, Learners)")
+ panic("cannot specify both newRaft(peers, learners) and ConfState.(Voters, Learners)")
}
- peers = cs.Nodes
- learners = cs.Learners
+ cs.Voters = c.peers
+ cs.Learners = c.learners
}
+
r := &raft{
id: c.ID,
lead: None,
isLearner: false,
raftLog: raftlog,
maxMsgSize: c.MaxSizePerMsg,
- maxInflight: c.MaxInflightMsgs,
maxUncommittedSize: c.MaxUncommittedEntriesSize,
- prs: make(map[uint64]*Progress),
- learnerPrs: make(map[uint64]*Progress),
+ prs: tracker.MakeProgressTracker(c.MaxInflightMsgs),
electionTimeout: c.ElectionTick,
heartbeatTimeout: c.HeartbeatTick,
logger: c.Logger,
@@ -360,20 +357,17 @@
readOnly: newReadOnly(c.ReadOnlyOption),
disableProposalForwarding: c.DisableProposalForwarding,
}
- for _, p := range peers {
- r.prs[p] = &Progress{Next: 1, ins: newInflights(r.maxInflight)}
- }
- for _, p := range learners {
- if _, ok := r.prs[p]; ok {
- panic(fmt.Sprintf("node %x is in both learner and peer list", p))
- }
- r.learnerPrs[p] = &Progress{Next: 1, ins: newInflights(r.maxInflight), IsLearner: true}
- if r.id == p {
- r.isLearner = true
- }
- }
- if !isHardStateEqual(hs, emptyState) {
+ cfg, prs, err := confchange.Restore(confchange.Changer{
+ Tracker: r.prs,
+ LastIndex: raftlog.lastIndex(),
+ }, cs)
+ if err != nil {
+ panic(err)
+ }
+ assertConfStatesEquivalent(r.logger, cs, r.switchToConfig(cfg, prs))
+
+ if !IsEmptyHardState(hs) {
r.loadState(hs)
}
if c.Applied > 0 {
@@ -382,7 +376,7 @@
r.becomeFollower(r.Term, None)
var nodesStrs []string
- for _, n := range r.nodes() {
+ for _, n := range r.prs.VoterNodes() {
nodesStrs = append(nodesStrs, fmt.Sprintf("%x", n))
}
@@ -403,26 +397,6 @@
}
}
-func (r *raft) quorum() int { return len(r.prs)/2 + 1 }
-
-func (r *raft) nodes() []uint64 {
- nodes := make([]uint64, 0, len(r.prs))
- for id := range r.prs {
- nodes = append(nodes, id)
- }
- sort.Sort(uint64Slice(nodes))
- return nodes
-}
-
-func (r *raft) learnerNodes() []uint64 {
- nodes := make([]uint64, 0, len(r.learnerPrs))
- for id := range r.learnerPrs {
- nodes = append(nodes, id)
- }
- sort.Sort(uint64Slice(nodes))
- return nodes
-}
-
// send persists state to stable storage and then sends to its mailbox.
func (r *raft) send(m pb.Message) {
m.From = r.id
@@ -457,14 +431,6 @@
r.msgs = append(r.msgs, m)
}
-func (r *raft) getProgress(id uint64) *Progress {
- if pr, ok := r.prs[id]; ok {
- return pr
- }
-
- return r.learnerPrs[id]
-}
-
// sendAppend sends an append RPC with new entries (if any) and the
// current commit index to the given peer.
func (r *raft) sendAppend(to uint64) {
@@ -477,7 +443,7 @@
// ("empty" messages are useful to convey updated Commit indexes, but
// are undesirable when we're sending multiple messages in a batch).
func (r *raft) maybeSendAppend(to uint64, sendIfEmpty bool) bool {
- pr := r.getProgress(to)
+ pr := r.prs.Progress[to]
if pr.IsPaused() {
return false
}
@@ -512,7 +478,7 @@
sindex, sterm := snapshot.Metadata.Index, snapshot.Metadata.Term
r.logger.Debugf("%x [firstindex: %d, commit: %d] sent snapshot[index: %d, term: %d] to %x [%s]",
r.id, r.raftLog.firstIndex(), r.raftLog.committed, sindex, sterm, to, pr)
- pr.becomeSnapshot(sindex)
+ pr.BecomeSnapshot(sindex)
r.logger.Debugf("%x paused sending replication messages to %x [%s]", r.id, to, pr)
} else {
m.Type = pb.MsgApp
@@ -522,13 +488,13 @@
m.Commit = r.raftLog.committed
if n := len(m.Entries); n != 0 {
switch pr.State {
- // optimistically increase the next when in ProgressStateReplicate
- case ProgressStateReplicate:
+ // optimistically increase the next when in StateReplicate
+ case tracker.StateReplicate:
last := m.Entries[n-1].Index
- pr.optimisticUpdate(last)
- pr.ins.add(last)
- case ProgressStateProbe:
- pr.pause()
+ pr.OptimisticUpdate(last)
+ pr.Inflights.Add(last)
+ case tracker.StateProbe:
+ pr.ProbeSent = true
default:
r.logger.Panicf("%x is sending append in unhandled state %s", r.id, pr.State)
}
@@ -546,7 +512,7 @@
// or it might not have all the committed entries.
// The leader MUST NOT forward the follower's commit to
// an unmatched index.
- commit := min(r.getProgress(to).Match, r.raftLog.committed)
+ commit := min(r.prs.Progress[to].Match, r.raftLog.committed)
m := pb.Message{
To: to,
Type: pb.MsgHeartbeat,
@@ -557,24 +523,13 @@
r.send(m)
}
-func (r *raft) forEachProgress(f func(id uint64, pr *Progress)) {
- for id, pr := range r.prs {
- f(id, pr)
- }
-
- for id, pr := range r.learnerPrs {
- f(id, pr)
- }
-}
-
// bcastAppend sends RPC, with entries to all peers that are not up-to-date
// according to the progress recorded in r.prs.
func (r *raft) bcastAppend() {
- r.forEachProgress(func(id uint64, _ *Progress) {
+ r.prs.Visit(func(id uint64, _ *tracker.Progress) {
if id == r.id {
return
}
-
r.sendAppend(id)
})
}
@@ -590,7 +545,7 @@
}
func (r *raft) bcastHeartbeatWithCtx(ctx []byte) {
- r.forEachProgress(func(id uint64, _ *Progress) {
+ r.prs.Visit(func(id uint64, _ *tracker.Progress) {
if id == r.id {
return
}
@@ -598,23 +553,51 @@
})
}
+func (r *raft) advance(rd Ready) {
+ // If entries were applied (or a snapshot), update our cursor for
+ // the next Ready. Note that if the current HardState contains a
+ // new Commit index, this does not mean that we're also applying
+ // all of the new entries due to commit pagination by size.
+ if index := rd.appliedCursor(); index > 0 {
+ r.raftLog.appliedTo(index)
+ if r.prs.Config.AutoLeave && index >= r.pendingConfIndex && r.state == StateLeader {
+ // If the current (and most recent, at least for this leader's term)
+ // configuration should be auto-left, initiate that now.
+ ccdata, err := (&pb.ConfChangeV2{}).Marshal()
+ if err != nil {
+ panic(err)
+ }
+ ent := pb.Entry{
+ Type: pb.EntryConfChangeV2,
+ Data: ccdata,
+ }
+ if !r.appendEntry(ent) {
+ // If we could not append the entry, bump the pending conf index
+ // so that we'll try again later.
+ //
+ // TODO(tbg): test this case.
+ r.pendingConfIndex = r.raftLog.lastIndex()
+ } else {
+ r.logger.Infof("initiating automatic transition out of joint configuration %s", r.prs.Config)
+ }
+ }
+ }
+ r.reduceUncommittedSize(rd.CommittedEntries)
+
+ if len(rd.Entries) > 0 {
+ e := rd.Entries[len(rd.Entries)-1]
+ r.raftLog.stableTo(e.Index, e.Term)
+ }
+ if !IsEmptySnap(rd.Snapshot) {
+ r.raftLog.stableSnapTo(rd.Snapshot.Metadata.Index)
+ }
+}
+
// maybeCommit attempts to advance the commit index. Returns true if
// the commit index changed (in which case the caller should call
// r.bcastAppend).
func (r *raft) maybeCommit() bool {
- // Preserving matchBuf across calls is an optimization
- // used to avoid allocating a new slice on each call.
- if cap(r.matchBuf) < len(r.prs) {
- r.matchBuf = make(uint64Slice, len(r.prs))
- }
- mis := r.matchBuf[:len(r.prs)]
- idx := 0
- for _, p := range r.prs {
- mis[idx] = p.Match
- idx++
- }
- sort.Sort(mis)
- mci := mis[len(mis)-r.quorum()]
+ mci := r.prs.Committed()
return r.raftLog.maybeCommit(mci, r.Term)
}
@@ -631,9 +614,14 @@
r.abortLeaderTransfer()
- r.votes = make(map[uint64]bool)
- r.forEachProgress(func(id uint64, pr *Progress) {
- *pr = Progress{Next: r.raftLog.lastIndex() + 1, ins: newInflights(r.maxInflight), IsLearner: pr.IsLearner}
+ r.prs.ResetVotes()
+ r.prs.Visit(func(id uint64, pr *tracker.Progress) {
+ *pr = tracker.Progress{
+ Match: 0,
+ Next: r.raftLog.lastIndex() + 1,
+ Inflights: tracker.NewInflights(r.prs.MaxInflight),
+ IsLearner: pr.IsLearner,
+ }
if id == r.id {
pr.Match = r.raftLog.lastIndex()
}
@@ -661,7 +649,7 @@
}
// use latest "last" index after truncate/append
li = r.raftLog.append(es...)
- r.getProgress(r.id).maybeUpdate(li)
+ r.prs.Progress[r.id].MaybeUpdate(li)
// Regardless of maybeCommit's return, our caller will call bcastAppend.
r.maybeCommit()
return true
@@ -734,7 +722,7 @@
// but doesn't change anything else. In particular it does not increase
// r.Term or change r.Vote.
r.step = stepCandidate
- r.votes = make(map[uint64]bool)
+ r.prs.ResetVotes()
r.tick = r.tickElection
r.lead = None
r.state = StatePreCandidate
@@ -755,7 +743,7 @@
// (perhaps after having received a snapshot as a result). The leader is
// trivially in this state. Note that r.reset() has initialized this
// progress with the last index already.
- r.prs[r.id].becomeReplicate()
+ r.prs.Progress[r.id].BecomeReplicate()
// Conservatively set the pendingConfIndex to the last index in the
// log. There may or may not be a pending config change, but it's
@@ -777,7 +765,14 @@
r.logger.Infof("%x became leader at term %d", r.id, r.Term)
}
+// campaign transitions the raft instance to candidate state. This must only be
+// called after verifying that this is a legitimate transition.
func (r *raft) campaign(t CampaignType) {
+ if !r.promotable() {
+ // This path should not be hit (callers are supposed to check), but
+ // better safe than sorry.
+ r.logger.Warningf("%x is unpromotable; campaign() should have been called", r.id)
+ }
var term uint64
var voteMsg pb.MessageType
if t == campaignPreElection {
@@ -790,7 +785,7 @@
voteMsg = pb.MsgVote
term = r.Term
}
- if r.quorum() == r.poll(r.id, voteRespMsgType(voteMsg), true) {
+ if _, _, res := r.poll(r.id, voteRespMsgType(voteMsg), true); res == quorum.VoteWon {
// We won the election after voting for ourselves (which must mean that
// this is a single-node cluster). Advance to the next state.
if t == campaignPreElection {
@@ -800,7 +795,16 @@
}
return
}
- for id := range r.prs {
+ var ids []uint64
+ {
+ idMap := r.prs.Voters.IDs()
+ ids = make([]uint64, 0, len(idMap))
+ for id := range idMap {
+ ids = append(ids, id)
+ }
+ sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] })
+ }
+ for _, id := range ids {
if id == r.id {
continue
}
@@ -815,21 +819,14 @@
}
}
-func (r *raft) poll(id uint64, t pb.MessageType, v bool) (granted int) {
+func (r *raft) poll(id uint64, t pb.MessageType, v bool) (granted int, rejected int, result quorum.VoteResult) {
if v {
r.logger.Infof("%x received %s from %x at term %d", r.id, t, id, r.Term)
} else {
r.logger.Infof("%x received %s rejection from %x at term %d", r.id, t, id, r.Term)
}
- if _, ok := r.votes[id]; !ok {
- r.votes[id] = v
- }
- for _, vv := range r.votes {
- if vv {
- granted++
- }
- }
- return granted
+ r.prs.RecordVote(id, v)
+ return r.prs.TallyVotes()
}
func (r *raft) Step(m pb.Message) error {
@@ -910,6 +907,10 @@
switch m.Type {
case pb.MsgHup:
if r.state != StateLeader {
+ if !r.promotable() {
+ r.logger.Warningf("%x is unpromotable and can not campaign; ignoring MsgHup", r.id)
+ return nil
+ }
ents, err := r.raftLog.slice(r.raftLog.applied+1, r.raftLog.committed+1, noLimit)
if err != nil {
r.logger.Panicf("unexpected error getting unapplied entries (%v)", err)
@@ -930,12 +931,6 @@
}
case pb.MsgVote, pb.MsgPreVote:
- if r.isLearner {
- // TODO: learner may need to vote, in case of node down when confchange.
- r.logger.Infof("%x [logterm: %d, index: %d, vote: %x] ignored %s from %x [logterm: %d, index: %d] at term %d: learner can not vote",
- r.id, r.raftLog.lastTerm(), r.raftLog.lastIndex(), r.Vote, m.Type, m.From, m.LogTerm, m.Index, r.Term)
- return nil
- }
// We can vote if this is a repeat of a vote we've already cast...
canVote := r.Vote == m.From ||
// ...we haven't voted and we don't think there's a leader yet in this term...
@@ -944,12 +939,30 @@
(m.Type == pb.MsgPreVote && m.Term > r.Term)
// ...and we believe the candidate is up to date.
if canVote && r.raftLog.isUpToDate(m.Index, m.LogTerm) {
+ // Note: it turns out that that learners must be allowed to cast votes.
+ // This seems counter- intuitive but is necessary in the situation in which
+ // a learner has been promoted (i.e. is now a voter) but has not learned
+ // about this yet.
+ // For example, consider a group in which id=1 is a learner and id=2 and
+ // id=3 are voters. A configuration change promoting 1 can be committed on
+ // the quorum `{2,3}` without the config change being appended to the
+ // learner's log. If the leader (say 2) fails, there are de facto two
+ // voters remaining. Only 3 can win an election (due to its log containing
+ // all committed entries), but to do so it will need 1 to vote. But 1
+ // considers itself a learner and will continue to do so until 3 has
+ // stepped up as leader, replicates the conf change to 1, and 1 applies it.
+ // Ultimately, by receiving a request to vote, the learner realizes that
+ // the candidate believes it to be a voter, and that it should act
+ // accordingly. The candidate's config may be stale, too; but in that case
+ // it won't win the election, at least in the absence of the bug discussed
+ // in:
+ // https://github.com/etcd-io/etcd/issues/7625#issuecomment-488798263.
r.logger.Infof("%x [logterm: %d, index: %d, vote: %x] cast %s for %x [logterm: %d, index: %d] at term %d",
r.id, r.raftLog.lastTerm(), r.raftLog.lastIndex(), r.Vote, m.Type, m.From, m.LogTerm, m.Index, r.Term)
// When responding to Msg{Pre,}Vote messages we include the term
- // from the message, not the local term. To see why consider the
+ // from the message, not the local term. To see why, consider the
// case where a single node was previously partitioned away and
- // it's local term is now of date. If we include the local term
+ // it's local term is now out of date. If we include the local term
// (recall that for pre-votes we don't update the local term), the
// (pre-)campaigning node on the other end will proceed to ignore
// the message (it ignores all out of date messages).
@@ -985,16 +998,32 @@
r.bcastHeartbeat()
return nil
case pb.MsgCheckQuorum:
- if !r.checkQuorumActive() {
+ // The leader should always see itself as active. As a precaution, handle
+ // the case in which the leader isn't in the configuration any more (for
+ // example if it just removed itself).
+ //
+ // TODO(tbg): I added a TODO in removeNode, it doesn't seem that the
+ // leader steps down when removing itself. I might be missing something.
+ if pr := r.prs.Progress[r.id]; pr != nil {
+ pr.RecentActive = true
+ }
+ if !r.prs.QuorumActive() {
r.logger.Warningf("%x stepped down to follower since quorum is not active", r.id)
r.becomeFollower(r.Term, None)
}
+ // Mark everyone (but ourselves) as inactive in preparation for the next
+ // CheckQuorum.
+ r.prs.Visit(func(id uint64, pr *tracker.Progress) {
+ if id != r.id {
+ pr.RecentActive = false
+ }
+ })
return nil
case pb.MsgProp:
if len(m.Entries) == 0 {
r.logger.Panicf("%x stepped empty MsgProp", r.id)
}
- if _, ok := r.prs[r.id]; !ok {
+ if r.prs.Progress[r.id] == nil {
// If we are not currently a member of the range (i.e. this node
// was removed from the configuration while serving as leader),
// drop any new proposals.
@@ -1005,11 +1034,38 @@
return ErrProposalDropped
}
- for i, e := range m.Entries {
+ for i := range m.Entries {
+ e := &m.Entries[i]
+ var cc pb.ConfChangeI
if e.Type == pb.EntryConfChange {
- if r.pendingConfIndex > r.raftLog.applied {
- r.logger.Infof("propose conf %s ignored since pending unapplied configuration [index %d, applied %d]",
- e.String(), r.pendingConfIndex, r.raftLog.applied)
+ var ccc pb.ConfChange
+ if err := ccc.Unmarshal(e.Data); err != nil {
+ panic(err)
+ }
+ cc = ccc
+ } else if e.Type == pb.EntryConfChangeV2 {
+ var ccc pb.ConfChangeV2
+ if err := ccc.Unmarshal(e.Data); err != nil {
+ panic(err)
+ }
+ cc = ccc
+ }
+ if cc != nil {
+ alreadyPending := r.pendingConfIndex > r.raftLog.applied
+ alreadyJoint := len(r.prs.Config.Voters[1]) > 0
+ wantsLeaveJoint := len(cc.AsV2().Changes) == 0
+
+ var refused string
+ if alreadyPending {
+ refused = fmt.Sprintf("possible unapplied conf change at index %d (applied to %d)", r.pendingConfIndex, r.raftLog.applied)
+ } else if alreadyJoint && !wantsLeaveJoint {
+ refused = "must transition out of joint config first"
+ } else if !alreadyJoint && wantsLeaveJoint {
+ refused = "not in joint state; refusing empty conf change"
+ }
+
+ if refused != "" {
+ r.logger.Infof("%x ignoring conf change %v at config %s: %s", r.id, cc, r.prs.Config, refused)
m.Entries[i] = pb.Entry{Type: pb.EntryNormal}
} else {
r.pendingConfIndex = r.raftLog.lastIndex() + uint64(i) + 1
@@ -1023,7 +1079,9 @@
r.bcastAppend()
return nil
case pb.MsgReadIndex:
- if r.quorum() > 1 {
+ // If more than the local vote is needed, go through a full broadcast,
+ // otherwise optimize.
+ if !r.prs.IsSingleton() {
if r.raftLog.zeroTermOnErrCompacted(r.raftLog.term(r.raftLog.committed)) != r.Term {
// Reject read only request when this leader has not committed any log entry at its term.
return nil
@@ -1035,24 +1093,30 @@
switch r.readOnly.option {
case ReadOnlySafe:
r.readOnly.addRequest(r.raftLog.committed, m)
+ // The local node automatically acks the request.
+ r.readOnly.recvAck(r.id, m.Entries[0].Data)
r.bcastHeartbeatWithCtx(m.Entries[0].Data)
case ReadOnlyLeaseBased:
ri := r.raftLog.committed
if m.From == None || m.From == r.id { // from local member
- r.readStates = append(r.readStates, ReadState{Index: r.raftLog.committed, RequestCtx: m.Entries[0].Data})
+ r.readStates = append(r.readStates, ReadState{Index: ri, RequestCtx: m.Entries[0].Data})
} else {
r.send(pb.Message{To: m.From, Type: pb.MsgReadIndexResp, Index: ri, Entries: m.Entries})
}
}
- } else {
- r.readStates = append(r.readStates, ReadState{Index: r.raftLog.committed, RequestCtx: m.Entries[0].Data})
+ } else { // only one voting member (the leader) in the cluster
+ if m.From == None || m.From == r.id { // from leader itself
+ r.readStates = append(r.readStates, ReadState{Index: r.raftLog.committed, RequestCtx: m.Entries[0].Data})
+ } else { // from learner member
+ r.send(pb.Message{To: m.From, Type: pb.MsgReadIndexResp, Index: r.raftLog.committed, Entries: m.Entries})
+ }
}
return nil
}
// All other message types require a progress for m.From (pr).
- pr := r.getProgress(m.From)
+ pr := r.prs.Progress[m.From]
if pr == nil {
r.logger.Debugf("%x no progress available for %x", r.id, m.From)
return nil
@@ -1062,32 +1126,35 @@
pr.RecentActive = true
if m.Reject {
- r.logger.Debugf("%x received msgApp rejection(lastindex: %d) from %x for index %d",
+ r.logger.Debugf("%x received MsgAppResp(MsgApp was rejected, lastindex: %d) from %x for index %d",
r.id, m.RejectHint, m.From, m.Index)
- if pr.maybeDecrTo(m.Index, m.RejectHint) {
+ if pr.MaybeDecrTo(m.Index, m.RejectHint) {
r.logger.Debugf("%x decreased progress of %x to [%s]", r.id, m.From, pr)
- if pr.State == ProgressStateReplicate {
- pr.becomeProbe()
+ if pr.State == tracker.StateReplicate {
+ pr.BecomeProbe()
}
r.sendAppend(m.From)
}
} else {
oldPaused := pr.IsPaused()
- if pr.maybeUpdate(m.Index) {
+ if pr.MaybeUpdate(m.Index) {
switch {
- case pr.State == ProgressStateProbe:
- pr.becomeReplicate()
- case pr.State == ProgressStateSnapshot && pr.needSnapshotAbort():
- r.logger.Debugf("%x snapshot aborted, resumed sending replication messages to %x [%s]", r.id, m.From, pr)
+ case pr.State == tracker.StateProbe:
+ pr.BecomeReplicate()
+ case pr.State == tracker.StateSnapshot && pr.Match >= pr.PendingSnapshot:
+ // TODO(tbg): we should also enter this branch if a snapshot is
+ // received that is below pr.PendingSnapshot but which makes it
+ // possible to use the log again.
+ r.logger.Debugf("%x recovered from needing snapshot, resumed sending replication messages to %x [%s]", r.id, m.From, pr)
// Transition back to replicating state via probing state
// (which takes the snapshot into account). If we didn't
// move to replicating state, that would only happen with
// the next round of appends (but there may not be a next
// round for a while, exposing an inconsistent RaftStatus).
- pr.becomeProbe()
- pr.becomeReplicate()
- case pr.State == ProgressStateReplicate:
- pr.ins.freeTo(m.Index)
+ pr.BecomeProbe()
+ pr.BecomeReplicate()
+ case pr.State == tracker.StateReplicate:
+ pr.Inflights.FreeLE(m.Index)
}
if r.maybeCommit() {
@@ -1114,11 +1181,11 @@
}
case pb.MsgHeartbeatResp:
pr.RecentActive = true
- pr.resume()
+ pr.ProbeSent = false
// free one slot for the full inflights window to allow progress.
- if pr.State == ProgressStateReplicate && pr.ins.full() {
- pr.ins.freeFirstOne()
+ if pr.State == tracker.StateReplicate && pr.Inflights.Full() {
+ pr.Inflights.FreeFirstOne()
}
if pr.Match < r.raftLog.lastIndex() {
r.sendAppend(m.From)
@@ -1128,8 +1195,7 @@
return nil
}
- ackCount := r.readOnly.recvAck(m)
- if ackCount < r.quorum() {
+ if r.prs.Voters.VoteResult(r.readOnly.recvAck(m.From, m.Context)) != quorum.VoteWon {
return nil
}
@@ -1143,26 +1209,32 @@
}
}
case pb.MsgSnapStatus:
- if pr.State != ProgressStateSnapshot {
+ if pr.State != tracker.StateSnapshot {
return nil
}
+ // TODO(tbg): this code is very similar to the snapshot handling in
+ // MsgAppResp above. In fact, the code there is more correct than the
+ // code here and should likely be updated to match (or even better, the
+ // logic pulled into a newly created Progress state machine handler).
if !m.Reject {
- pr.becomeProbe()
+ pr.BecomeProbe()
r.logger.Debugf("%x snapshot succeeded, resumed sending replication messages to %x [%s]", r.id, m.From, pr)
} else {
- pr.snapshotFailure()
- pr.becomeProbe()
+ // NB: the order here matters or we'll be probing erroneously from
+ // the snapshot index, but the snapshot never applied.
+ pr.PendingSnapshot = 0
+ pr.BecomeProbe()
r.logger.Debugf("%x snapshot failed, resumed sending replication messages to %x [%s]", r.id, m.From, pr)
}
- // If snapshot finish, wait for the msgAppResp from the remote node before sending
- // out the next msgApp.
+ // If snapshot finish, wait for the MsgAppResp from the remote node before sending
+ // out the next MsgApp.
// If snapshot failure, wait for a heartbeat interval before next try
- pr.pause()
+ pr.ProbeSent = true
case pb.MsgUnreachable:
// During optimistic replication, if the remote becomes unreachable,
// there is huge probability that a MsgApp is lost.
- if pr.State == ProgressStateReplicate {
- pr.becomeProbe()
+ if pr.State == tracker.StateReplicate {
+ pr.BecomeProbe()
}
r.logger.Debugf("%x failed to send message to %x because it is unreachable [%s]", r.id, m.From, pr)
case pb.MsgTransferLeader:
@@ -1226,17 +1298,17 @@
r.becomeFollower(m.Term, m.From) // always m.Term == r.Term
r.handleSnapshot(m)
case myVoteRespType:
- gr := r.poll(m.From, m.Type, !m.Reject)
- r.logger.Infof("%x [quorum:%d] has received %d %s votes and %d vote rejections", r.id, r.quorum(), gr, m.Type, len(r.votes)-gr)
- switch r.quorum() {
- case gr:
+ gr, rj, res := r.poll(m.From, m.Type, !m.Reject)
+ r.logger.Infof("%x has received %d %s votes and %d vote rejections", r.id, gr, m.Type, rj)
+ switch res {
+ case quorum.VoteWon:
if r.state == StatePreCandidate {
r.campaign(campaignElection)
} else {
r.becomeLeader()
r.bcastAppend()
}
- case len(r.votes) - gr:
+ case quorum.VoteLost:
// pb.MsgPreVoteResp contains future term of pre-candidate
// m.Term > r.Term; reuse r.Term
r.becomeFollower(r.Term, None)
@@ -1314,7 +1386,7 @@
if mlastIndex, ok := r.raftLog.maybeAppend(m.Index, m.LogTerm, m.Commit, m.Entries...); ok {
r.send(pb.Message{To: m.From, Type: pb.MsgAppResp, Index: mlastIndex})
} else {
- r.logger.Debugf("%x [logterm: %d, index: %d] rejected msgApp [logterm: %d, index: %d] from %x",
+ r.logger.Debugf("%x [logterm: %d, index: %d] rejected MsgApp [logterm: %d, index: %d] from %x",
r.id, r.raftLog.zeroTermOnErrCompacted(r.raftLog.term(m.Index)), m.Index, m.LogTerm, m.Index, m.From)
r.send(pb.Message{To: m.From, Type: pb.MsgAppResp, Index: m.Index, Reject: true, RejectHint: r.raftLog.lastIndex()})
}
@@ -1339,11 +1411,51 @@
}
// restore recovers the state machine from a snapshot. It restores the log and the
-// configuration of state machine.
+// configuration of state machine. If this method returns false, the snapshot was
+// ignored, either because it was obsolete or because of an error.
func (r *raft) restore(s pb.Snapshot) bool {
if s.Metadata.Index <= r.raftLog.committed {
return false
}
+ if r.state != StateFollower {
+ // This is defense-in-depth: if the leader somehow ended up applying a
+ // snapshot, it could move into a new term without moving into a
+ // follower state. This should never fire, but if it did, we'd have
+ // prevented damage by returning early, so log only a loud warning.
+ //
+ // At the time of writing, the instance is guaranteed to be in follower
+ // state when this method is called.
+ r.logger.Warningf("%x attempted to restore snapshot as leader; should never happen", r.id)
+ r.becomeFollower(r.Term+1, None)
+ return false
+ }
+
+ // More defense-in-depth: throw away snapshot if recipient is not in the
+ // config. This shouldn't ever happen (at the time of writing) but lots of
+ // code here and there assumes that r.id is in the progress tracker.
+ found := false
+ cs := s.Metadata.ConfState
+ for _, set := range [][]uint64{
+ cs.Voters,
+ cs.Learners,
+ } {
+ for _, id := range set {
+ if id == r.id {
+ found = true
+ break
+ }
+ }
+ }
+ if !found {
+ r.logger.Warningf(
+ "%x attempted to restore snapshot but it is not in the ConfState %v; should never happen",
+ r.id, cs,
+ )
+ return false
+ }
+
+ // Now go ahead and actually restore.
+
if r.raftLog.matchTerm(s.Metadata.Index, s.Metadata.Term) {
r.logger.Infof("%x [commit: %d, lastindex: %d, lastterm: %d] fast-forwarded commit to snapshot [index: %d, term: %d]",
r.id, r.raftLog.committed, r.raftLog.lastIndex(), r.raftLog.lastTerm(), s.Metadata.Index, s.Metadata.Term)
@@ -1351,123 +1463,115 @@
return false
}
- // The normal peer can't become learner.
- if !r.isLearner {
- for _, id := range s.Metadata.ConfState.Learners {
- if id == r.id {
- r.logger.Errorf("%x can't become learner when restores snapshot [index: %d, term: %d]", r.id, s.Metadata.Index, s.Metadata.Term)
- return false
- }
- }
- }
-
- r.logger.Infof("%x [commit: %d, lastindex: %d, lastterm: %d] starts to restore snapshot [index: %d, term: %d]",
- r.id, r.raftLog.committed, r.raftLog.lastIndex(), r.raftLog.lastTerm(), s.Metadata.Index, s.Metadata.Term)
-
r.raftLog.restore(s)
- r.prs = make(map[uint64]*Progress)
- r.learnerPrs = make(map[uint64]*Progress)
- r.restoreNode(s.Metadata.ConfState.Nodes, false)
- r.restoreNode(s.Metadata.ConfState.Learners, true)
- return true
-}
-func (r *raft) restoreNode(nodes []uint64, isLearner bool) {
- for _, n := range nodes {
- match, next := uint64(0), r.raftLog.lastIndex()+1
- if n == r.id {
- match = next - 1
- r.isLearner = isLearner
- }
- r.setProgress(n, match, next, isLearner)
- r.logger.Infof("%x restored progress of %x [%s]", r.id, n, r.getProgress(n))
+ // Reset the configuration and add the (potentially updated) peers in anew.
+ r.prs = tracker.MakeProgressTracker(r.prs.MaxInflight)
+ cfg, prs, err := confchange.Restore(confchange.Changer{
+ Tracker: r.prs,
+ LastIndex: r.raftLog.lastIndex(),
+ }, cs)
+
+ if err != nil {
+ // This should never happen. Either there's a bug in our config change
+ // handling or the client corrupted the conf change.
+ panic(fmt.Sprintf("unable to restore config %+v: %s", cs, err))
}
+
+ assertConfStatesEquivalent(r.logger, cs, r.switchToConfig(cfg, prs))
+
+ pr := r.prs.Progress[r.id]
+ pr.MaybeUpdate(pr.Next - 1) // TODO(tbg): this is untested and likely unneeded
+
+ r.logger.Infof("%x [commit: %d, lastindex: %d, lastterm: %d] restored snapshot [index: %d, term: %d]",
+ r.id, r.raftLog.committed, r.raftLog.lastIndex(), r.raftLog.lastTerm(), s.Metadata.Index, s.Metadata.Term)
+ return true
}
// promotable indicates whether state machine can be promoted to leader,
// which is true when its own id is in progress list.
func (r *raft) promotable() bool {
- _, ok := r.prs[r.id]
- return ok
+ pr := r.prs.Progress[r.id]
+ return pr != nil && !pr.IsLearner
}
-func (r *raft) addNode(id uint64) {
- r.addNodeOrLearnerNode(id, false)
-}
-
-func (r *raft) addLearner(id uint64) {
- r.addNodeOrLearnerNode(id, true)
-}
-
-func (r *raft) addNodeOrLearnerNode(id uint64, isLearner bool) {
- pr := r.getProgress(id)
- if pr == nil {
- r.setProgress(id, 0, r.raftLog.lastIndex()+1, isLearner)
- } else {
- if isLearner && !pr.IsLearner {
- // can only change Learner to Voter
- r.logger.Infof("%x ignored addLearner: do not support changing %x from raft peer to learner.", r.id, id)
- return
+func (r *raft) applyConfChange(cc pb.ConfChangeV2) pb.ConfState {
+ cfg, prs, err := func() (tracker.Config, tracker.ProgressMap, error) {
+ changer := confchange.Changer{
+ Tracker: r.prs,
+ LastIndex: r.raftLog.lastIndex(),
}
-
- if isLearner == pr.IsLearner {
- // Ignore any redundant addNode calls (which can happen because the
- // initial bootstrapping entries are applied twice).
- return
+ if cc.LeaveJoint() {
+ return changer.LeaveJoint()
+ } else if autoLeave, ok := cc.EnterJoint(); ok {
+ return changer.EnterJoint(autoLeave, cc.Changes...)
}
+ return changer.Simple(cc.Changes...)
+ }()
- // change Learner to Voter, use origin Learner progress
- delete(r.learnerPrs, id)
- pr.IsLearner = false
- r.prs[id] = pr
+ if err != nil {
+ // TODO(tbg): return the error to the caller.
+ panic(err)
}
- if r.id == id {
- r.isLearner = isLearner
- }
-
- // When a node is first added, we should mark it as recently active.
- // Otherwise, CheckQuorum may cause us to step down if it is invoked
- // before the added node has a chance to communicate with us.
- pr = r.getProgress(id)
- pr.RecentActive = true
+ return r.switchToConfig(cfg, prs)
}
-func (r *raft) removeNode(id uint64) {
- r.delProgress(id)
+// switchToConfig reconfigures this node to use the provided configuration. It
+// updates the in-memory state and, when necessary, carries out additional
+// actions such as reacting to the removal of nodes or changed quorum
+// requirements.
+//
+// The inputs usually result from restoring a ConfState or applying a ConfChange.
+func (r *raft) switchToConfig(cfg tracker.Config, prs tracker.ProgressMap) pb.ConfState {
+ r.prs.Config = cfg
+ r.prs.Progress = prs
- // do not try to commit or abort transferring if there is no nodes in the cluster.
- if len(r.prs) == 0 && len(r.learnerPrs) == 0 {
- return
+ r.logger.Infof("%x switched to configuration %s", r.id, r.prs.Config)
+ cs := r.prs.ConfState()
+ pr, ok := r.prs.Progress[r.id]
+
+ // Update whether the node itself is a learner, resetting to false when the
+ // node is removed.
+ r.isLearner = ok && pr.IsLearner
+
+ if (!ok || r.isLearner) && r.state == StateLeader {
+ // This node is leader and was removed or demoted. We prevent demotions
+ // at the time writing but hypothetically we handle them the same way as
+ // removing the leader: stepping down into the next Term.
+ //
+ // TODO(tbg): step down (for sanity) and ask follower with largest Match
+ // to TimeoutNow (to avoid interruption). This might still drop some
+ // proposals but it's better than nothing.
+ //
+ // TODO(tbg): test this branch. It is untested at the time of writing.
+ return cs
}
- // The quorum size is now smaller, so see if any pending entries can
- // be committed.
+ // The remaining steps only make sense if this node is the leader and there
+ // are other nodes.
+ if r.state != StateLeader || len(cs.Voters) == 0 {
+ return cs
+ }
+
if r.maybeCommit() {
+ // If the configuration change means that more entries are committed now,
+ // broadcast/append to everyone in the updated config.
r.bcastAppend()
+ } else {
+ // Otherwise, still probe the newly added replicas; there's no reason to
+ // let them wait out a heartbeat interval (or the next incoming
+ // proposal).
+ r.prs.Visit(func(id uint64, pr *tracker.Progress) {
+ r.maybeSendAppend(id, false /* sendIfEmpty */)
+ })
}
- // If the removed node is the leadTransferee, then abort the leadership transferring.
- if r.state == StateLeader && r.leadTransferee == id {
+ // If the the leadTransferee was removed, abort the leadership transfer.
+ if _, tOK := r.prs.Progress[r.leadTransferee]; !tOK && r.leadTransferee != 0 {
r.abortLeaderTransfer()
}
-}
-func (r *raft) setProgress(id, match, next uint64, isLearner bool) {
- if !isLearner {
- delete(r.learnerPrs, id)
- r.prs[id] = &Progress{Next: next, Match: match, ins: newInflights(r.maxInflight)}
- return
- }
-
- if _, ok := r.prs[id]; ok {
- panic(fmt.Sprintf("%x unexpected changing from voter to learner for %x", r.id, id))
- }
- r.learnerPrs[id] = &Progress{Next: next, Match: match, ins: newInflights(r.maxInflight), IsLearner: true}
-}
-
-func (r *raft) delProgress(id uint64) {
- delete(r.prs, id)
- delete(r.learnerPrs, id)
+ return cs
}
func (r *raft) loadState(state pb.HardState) {
@@ -1490,29 +1594,6 @@
r.randomizedElectionTimeout = r.electionTimeout + globalRand.Intn(r.electionTimeout)
}
-// checkQuorumActive returns true if the quorum is active from
-// the view of the local raft state machine. Otherwise, it returns
-// false.
-// checkQuorumActive also resets all RecentActive to false.
-func (r *raft) checkQuorumActive() bool {
- var act int
-
- r.forEachProgress(func(id uint64, pr *Progress) {
- if id == r.id { // self is always active
- act++
- return
- }
-
- if pr.RecentActive && !pr.IsLearner {
- act++
- }
-
- pr.RecentActive = false
- })
-
- return act >= r.quorum()
-}
-
func (r *raft) sendTimeoutNow(to uint64) {
r.send(pb.Message{To: to, Type: pb.MsgTimeoutNow})
}
diff --git a/vendor/go.etcd.io/etcd/raft/raftpb/confchange.go b/vendor/go.etcd.io/etcd/raft/raftpb/confchange.go
new file mode 100644
index 0000000..46a7a70
--- /dev/null
+++ b/vendor/go.etcd.io/etcd/raft/raftpb/confchange.go
@@ -0,0 +1,170 @@
+// Copyright 2019 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package raftpb
+
+import (
+ "fmt"
+ "strconv"
+ "strings"
+
+ "github.com/gogo/protobuf/proto"
+)
+
+// ConfChangeI abstracts over ConfChangeV2 and (legacy) ConfChange to allow
+// treating them in a unified manner.
+type ConfChangeI interface {
+ AsV2() ConfChangeV2
+ AsV1() (ConfChange, bool)
+}
+
+// MarshalConfChange calls Marshal on the underlying ConfChange or ConfChangeV2
+// and returns the result along with the corresponding EntryType.
+func MarshalConfChange(c ConfChangeI) (EntryType, []byte, error) {
+ var typ EntryType
+ var ccdata []byte
+ var err error
+ if ccv1, ok := c.AsV1(); ok {
+ typ = EntryConfChange
+ ccdata, err = ccv1.Marshal()
+ } else {
+ ccv2 := c.AsV2()
+ typ = EntryConfChangeV2
+ ccdata, err = ccv2.Marshal()
+ }
+ return typ, ccdata, err
+}
+
+// AsV2 returns a V2 configuration change carrying out the same operation.
+func (c ConfChange) AsV2() ConfChangeV2 {
+ return ConfChangeV2{
+ Changes: []ConfChangeSingle{{
+ Type: c.Type,
+ NodeID: c.NodeID,
+ }},
+ Context: c.Context,
+ }
+}
+
+// AsV1 returns the ConfChange and true.
+func (c ConfChange) AsV1() (ConfChange, bool) {
+ return c, true
+}
+
+// AsV2 is the identity.
+func (c ConfChangeV2) AsV2() ConfChangeV2 { return c }
+
+// AsV1 returns ConfChange{} and false.
+func (c ConfChangeV2) AsV1() (ConfChange, bool) { return ConfChange{}, false }
+
+// EnterJoint returns two bools. The second bool is true if and only if this
+// config change will use Joint Consensus, which is the case if it contains more
+// than one change or if the use of Joint Consensus was requested explicitly.
+// The first bool can only be true if second one is, and indicates whether the
+// Joint State will be left automatically.
+func (c *ConfChangeV2) EnterJoint() (autoLeave bool, ok bool) {
+ // NB: in theory, more config changes could qualify for the "simple"
+ // protocol but it depends on the config on top of which the changes apply.
+ // For example, adding two learners is not OK if both nodes are part of the
+ // base config (i.e. two voters are turned into learners in the process of
+ // applying the conf change). In practice, these distinctions should not
+ // matter, so we keep it simple and use Joint Consensus liberally.
+ if c.Transition != ConfChangeTransitionAuto || len(c.Changes) > 1 {
+ // Use Joint Consensus.
+ var autoLeave bool
+ switch c.Transition {
+ case ConfChangeTransitionAuto:
+ autoLeave = true
+ case ConfChangeTransitionJointImplicit:
+ autoLeave = true
+ case ConfChangeTransitionJointExplicit:
+ default:
+ panic(fmt.Sprintf("unknown transition: %+v", c))
+ }
+ return autoLeave, true
+ }
+ return false, false
+}
+
+// LeaveJoint is true if the configuration change leaves a joint configuration.
+// This is the case if the ConfChangeV2 is zero, with the possible exception of
+// the Context field.
+func (c *ConfChangeV2) LeaveJoint() bool {
+ cpy := *c
+ cpy.Context = nil
+ return proto.Equal(&cpy, &ConfChangeV2{})
+}
+
+// ConfChangesFromString parses a Space-delimited sequence of operations into a
+// slice of ConfChangeSingle. The supported operations are:
+// - vn: make n a voter,
+// - ln: make n a learner,
+// - rn: remove n, and
+// - un: update n.
+func ConfChangesFromString(s string) ([]ConfChangeSingle, error) {
+ var ccs []ConfChangeSingle
+ toks := strings.Split(strings.TrimSpace(s), " ")
+ if toks[0] == "" {
+ toks = nil
+ }
+ for _, tok := range toks {
+ if len(tok) < 2 {
+ return nil, fmt.Errorf("unknown token %s", tok)
+ }
+ var cc ConfChangeSingle
+ switch tok[0] {
+ case 'v':
+ cc.Type = ConfChangeAddNode
+ case 'l':
+ cc.Type = ConfChangeAddLearnerNode
+ case 'r':
+ cc.Type = ConfChangeRemoveNode
+ case 'u':
+ cc.Type = ConfChangeUpdateNode
+ default:
+ return nil, fmt.Errorf("unknown input: %s", tok)
+ }
+ id, err := strconv.ParseUint(tok[1:], 10, 64)
+ if err != nil {
+ return nil, err
+ }
+ cc.NodeID = id
+ ccs = append(ccs, cc)
+ }
+ return ccs, nil
+}
+
+// ConfChangesToString is the inverse to ConfChangesFromString.
+func ConfChangesToString(ccs []ConfChangeSingle) string {
+ var buf strings.Builder
+ for i, cc := range ccs {
+ if i > 0 {
+ buf.WriteByte(' ')
+ }
+ switch cc.Type {
+ case ConfChangeAddNode:
+ buf.WriteByte('v')
+ case ConfChangeAddLearnerNode:
+ buf.WriteByte('l')
+ case ConfChangeRemoveNode:
+ buf.WriteByte('r')
+ case ConfChangeUpdateNode:
+ buf.WriteByte('u')
+ default:
+ buf.WriteString("unknown")
+ }
+ fmt.Fprintf(&buf, "%d", cc.NodeID)
+ }
+ return buf.String()
+}
diff --git a/vendor/go.etcd.io/etcd/raft/raftpb/confstate.go b/vendor/go.etcd.io/etcd/raft/raftpb/confstate.go
new file mode 100644
index 0000000..4bda932
--- /dev/null
+++ b/vendor/go.etcd.io/etcd/raft/raftpb/confstate.go
@@ -0,0 +1,45 @@
+// Copyright 2019 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package raftpb
+
+import (
+ "fmt"
+ "reflect"
+ "sort"
+)
+
+// Equivalent returns a nil error if the inputs describe the same configuration.
+// On mismatch, returns a descriptive error showing the differences.
+func (cs ConfState) Equivalent(cs2 ConfState) error {
+ cs1 := cs
+ orig1, orig2 := cs1, cs2
+ s := func(sl *[]uint64) {
+ *sl = append([]uint64(nil), *sl...)
+ sort.Slice(*sl, func(i, j int) bool { return (*sl)[i] < (*sl)[j] })
+ }
+
+ for _, cs := range []*ConfState{&cs1, &cs2} {
+ s(&cs.Voters)
+ s(&cs.Learners)
+ s(&cs.VotersOutgoing)
+ s(&cs.LearnersNext)
+ cs.XXX_unrecognized = nil
+ }
+
+ if !reflect.DeepEqual(cs1, cs2) {
+ return fmt.Errorf("ConfStates not equivalent after sorting:\n%+#v\n%+#v\nInputs were:\n%+#v\n%+#v", cs1, cs2, orig1, orig2)
+ }
+ return nil
+}
diff --git a/vendor/go.etcd.io/etcd/raft/raftpb/raft.pb.go b/vendor/go.etcd.io/etcd/raft/raftpb/raft.pb.go
index fd9ee37..fcf259c 100644
--- a/vendor/go.etcd.io/etcd/raft/raftpb/raft.pb.go
+++ b/vendor/go.etcd.io/etcd/raft/raftpb/raft.pb.go
@@ -15,6 +15,8 @@
HardState
ConfState
ConfChange
+ ConfChangeSingle
+ ConfChangeV2
*/
package raftpb
@@ -44,17 +46,20 @@
type EntryType int32
const (
- EntryNormal EntryType = 0
- EntryConfChange EntryType = 1
+ EntryNormal EntryType = 0
+ EntryConfChange EntryType = 1
+ EntryConfChangeV2 EntryType = 2
)
var EntryType_name = map[int32]string{
0: "EntryNormal",
1: "EntryConfChange",
+ 2: "EntryConfChangeV2",
}
var EntryType_value = map[string]int32{
- "EntryNormal": 0,
- "EntryConfChange": 1,
+ "EntryNormal": 0,
+ "EntryConfChange": 1,
+ "EntryConfChangeV2": 2,
}
func (x EntryType) Enum() *EntryType {
@@ -160,6 +165,57 @@
}
func (MessageType) EnumDescriptor() ([]byte, []int) { return fileDescriptorRaft, []int{1} }
+// ConfChangeTransition specifies the behavior of a configuration change with
+// respect to joint consensus.
+type ConfChangeTransition int32
+
+const (
+ // Automatically use the simple protocol if possible, otherwise fall back
+ // to ConfChangeJointImplicit. Most applications will want to use this.
+ ConfChangeTransitionAuto ConfChangeTransition = 0
+ // Use joint consensus unconditionally, and transition out of them
+ // automatically (by proposing a zero configuration change).
+ //
+ // This option is suitable for applications that want to minimize the time
+ // spent in the joint configuration and do not store the joint configuration
+ // in the state machine (outside of InitialState).
+ ConfChangeTransitionJointImplicit ConfChangeTransition = 1
+ // Use joint consensus and remain in the joint configuration until the
+ // application proposes a no-op configuration change. This is suitable for
+ // applications that want to explicitly control the transitions, for example
+ // to use a custom payload (via the Context field).
+ ConfChangeTransitionJointExplicit ConfChangeTransition = 2
+)
+
+var ConfChangeTransition_name = map[int32]string{
+ 0: "ConfChangeTransitionAuto",
+ 1: "ConfChangeTransitionJointImplicit",
+ 2: "ConfChangeTransitionJointExplicit",
+}
+var ConfChangeTransition_value = map[string]int32{
+ "ConfChangeTransitionAuto": 0,
+ "ConfChangeTransitionJointImplicit": 1,
+ "ConfChangeTransitionJointExplicit": 2,
+}
+
+func (x ConfChangeTransition) Enum() *ConfChangeTransition {
+ p := new(ConfChangeTransition)
+ *p = x
+ return p
+}
+func (x ConfChangeTransition) String() string {
+ return proto.EnumName(ConfChangeTransition_name, int32(x))
+}
+func (x *ConfChangeTransition) UnmarshalJSON(data []byte) error {
+ value, err := proto.UnmarshalJSONEnum(ConfChangeTransition_value, data, "ConfChangeTransition")
+ if err != nil {
+ return err
+ }
+ *x = ConfChangeTransition(value)
+ return nil
+}
+func (ConfChangeTransition) EnumDescriptor() ([]byte, []int) { return fileDescriptorRaft, []int{2} }
+
type ConfChangeType int32
const (
@@ -198,7 +254,7 @@
*x = ConfChangeType(value)
return nil
}
-func (ConfChangeType) EnumDescriptor() ([]byte, []int) { return fileDescriptorRaft, []int{2} }
+func (ConfChangeType) EnumDescriptor() ([]byte, []int) { return fileDescriptorRaft, []int{3} }
type Entry struct {
Term uint64 `protobuf:"varint,2,opt,name=Term" json:"Term"`
@@ -270,9 +326,21 @@
func (*HardState) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{4} }
type ConfState struct {
- Nodes []uint64 `protobuf:"varint,1,rep,name=nodes" json:"nodes,omitempty"`
- Learners []uint64 `protobuf:"varint,2,rep,name=learners" json:"learners,omitempty"`
- XXX_unrecognized []byte `json:"-"`
+ // The voters in the incoming config. (If the configuration is not joint,
+ // then the outgoing config is empty).
+ Voters []uint64 `protobuf:"varint,1,rep,name=voters" json:"voters,omitempty"`
+ // The learners in the incoming config.
+ Learners []uint64 `protobuf:"varint,2,rep,name=learners" json:"learners,omitempty"`
+ // The voters in the outgoing config.
+ VotersOutgoing []uint64 `protobuf:"varint,3,rep,name=voters_outgoing,json=votersOutgoing" json:"voters_outgoing,omitempty"`
+ // The nodes that will become learners when the outgoing config is removed.
+ // These nodes are necessarily currently in nodes_joint (or they would have
+ // been added to the incoming config right away).
+ LearnersNext []uint64 `protobuf:"varint,4,rep,name=learners_next,json=learnersNext" json:"learners_next,omitempty"`
+ // If set, the config is joint and Raft will automatically transition into
+ // the final config (i.e. remove the outgoing config) when this is safe.
+ AutoLeave bool `protobuf:"varint,5,opt,name=auto_leave,json=autoLeave" json:"auto_leave"`
+ XXX_unrecognized []byte `json:"-"`
}
func (m *ConfState) Reset() { *m = ConfState{} }
@@ -281,11 +349,14 @@
func (*ConfState) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{5} }
type ConfChange struct {
- ID uint64 `protobuf:"varint,1,opt,name=ID" json:"ID"`
- Type ConfChangeType `protobuf:"varint,2,opt,name=Type,enum=raftpb.ConfChangeType" json:"Type"`
- NodeID uint64 `protobuf:"varint,3,opt,name=NodeID" json:"NodeID"`
- Context []byte `protobuf:"bytes,4,opt,name=Context" json:"Context,omitempty"`
- XXX_unrecognized []byte `json:"-"`
+ Type ConfChangeType `protobuf:"varint,2,opt,name=type,enum=raftpb.ConfChangeType" json:"type"`
+ NodeID uint64 `protobuf:"varint,3,opt,name=node_id,json=nodeId" json:"node_id"`
+ Context []byte `protobuf:"bytes,4,opt,name=context" json:"context,omitempty"`
+ // NB: this is used only by etcd to thread through a unique identifier.
+ // Ideally it should really use the Context instead. No counterpart to
+ // this field exists in ConfChangeV2.
+ ID uint64 `protobuf:"varint,1,opt,name=id" json:"id"`
+ XXX_unrecognized []byte `json:"-"`
}
func (m *ConfChange) Reset() { *m = ConfChange{} }
@@ -293,6 +364,63 @@
func (*ConfChange) ProtoMessage() {}
func (*ConfChange) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{6} }
+// ConfChangeSingle is an individual configuration change operation. Multiple
+// such operations can be carried out atomically via a ConfChangeV2.
+type ConfChangeSingle struct {
+ Type ConfChangeType `protobuf:"varint,1,opt,name=type,enum=raftpb.ConfChangeType" json:"type"`
+ NodeID uint64 `protobuf:"varint,2,opt,name=node_id,json=nodeId" json:"node_id"`
+ XXX_unrecognized []byte `json:"-"`
+}
+
+func (m *ConfChangeSingle) Reset() { *m = ConfChangeSingle{} }
+func (m *ConfChangeSingle) String() string { return proto.CompactTextString(m) }
+func (*ConfChangeSingle) ProtoMessage() {}
+func (*ConfChangeSingle) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{7} }
+
+// ConfChangeV2 messages initiate configuration changes. They support both the
+// simple "one at a time" membership change protocol and full Joint Consensus
+// allowing for arbitrary changes in membership.
+//
+// The supplied context is treated as an opaque payload and can be used to
+// attach an action on the state machine to the application of the config change
+// proposal. Note that contrary to Joint Consensus as outlined in the Raft
+// paper[1], configuration changes become active when they are *applied* to the
+// state machine (not when they are appended to the log).
+//
+// The simple protocol can be used whenever only a single change is made.
+//
+// Non-simple changes require the use of Joint Consensus, for which two
+// configuration changes are run. The first configuration change specifies the
+// desired changes and transitions the Raft group into the joint configuration,
+// in which quorum requires a majority of both the pre-changes and post-changes
+// configuration. Joint Consensus avoids entering fragile intermediate
+// configurations that could compromise survivability. For example, without the
+// use of Joint Consensus and running across three availability zones with a
+// replication factor of three, it is not possible to replace a voter without
+// entering an intermediate configuration that does not survive the outage of
+// one availability zone.
+//
+// The provided ConfChangeTransition specifies how (and whether) Joint Consensus
+// is used, and assigns the task of leaving the joint configuration either to
+// Raft or the application. Leaving the joint configuration is accomplished by
+// proposing a ConfChangeV2 with only and optionally the Context field
+// populated.
+//
+// For details on Raft membership changes, see:
+//
+// [1]: https://github.com/ongardie/dissertation/blob/master/online-trim.pdf
+type ConfChangeV2 struct {
+ Transition ConfChangeTransition `protobuf:"varint,1,opt,name=transition,enum=raftpb.ConfChangeTransition" json:"transition"`
+ Changes []ConfChangeSingle `protobuf:"bytes,2,rep,name=changes" json:"changes"`
+ Context []byte `protobuf:"bytes,3,opt,name=context" json:"context,omitempty"`
+ XXX_unrecognized []byte `json:"-"`
+}
+
+func (m *ConfChangeV2) Reset() { *m = ConfChangeV2{} }
+func (m *ConfChangeV2) String() string { return proto.CompactTextString(m) }
+func (*ConfChangeV2) ProtoMessage() {}
+func (*ConfChangeV2) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{8} }
+
func init() {
proto.RegisterType((*Entry)(nil), "raftpb.Entry")
proto.RegisterType((*SnapshotMetadata)(nil), "raftpb.SnapshotMetadata")
@@ -301,8 +429,11 @@
proto.RegisterType((*HardState)(nil), "raftpb.HardState")
proto.RegisterType((*ConfState)(nil), "raftpb.ConfState")
proto.RegisterType((*ConfChange)(nil), "raftpb.ConfChange")
+ proto.RegisterType((*ConfChangeSingle)(nil), "raftpb.ConfChangeSingle")
+ proto.RegisterType((*ConfChangeV2)(nil), "raftpb.ConfChangeV2")
proto.RegisterEnum("raftpb.EntryType", EntryType_name, EntryType_value)
proto.RegisterEnum("raftpb.MessageType", MessageType_name, MessageType_value)
+ proto.RegisterEnum("raftpb.ConfChangeTransition", ConfChangeTransition_name, ConfChangeTransition_value)
proto.RegisterEnum("raftpb.ConfChangeType", ConfChangeType_name, ConfChangeType_value)
}
func (m *Entry) Marshal() (dAtA []byte, err error) {
@@ -535,8 +666,8 @@
_ = i
var l int
_ = l
- if len(m.Nodes) > 0 {
- for _, num := range m.Nodes {
+ if len(m.Voters) > 0 {
+ for _, num := range m.Voters {
dAtA[i] = 0x8
i++
i = encodeVarintRaft(dAtA, i, uint64(num))
@@ -549,6 +680,28 @@
i = encodeVarintRaft(dAtA, i, uint64(num))
}
}
+ if len(m.VotersOutgoing) > 0 {
+ for _, num := range m.VotersOutgoing {
+ dAtA[i] = 0x18
+ i++
+ i = encodeVarintRaft(dAtA, i, uint64(num))
+ }
+ }
+ if len(m.LearnersNext) > 0 {
+ for _, num := range m.LearnersNext {
+ dAtA[i] = 0x20
+ i++
+ i = encodeVarintRaft(dAtA, i, uint64(num))
+ }
+ }
+ dAtA[i] = 0x28
+ i++
+ if m.AutoLeave {
+ dAtA[i] = 1
+ } else {
+ dAtA[i] = 0
+ }
+ i++
if m.XXX_unrecognized != nil {
i += copy(dAtA[i:], m.XXX_unrecognized)
}
@@ -591,6 +744,75 @@
return i, nil
}
+func (m *ConfChangeSingle) Marshal() (dAtA []byte, err error) {
+ size := m.Size()
+ dAtA = make([]byte, size)
+ n, err := m.MarshalTo(dAtA)
+ if err != nil {
+ return nil, err
+ }
+ return dAtA[:n], nil
+}
+
+func (m *ConfChangeSingle) MarshalTo(dAtA []byte) (int, error) {
+ var i int
+ _ = i
+ var l int
+ _ = l
+ dAtA[i] = 0x8
+ i++
+ i = encodeVarintRaft(dAtA, i, uint64(m.Type))
+ dAtA[i] = 0x10
+ i++
+ i = encodeVarintRaft(dAtA, i, uint64(m.NodeID))
+ if m.XXX_unrecognized != nil {
+ i += copy(dAtA[i:], m.XXX_unrecognized)
+ }
+ return i, nil
+}
+
+func (m *ConfChangeV2) Marshal() (dAtA []byte, err error) {
+ size := m.Size()
+ dAtA = make([]byte, size)
+ n, err := m.MarshalTo(dAtA)
+ if err != nil {
+ return nil, err
+ }
+ return dAtA[:n], nil
+}
+
+func (m *ConfChangeV2) MarshalTo(dAtA []byte) (int, error) {
+ var i int
+ _ = i
+ var l int
+ _ = l
+ dAtA[i] = 0x8
+ i++
+ i = encodeVarintRaft(dAtA, i, uint64(m.Transition))
+ if len(m.Changes) > 0 {
+ for _, msg := range m.Changes {
+ dAtA[i] = 0x12
+ i++
+ i = encodeVarintRaft(dAtA, i, uint64(msg.Size()))
+ n, err := msg.MarshalTo(dAtA[i:])
+ if err != nil {
+ return 0, err
+ }
+ i += n
+ }
+ }
+ if m.Context != nil {
+ dAtA[i] = 0x1a
+ i++
+ i = encodeVarintRaft(dAtA, i, uint64(len(m.Context)))
+ i += copy(dAtA[i:], m.Context)
+ }
+ if m.XXX_unrecognized != nil {
+ i += copy(dAtA[i:], m.XXX_unrecognized)
+ }
+ return i, nil
+}
+
func encodeVarintRaft(dAtA []byte, offset int, v uint64) int {
for v >= 1<<7 {
dAtA[offset] = uint8(v&0x7f | 0x80)
@@ -689,8 +911,8 @@
func (m *ConfState) Size() (n int) {
var l int
_ = l
- if len(m.Nodes) > 0 {
- for _, e := range m.Nodes {
+ if len(m.Voters) > 0 {
+ for _, e := range m.Voters {
n += 1 + sovRaft(uint64(e))
}
}
@@ -699,6 +921,17 @@
n += 1 + sovRaft(uint64(e))
}
}
+ if len(m.VotersOutgoing) > 0 {
+ for _, e := range m.VotersOutgoing {
+ n += 1 + sovRaft(uint64(e))
+ }
+ }
+ if len(m.LearnersNext) > 0 {
+ for _, e := range m.LearnersNext {
+ n += 1 + sovRaft(uint64(e))
+ }
+ }
+ n += 2
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
@@ -721,6 +954,37 @@
return n
}
+func (m *ConfChangeSingle) Size() (n int) {
+ var l int
+ _ = l
+ n += 1 + sovRaft(uint64(m.Type))
+ n += 1 + sovRaft(uint64(m.NodeID))
+ if m.XXX_unrecognized != nil {
+ n += len(m.XXX_unrecognized)
+ }
+ return n
+}
+
+func (m *ConfChangeV2) Size() (n int) {
+ var l int
+ _ = l
+ n += 1 + sovRaft(uint64(m.Transition))
+ if len(m.Changes) > 0 {
+ for _, e := range m.Changes {
+ l = e.Size()
+ n += 1 + l + sovRaft(uint64(l))
+ }
+ }
+ if m.Context != nil {
+ l = len(m.Context)
+ n += 1 + l + sovRaft(uint64(l))
+ }
+ if m.XXX_unrecognized != nil {
+ n += len(m.XXX_unrecognized)
+ }
+ return n
+}
+
func sovRaft(x uint64) (n int) {
for {
n++
@@ -1573,7 +1837,7 @@
break
}
}
- m.Nodes = append(m.Nodes, v)
+ m.Voters = append(m.Voters, v)
} else if wireType == 2 {
var packedLen int
for shift := uint(0); ; shift += 7 {
@@ -1613,10 +1877,10 @@
break
}
}
- m.Nodes = append(m.Nodes, v)
+ m.Voters = append(m.Voters, v)
}
} else {
- return fmt.Errorf("proto: wrong wireType = %d for field Nodes", wireType)
+ return fmt.Errorf("proto: wrong wireType = %d for field Voters", wireType)
}
case 2:
if wireType == 0 {
@@ -1680,6 +1944,150 @@
} else {
return fmt.Errorf("proto: wrong wireType = %d for field Learners", wireType)
}
+ case 3:
+ if wireType == 0 {
+ var v uint64
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowRaft
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ v |= (uint64(b) & 0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ m.VotersOutgoing = append(m.VotersOutgoing, v)
+ } else if wireType == 2 {
+ var packedLen int
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowRaft
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ packedLen |= (int(b) & 0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ if packedLen < 0 {
+ return ErrInvalidLengthRaft
+ }
+ postIndex := iNdEx + packedLen
+ if postIndex > l {
+ return io.ErrUnexpectedEOF
+ }
+ for iNdEx < postIndex {
+ var v uint64
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowRaft
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ v |= (uint64(b) & 0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ m.VotersOutgoing = append(m.VotersOutgoing, v)
+ }
+ } else {
+ return fmt.Errorf("proto: wrong wireType = %d for field VotersOutgoing", wireType)
+ }
+ case 4:
+ if wireType == 0 {
+ var v uint64
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowRaft
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ v |= (uint64(b) & 0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ m.LearnersNext = append(m.LearnersNext, v)
+ } else if wireType == 2 {
+ var packedLen int
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowRaft
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ packedLen |= (int(b) & 0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ if packedLen < 0 {
+ return ErrInvalidLengthRaft
+ }
+ postIndex := iNdEx + packedLen
+ if postIndex > l {
+ return io.ErrUnexpectedEOF
+ }
+ for iNdEx < postIndex {
+ var v uint64
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowRaft
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ v |= (uint64(b) & 0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ m.LearnersNext = append(m.LearnersNext, v)
+ }
+ } else {
+ return fmt.Errorf("proto: wrong wireType = %d for field LearnersNext", wireType)
+ }
+ case 5:
+ if wireType != 0 {
+ return fmt.Errorf("proto: wrong wireType = %d for field AutoLeave", wireType)
+ }
+ var v int
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowRaft
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ v |= (int(b) & 0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ m.AutoLeave = bool(v != 0)
default:
iNdEx = preIndex
skippy, err := skipRaft(dAtA[iNdEx:])
@@ -1841,6 +2249,227 @@
}
return nil
}
+func (m *ConfChangeSingle) Unmarshal(dAtA []byte) error {
+ l := len(dAtA)
+ iNdEx := 0
+ for iNdEx < l {
+ preIndex := iNdEx
+ var wire uint64
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowRaft
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ wire |= (uint64(b) & 0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ fieldNum := int32(wire >> 3)
+ wireType := int(wire & 0x7)
+ if wireType == 4 {
+ return fmt.Errorf("proto: ConfChangeSingle: wiretype end group for non-group")
+ }
+ if fieldNum <= 0 {
+ return fmt.Errorf("proto: ConfChangeSingle: illegal tag %d (wire type %d)", fieldNum, wire)
+ }
+ switch fieldNum {
+ case 1:
+ if wireType != 0 {
+ return fmt.Errorf("proto: wrong wireType = %d for field Type", wireType)
+ }
+ m.Type = 0
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowRaft
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ m.Type |= (ConfChangeType(b) & 0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ case 2:
+ if wireType != 0 {
+ return fmt.Errorf("proto: wrong wireType = %d for field NodeID", wireType)
+ }
+ m.NodeID = 0
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowRaft
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ m.NodeID |= (uint64(b) & 0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ default:
+ iNdEx = preIndex
+ skippy, err := skipRaft(dAtA[iNdEx:])
+ if err != nil {
+ return err
+ }
+ if skippy < 0 {
+ return ErrInvalidLengthRaft
+ }
+ if (iNdEx + skippy) > l {
+ return io.ErrUnexpectedEOF
+ }
+ m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...)
+ iNdEx += skippy
+ }
+ }
+
+ if iNdEx > l {
+ return io.ErrUnexpectedEOF
+ }
+ return nil
+}
+func (m *ConfChangeV2) Unmarshal(dAtA []byte) error {
+ l := len(dAtA)
+ iNdEx := 0
+ for iNdEx < l {
+ preIndex := iNdEx
+ var wire uint64
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowRaft
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ wire |= (uint64(b) & 0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ fieldNum := int32(wire >> 3)
+ wireType := int(wire & 0x7)
+ if wireType == 4 {
+ return fmt.Errorf("proto: ConfChangeV2: wiretype end group for non-group")
+ }
+ if fieldNum <= 0 {
+ return fmt.Errorf("proto: ConfChangeV2: illegal tag %d (wire type %d)", fieldNum, wire)
+ }
+ switch fieldNum {
+ case 1:
+ if wireType != 0 {
+ return fmt.Errorf("proto: wrong wireType = %d for field Transition", wireType)
+ }
+ m.Transition = 0
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowRaft
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ m.Transition |= (ConfChangeTransition(b) & 0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ case 2:
+ if wireType != 2 {
+ return fmt.Errorf("proto: wrong wireType = %d for field Changes", wireType)
+ }
+ var msglen int
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowRaft
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ msglen |= (int(b) & 0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ if msglen < 0 {
+ return ErrInvalidLengthRaft
+ }
+ postIndex := iNdEx + msglen
+ if postIndex > l {
+ return io.ErrUnexpectedEOF
+ }
+ m.Changes = append(m.Changes, ConfChangeSingle{})
+ if err := m.Changes[len(m.Changes)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+ return err
+ }
+ iNdEx = postIndex
+ case 3:
+ if wireType != 2 {
+ return fmt.Errorf("proto: wrong wireType = %d for field Context", wireType)
+ }
+ var byteLen int
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowRaft
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ byteLen |= (int(b) & 0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ if byteLen < 0 {
+ return ErrInvalidLengthRaft
+ }
+ postIndex := iNdEx + byteLen
+ if postIndex > l {
+ return io.ErrUnexpectedEOF
+ }
+ m.Context = append(m.Context[:0], dAtA[iNdEx:postIndex]...)
+ if m.Context == nil {
+ m.Context = []byte{}
+ }
+ iNdEx = postIndex
+ default:
+ iNdEx = preIndex
+ skippy, err := skipRaft(dAtA[iNdEx:])
+ if err != nil {
+ return err
+ }
+ if skippy < 0 {
+ return ErrInvalidLengthRaft
+ }
+ if (iNdEx + skippy) > l {
+ return io.ErrUnexpectedEOF
+ }
+ m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...)
+ iNdEx += skippy
+ }
+ }
+
+ if iNdEx > l {
+ return io.ErrUnexpectedEOF
+ }
+ return nil
+}
func skipRaft(dAtA []byte) (n int, err error) {
l := len(dAtA)
iNdEx := 0
@@ -1949,56 +2578,69 @@
func init() { proto.RegisterFile("raft.proto", fileDescriptorRaft) }
var fileDescriptorRaft = []byte{
- // 815 bytes of a gzipped FileDescriptorProto
- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x64, 0x54, 0xcd, 0x6e, 0x23, 0x45,
- 0x10, 0xf6, 0x8c, 0xc7, 0x7f, 0x35, 0x8e, 0xd3, 0xa9, 0x35, 0xa8, 0x15, 0x45, 0xc6, 0xb2, 0x38,
- 0x58, 0x41, 0x1b, 0x20, 0x07, 0x0e, 0x48, 0x1c, 0x36, 0x09, 0x52, 0x22, 0xad, 0xa3, 0xc5, 0x9b,
- 0xe5, 0x80, 0x84, 0x50, 0xc7, 0x53, 0x9e, 0x18, 0x32, 0xd3, 0xa3, 0x9e, 0xf6, 0xb2, 0xb9, 0x20,
- 0x1e, 0x80, 0x07, 0xe0, 0xc2, 0xfb, 0xe4, 0xb8, 0x12, 0x77, 0xc4, 0x86, 0x17, 0x41, 0xdd, 0xd3,
- 0x63, 0xcf, 0x24, 0xb7, 0xae, 0xef, 0xab, 0xae, 0xfa, 0xea, 0xeb, 0x9a, 0x01, 0x50, 0x62, 0xa9,
- 0x8f, 0x32, 0x25, 0xb5, 0xc4, 0xb6, 0x39, 0x67, 0xd7, 0xfb, 0xc3, 0x58, 0xc6, 0xd2, 0x42, 0x9f,
- 0x9b, 0x53, 0xc1, 0x4e, 0x7e, 0x83, 0xd6, 0xb7, 0xa9, 0x56, 0x77, 0xf8, 0x19, 0x04, 0x57, 0x77,
- 0x19, 0x71, 0x6f, 0xec, 0x4d, 0x07, 0xc7, 0x7b, 0x47, 0xc5, 0xad, 0x23, 0x4b, 0x1a, 0xe2, 0x24,
- 0xb8, 0xff, 0xe7, 0x93, 0xc6, 0xdc, 0x26, 0x21, 0x87, 0xe0, 0x8a, 0x54, 0xc2, 0xfd, 0xb1, 0x37,
- 0x0d, 0x36, 0x0c, 0xa9, 0x04, 0xf7, 0xa1, 0x75, 0x91, 0x46, 0xf4, 0x8e, 0x37, 0x2b, 0x54, 0x01,
- 0x21, 0x42, 0x70, 0x26, 0xb4, 0xe0, 0xc1, 0xd8, 0x9b, 0xf6, 0xe7, 0xf6, 0x3c, 0xf9, 0xdd, 0x03,
- 0xf6, 0x3a, 0x15, 0x59, 0x7e, 0x23, 0xf5, 0x8c, 0xb4, 0x88, 0x84, 0x16, 0xf8, 0x15, 0xc0, 0x42,
- 0xa6, 0xcb, 0x9f, 0x72, 0x2d, 0x74, 0xa1, 0x28, 0xdc, 0x2a, 0x3a, 0x95, 0xe9, 0xf2, 0xb5, 0x21,
- 0x5c, 0xf1, 0xde, 0xa2, 0x04, 0x4c, 0xf3, 0x95, 0x6d, 0x5e, 0xd5, 0x55, 0x40, 0x46, 0xb2, 0x36,
- 0x92, 0xab, 0xba, 0x2c, 0x32, 0xf9, 0x01, 0xba, 0xa5, 0x02, 0x23, 0xd1, 0x28, 0xb0, 0x3d, 0xfb,
- 0x73, 0x7b, 0xc6, 0xaf, 0xa1, 0x9b, 0x38, 0x65, 0xb6, 0x70, 0x78, 0xcc, 0x4b, 0x2d, 0x8f, 0x95,
- 0xbb, 0xba, 0x9b, 0xfc, 0xc9, 0x5f, 0x4d, 0xe8, 0xcc, 0x28, 0xcf, 0x45, 0x4c, 0xf8, 0x1c, 0x02,
- 0xbd, 0x75, 0xf8, 0x59, 0x59, 0xc3, 0xd1, 0x55, 0x8f, 0x4d, 0x1a, 0x0e, 0xc1, 0xd7, 0xb2, 0x36,
- 0x89, 0xaf, 0xa5, 0x19, 0x63, 0xa9, 0xe4, 0xa3, 0x31, 0x0c, 0xb2, 0x19, 0x30, 0x78, 0x3c, 0x20,
- 0x8e, 0xa0, 0x73, 0x2b, 0x63, 0xfb, 0x60, 0xad, 0x0a, 0x59, 0x82, 0x5b, 0xdb, 0xda, 0x4f, 0x6d,
- 0x7b, 0x0e, 0x1d, 0x4a, 0xb5, 0x5a, 0x51, 0xce, 0x3b, 0xe3, 0xe6, 0x34, 0x3c, 0xde, 0xa9, 0x6d,
- 0x46, 0x59, 0xca, 0xe5, 0xe0, 0x01, 0xb4, 0x17, 0x32, 0x49, 0x56, 0x9a, 0x77, 0x2b, 0xb5, 0x1c,
- 0x86, 0xc7, 0xd0, 0xcd, 0x9d, 0x63, 0xbc, 0x67, 0x9d, 0x64, 0x8f, 0x9d, 0x2c, 0x1d, 0x2c, 0xf3,
- 0x4c, 0x45, 0x45, 0x3f, 0xd3, 0x42, 0x73, 0x18, 0x7b, 0xd3, 0x6e, 0x59, 0xb1, 0xc0, 0xf0, 0x53,
- 0x80, 0xe2, 0x74, 0xbe, 0x4a, 0x35, 0x0f, 0x2b, 0x3d, 0x2b, 0x38, 0x72, 0xe8, 0x2c, 0x64, 0xaa,
- 0xe9, 0x9d, 0xe6, 0x7d, 0xfb, 0xb0, 0x65, 0x38, 0xf9, 0x11, 0x7a, 0xe7, 0x42, 0x45, 0xc5, 0xfa,
- 0x94, 0x0e, 0x7a, 0x4f, 0x1c, 0xe4, 0x10, 0xbc, 0x95, 0x9a, 0xea, 0xfb, 0x6e, 0x90, 0xca, 0xc0,
- 0xcd, 0xa7, 0x03, 0x4f, 0xbe, 0x81, 0xde, 0x66, 0x5d, 0x71, 0x08, 0xad, 0x54, 0x46, 0x94, 0x73,
- 0x6f, 0xdc, 0x9c, 0x06, 0xf3, 0x22, 0xc0, 0x7d, 0xe8, 0xde, 0x92, 0x50, 0x29, 0xa9, 0x9c, 0xfb,
- 0x96, 0xd8, 0xc4, 0x93, 0x3f, 0x3c, 0x00, 0x73, 0xff, 0xf4, 0x46, 0xa4, 0xb1, 0xdd, 0x88, 0x8b,
- 0xb3, 0x9a, 0x3a, 0xff, 0xe2, 0x0c, 0xbf, 0x70, 0x1f, 0xae, 0x6f, 0xd7, 0xea, 0xe3, 0xea, 0x67,
- 0x52, 0xdc, 0x7b, 0xf2, 0xf5, 0x1e, 0x40, 0xfb, 0x52, 0x46, 0x74, 0x71, 0x56, 0xd7, 0x5c, 0x60,
- 0xc6, 0xac, 0x53, 0x67, 0x56, 0xf1, 0xa1, 0x96, 0xe1, 0xe1, 0x97, 0xd0, 0xdb, 0xfc, 0x0e, 0x70,
- 0x17, 0x42, 0x1b, 0x5c, 0x4a, 0x95, 0x88, 0x5b, 0xd6, 0xc0, 0x67, 0xb0, 0x6b, 0x81, 0x6d, 0x63,
- 0xe6, 0x1d, 0xfe, 0xed, 0x43, 0x58, 0x59, 0x70, 0x04, 0x68, 0xcf, 0xf2, 0xf8, 0x7c, 0x9d, 0xb1,
- 0x06, 0x86, 0xd0, 0x99, 0xe5, 0xf1, 0x09, 0x09, 0xcd, 0x3c, 0x17, 0xbc, 0x52, 0x32, 0x63, 0xbe,
- 0xcb, 0x7a, 0x91, 0x65, 0xac, 0x89, 0x03, 0x80, 0xe2, 0x3c, 0xa7, 0x3c, 0x63, 0x81, 0x4b, 0xfc,
- 0x5e, 0x6a, 0x62, 0x2d, 0x23, 0xc2, 0x05, 0x96, 0x6d, 0x3b, 0xd6, 0x2c, 0x13, 0xeb, 0x20, 0x83,
- 0xbe, 0x69, 0x46, 0x42, 0xe9, 0x6b, 0xd3, 0xa5, 0x8b, 0x43, 0x60, 0x55, 0xc4, 0x5e, 0xea, 0x21,
- 0xc2, 0x60, 0x96, 0xc7, 0x6f, 0x52, 0x45, 0x62, 0x71, 0x23, 0xae, 0x6f, 0x89, 0x01, 0xee, 0xc1,
- 0x8e, 0x2b, 0x64, 0x1e, 0x6f, 0x9d, 0xb3, 0xd0, 0xa5, 0x9d, 0xde, 0xd0, 0xe2, 0x97, 0xef, 0xd6,
- 0x52, 0xad, 0x13, 0xd6, 0xc7, 0x8f, 0x60, 0x6f, 0x96, 0xc7, 0x57, 0x4a, 0xa4, 0xf9, 0x92, 0xd4,
- 0x4b, 0x12, 0x11, 0x29, 0xb6, 0xe3, 0x6e, 0x5f, 0xad, 0x12, 0x92, 0x6b, 0x7d, 0x29, 0x7f, 0x65,
- 0x03, 0x27, 0x66, 0x4e, 0x22, 0xb2, 0x3f, 0x43, 0xb6, 0xeb, 0xc4, 0x6c, 0x10, 0x2b, 0x86, 0xb9,
- 0x79, 0x5f, 0x29, 0xb2, 0x23, 0xee, 0xb9, 0xae, 0x2e, 0xb6, 0x39, 0x78, 0x78, 0x07, 0x83, 0xfa,
- 0xf3, 0x1a, 0x1d, 0x5b, 0xe4, 0x45, 0x14, 0x99, 0xb7, 0x64, 0x0d, 0xe4, 0x30, 0xdc, 0xc2, 0x73,
- 0x4a, 0xe4, 0x5b, 0xb2, 0x8c, 0x57, 0x67, 0xde, 0x64, 0x91, 0xd0, 0x05, 0xe3, 0xe3, 0x01, 0xf0,
- 0x5a, 0xa9, 0x97, 0xc5, 0x36, 0x5a, 0xb6, 0x79, 0xc2, 0xef, 0x3f, 0x8c, 0x1a, 0xef, 0x3f, 0x8c,
- 0x1a, 0xf7, 0x0f, 0x23, 0xef, 0xfd, 0xc3, 0xc8, 0xfb, 0xf7, 0x61, 0xe4, 0xfd, 0xf9, 0xdf, 0xa8,
- 0xf1, 0x7f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x86, 0x52, 0x5b, 0xe0, 0x74, 0x06, 0x00, 0x00,
+ // 1009 bytes of a gzipped FileDescriptorProto
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x55, 0xcd, 0x6e, 0xe3, 0x36,
+ 0x17, 0xb5, 0x64, 0xc5, 0x3f, 0xd7, 0x8e, 0xc3, 0xdc, 0xc9, 0x37, 0x20, 0x82, 0xc0, 0xe3, 0xcf,
+ 0xd3, 0x62, 0x8c, 0x14, 0x93, 0x16, 0x5e, 0x14, 0x45, 0x77, 0xf9, 0x19, 0x20, 0x29, 0xe2, 0x74,
+ 0xea, 0x64, 0xb2, 0x28, 0x50, 0x04, 0x8c, 0x45, 0x2b, 0x6a, 0x2d, 0x51, 0xa0, 0xe8, 0x34, 0xd9,
+ 0x14, 0x45, 0x9f, 0xa2, 0x9b, 0xd9, 0xf6, 0x01, 0xfa, 0x14, 0x59, 0x0e, 0xd0, 0xfd, 0xa0, 0x93,
+ 0xbe, 0x48, 0x41, 0x8a, 0xb2, 0x65, 0x27, 0x98, 0x45, 0x77, 0xe4, 0x39, 0x87, 0xf7, 0x9e, 0x7b,
+ 0x79, 0x45, 0x01, 0x48, 0x36, 0x56, 0x3b, 0x89, 0x14, 0x4a, 0x60, 0x45, 0xaf, 0x93, 0xcb, 0xcd,
+ 0x8d, 0x40, 0x04, 0xc2, 0x40, 0x9f, 0xeb, 0x55, 0xc6, 0x76, 0x7f, 0x81, 0x95, 0x57, 0xb1, 0x92,
+ 0xb7, 0xf8, 0x19, 0x78, 0x67, 0xb7, 0x09, 0xa7, 0x4e, 0xc7, 0xe9, 0xb5, 0xfa, 0xeb, 0x3b, 0xd9,
+ 0xa9, 0x1d, 0x43, 0x6a, 0x62, 0xcf, 0xbb, 0x7b, 0xff, 0xac, 0x34, 0x34, 0x22, 0xa4, 0xe0, 0x9d,
+ 0x71, 0x19, 0x51, 0xb7, 0xe3, 0xf4, 0xbc, 0x19, 0xc3, 0x65, 0x84, 0x9b, 0xb0, 0x72, 0x14, 0xfb,
+ 0xfc, 0x86, 0x96, 0x0b, 0x54, 0x06, 0x21, 0x82, 0x77, 0xc0, 0x14, 0xa3, 0x5e, 0xc7, 0xe9, 0x35,
+ 0x87, 0x66, 0xdd, 0xfd, 0xd5, 0x01, 0x72, 0x1a, 0xb3, 0x24, 0xbd, 0x12, 0x6a, 0xc0, 0x15, 0xf3,
+ 0x99, 0x62, 0xf8, 0x25, 0xc0, 0x48, 0xc4, 0xe3, 0x8b, 0x54, 0x31, 0x95, 0x39, 0x6a, 0xcc, 0x1d,
+ 0xed, 0x8b, 0x78, 0x7c, 0xaa, 0x09, 0x1b, 0xbc, 0x3e, 0xca, 0x01, 0x9d, 0x3c, 0x34, 0xc9, 0x8b,
+ 0xbe, 0x32, 0x48, 0x5b, 0x56, 0xda, 0x72, 0xd1, 0x97, 0x41, 0xba, 0xdf, 0x43, 0x2d, 0x77, 0xa0,
+ 0x2d, 0x6a, 0x07, 0x26, 0x67, 0x73, 0x68, 0xd6, 0xf8, 0x35, 0xd4, 0x22, 0xeb, 0xcc, 0x04, 0x6e,
+ 0xf4, 0x69, 0xee, 0x65, 0xd9, 0xb9, 0x8d, 0x3b, 0xd3, 0x77, 0xdf, 0x96, 0xa1, 0x3a, 0xe0, 0x69,
+ 0xca, 0x02, 0x8e, 0x2f, 0xc1, 0x53, 0xf3, 0x0e, 0x3f, 0xc9, 0x63, 0x58, 0xba, 0xd8, 0x63, 0x2d,
+ 0xc3, 0x0d, 0x70, 0x95, 0x58, 0xa8, 0xc4, 0x55, 0x42, 0x97, 0x31, 0x96, 0x62, 0xa9, 0x0c, 0x8d,
+ 0xcc, 0x0a, 0xf4, 0x96, 0x0b, 0xc4, 0x36, 0x54, 0x27, 0x22, 0x30, 0x17, 0xb6, 0x52, 0x20, 0x73,
+ 0x70, 0xde, 0xb6, 0xca, 0xc3, 0xb6, 0xbd, 0x84, 0x2a, 0x8f, 0x95, 0x0c, 0x79, 0x4a, 0xab, 0x9d,
+ 0x72, 0xaf, 0xd1, 0x5f, 0x5d, 0x98, 0x8c, 0x3c, 0x94, 0xd5, 0xe0, 0x16, 0x54, 0x46, 0x22, 0x8a,
+ 0x42, 0x45, 0x6b, 0x85, 0x58, 0x16, 0xc3, 0x3e, 0xd4, 0x52, 0xdb, 0x31, 0x5a, 0x37, 0x9d, 0x24,
+ 0xcb, 0x9d, 0xcc, 0x3b, 0x98, 0xeb, 0x74, 0x44, 0xc9, 0x7f, 0xe4, 0x23, 0x45, 0xa1, 0xe3, 0xf4,
+ 0x6a, 0x79, 0xc4, 0x0c, 0xc3, 0x4f, 0x00, 0xb2, 0xd5, 0x61, 0x18, 0x2b, 0xda, 0x28, 0xe4, 0x2c,
+ 0xe0, 0x48, 0xa1, 0x3a, 0x12, 0xb1, 0xe2, 0x37, 0x8a, 0x36, 0xcd, 0xc5, 0xe6, 0xdb, 0xee, 0x0f,
+ 0x50, 0x3f, 0x64, 0xd2, 0xcf, 0xc6, 0x27, 0xef, 0xa0, 0xf3, 0xa0, 0x83, 0x14, 0xbc, 0x6b, 0xa1,
+ 0xf8, 0xe2, 0xbc, 0x6b, 0xa4, 0x50, 0x70, 0xf9, 0x61, 0xc1, 0xdd, 0x3f, 0x1d, 0xa8, 0xcf, 0xe6,
+ 0x15, 0x9f, 0x42, 0x45, 0x9f, 0x91, 0x29, 0x75, 0x3a, 0xe5, 0x9e, 0x37, 0xb4, 0x3b, 0xdc, 0x84,
+ 0xda, 0x84, 0x33, 0x19, 0x6b, 0xc6, 0x35, 0xcc, 0x6c, 0x8f, 0x2f, 0x60, 0x2d, 0x53, 0x5d, 0x88,
+ 0xa9, 0x0a, 0x44, 0x18, 0x07, 0xb4, 0x6c, 0x24, 0xad, 0x0c, 0xfe, 0xd6, 0xa2, 0xf8, 0x1c, 0x56,
+ 0xf3, 0x43, 0x17, 0xb1, 0xae, 0xd4, 0x33, 0xb2, 0x66, 0x0e, 0x9e, 0xf0, 0x1b, 0x85, 0xcf, 0x01,
+ 0xd8, 0x54, 0x89, 0x8b, 0x09, 0x67, 0xd7, 0xdc, 0x0c, 0x43, 0xde, 0xd0, 0xba, 0xc6, 0x8f, 0x35,
+ 0xdc, 0x7d, 0xeb, 0x00, 0x68, 0xd3, 0xfb, 0x57, 0x2c, 0x0e, 0xf4, 0x47, 0xe5, 0x86, 0xbe, 0xed,
+ 0x09, 0x68, 0xed, 0xfd, 0xfb, 0x67, 0xee, 0xd1, 0xc1, 0xd0, 0x0d, 0x7d, 0xfc, 0xc2, 0x8e, 0xb4,
+ 0x6b, 0x46, 0xfa, 0x69, 0xf1, 0x13, 0xcd, 0x4e, 0x3f, 0x98, 0xea, 0x17, 0x50, 0x8d, 0x85, 0xcf,
+ 0x2f, 0x42, 0xdf, 0x36, 0xac, 0x65, 0x43, 0x56, 0x4e, 0x84, 0xcf, 0x8f, 0x0e, 0x86, 0x15, 0x4d,
+ 0x1f, 0xf9, 0xc5, 0x3b, 0xf3, 0x16, 0xef, 0x2c, 0x02, 0x32, 0x4f, 0x70, 0x1a, 0xc6, 0xc1, 0x84,
+ 0xcf, 0x8c, 0x38, 0xff, 0xc5, 0x88, 0xfb, 0x31, 0x23, 0xdd, 0x3f, 0x1c, 0x68, 0xce, 0xe3, 0x9c,
+ 0xf7, 0x71, 0x0f, 0x40, 0x49, 0x16, 0xa7, 0xa1, 0x0a, 0x45, 0x6c, 0x33, 0x6e, 0x3d, 0x92, 0x71,
+ 0xa6, 0xc9, 0x27, 0x72, 0x7e, 0x0a, 0xbf, 0x82, 0xea, 0xc8, 0xa8, 0xb2, 0x1b, 0x2f, 0x3c, 0x29,
+ 0xcb, 0xa5, 0xe5, 0x5f, 0x98, 0x95, 0x17, 0xfb, 0x52, 0x5e, 0xe8, 0xcb, 0xf6, 0x21, 0xd4, 0x67,
+ 0xaf, 0x35, 0xae, 0x41, 0xc3, 0x6c, 0x4e, 0x84, 0x8c, 0xd8, 0x84, 0x94, 0xf0, 0x09, 0xac, 0x19,
+ 0x60, 0x1e, 0x9f, 0x38, 0xf8, 0x3f, 0x58, 0x5f, 0x02, 0xcf, 0xfb, 0xc4, 0xdd, 0xfe, 0xcb, 0x85,
+ 0x46, 0xe1, 0x59, 0x42, 0x80, 0xca, 0x20, 0x0d, 0x0e, 0xa7, 0x09, 0x29, 0x61, 0x03, 0xaa, 0x83,
+ 0x34, 0xd8, 0xe3, 0x4c, 0x11, 0xc7, 0x6e, 0x5e, 0x4b, 0x91, 0x10, 0xd7, 0xaa, 0x76, 0x93, 0x84,
+ 0x94, 0xb1, 0x05, 0x90, 0xad, 0x87, 0x3c, 0x4d, 0x88, 0x67, 0x85, 0xe7, 0x42, 0x71, 0xb2, 0xa2,
+ 0xbd, 0xd9, 0x8d, 0x61, 0x2b, 0x96, 0xd5, 0x4f, 0x00, 0xa9, 0x22, 0x81, 0xa6, 0x4e, 0xc6, 0x99,
+ 0x54, 0x97, 0x3a, 0x4b, 0x0d, 0x37, 0x80, 0x14, 0x11, 0x73, 0xa8, 0x8e, 0x08, 0xad, 0x41, 0x1a,
+ 0xbc, 0x89, 0x25, 0x67, 0xa3, 0x2b, 0x76, 0x39, 0xe1, 0x04, 0x70, 0x1d, 0x56, 0x6d, 0x20, 0xfd,
+ 0xc5, 0x4d, 0x53, 0xd2, 0xb0, 0xb2, 0xfd, 0x2b, 0x3e, 0xfa, 0xe9, 0xbb, 0xa9, 0x90, 0xd3, 0x88,
+ 0x34, 0x75, 0xd9, 0x83, 0x34, 0x30, 0x17, 0x34, 0xe6, 0xf2, 0x98, 0x33, 0x9f, 0x4b, 0xb2, 0x6a,
+ 0x4f, 0x9f, 0x85, 0x11, 0x17, 0x53, 0x75, 0x22, 0x7e, 0x26, 0x2d, 0x6b, 0x66, 0xc8, 0x99, 0x6f,
+ 0x7e, 0x61, 0x64, 0xcd, 0x9a, 0x99, 0x21, 0xc6, 0x0c, 0xb1, 0xf5, 0xbe, 0x96, 0xdc, 0x94, 0xb8,
+ 0x6e, 0xb3, 0xda, 0xbd, 0xd1, 0xe0, 0xf6, 0x6f, 0x0e, 0x6c, 0x3c, 0x36, 0x1e, 0xb8, 0x05, 0xf4,
+ 0x31, 0x7c, 0x77, 0xaa, 0x04, 0x29, 0xe1, 0xa7, 0xf0, 0xff, 0xc7, 0xd8, 0x6f, 0x44, 0x18, 0xab,
+ 0xa3, 0x28, 0x99, 0x84, 0xa3, 0x50, 0x5f, 0xc5, 0xc7, 0x64, 0xaf, 0x6e, 0xac, 0xcc, 0xdd, 0xbe,
+ 0x85, 0xd6, 0xe2, 0x47, 0xa1, 0x9b, 0x31, 0x47, 0x76, 0x7d, 0x5f, 0x8f, 0x3f, 0x29, 0x21, 0x2d,
+ 0x9a, 0x1d, 0xf2, 0x48, 0x5c, 0x73, 0xc3, 0x38, 0x8b, 0xcc, 0x9b, 0xc4, 0x67, 0x2a, 0x63, 0xdc,
+ 0xc5, 0x42, 0x76, 0x7d, 0xff, 0x38, 0x7b, 0x7b, 0x0c, 0x5b, 0xde, 0xa3, 0x77, 0x1f, 0xda, 0xa5,
+ 0x77, 0x1f, 0xda, 0xa5, 0xbb, 0xfb, 0xb6, 0xf3, 0xee, 0xbe, 0xed, 0xfc, 0x7d, 0xdf, 0x76, 0x7e,
+ 0xff, 0xa7, 0x5d, 0xfa, 0x37, 0x00, 0x00, 0xff, 0xff, 0x87, 0x11, 0x6d, 0xd6, 0xaf, 0x08, 0x00,
+ 0x00,
}
diff --git a/vendor/go.etcd.io/etcd/raft/raftpb/raft.proto b/vendor/go.etcd.io/etcd/raft/raftpb/raft.proto
index 644ce7b..23d62ec 100644
--- a/vendor/go.etcd.io/etcd/raft/raftpb/raft.proto
+++ b/vendor/go.etcd.io/etcd/raft/raftpb/raft.proto
@@ -10,8 +10,9 @@
option (gogoproto.goproto_enum_prefix_all) = false;
enum EntryType {
- EntryNormal = 0;
- EntryConfChange = 1;
+ EntryNormal = 0;
+ EntryConfChange = 1; // corresponds to pb.ConfChange
+ EntryConfChangeV2 = 2; // corresponds to pb.ConfChangeV2
}
message Entry {
@@ -75,9 +76,41 @@
optional uint64 commit = 3 [(gogoproto.nullable) = false];
}
+// ConfChangeTransition specifies the behavior of a configuration change with
+// respect to joint consensus.
+enum ConfChangeTransition {
+ // Automatically use the simple protocol if possible, otherwise fall back
+ // to ConfChangeJointImplicit. Most applications will want to use this.
+ ConfChangeTransitionAuto = 0;
+ // Use joint consensus unconditionally, and transition out of them
+ // automatically (by proposing a zero configuration change).
+ //
+ // This option is suitable for applications that want to minimize the time
+ // spent in the joint configuration and do not store the joint configuration
+ // in the state machine (outside of InitialState).
+ ConfChangeTransitionJointImplicit = 1;
+ // Use joint consensus and remain in the joint configuration until the
+ // application proposes a no-op configuration change. This is suitable for
+ // applications that want to explicitly control the transitions, for example
+ // to use a custom payload (via the Context field).
+ ConfChangeTransitionJointExplicit = 2;
+}
+
message ConfState {
- repeated uint64 nodes = 1;
- repeated uint64 learners = 2;
+ // The voters in the incoming config. (If the configuration is not joint,
+ // then the outgoing config is empty).
+ repeated uint64 voters = 1;
+ // The learners in the incoming config.
+ repeated uint64 learners = 2;
+ // The voters in the outgoing config.
+ repeated uint64 voters_outgoing = 3;
+ // The nodes that will become learners when the outgoing config is removed.
+ // These nodes are necessarily currently in nodes_joint (or they would have
+ // been added to the incoming config right away).
+ repeated uint64 learners_next = 4;
+ // If set, the config is joint and Raft will automatically transition into
+ // the final config (i.e. remove the outgoing config) when this is safe.
+ optional bool auto_leave = 5 [(gogoproto.nullable) = false];
}
enum ConfChangeType {
@@ -88,8 +121,57 @@
}
message ConfChange {
- optional uint64 ID = 1 [(gogoproto.nullable) = false];
- optional ConfChangeType Type = 2 [(gogoproto.nullable) = false];
- optional uint64 NodeID = 3 [(gogoproto.nullable) = false];
- optional bytes Context = 4;
+ optional ConfChangeType type = 2 [(gogoproto.nullable) = false];
+ optional uint64 node_id = 3 [(gogoproto.nullable) = false, (gogoproto.customname) = "NodeID" ];
+ optional bytes context = 4;
+
+ // NB: this is used only by etcd to thread through a unique identifier.
+ // Ideally it should really use the Context instead. No counterpart to
+ // this field exists in ConfChangeV2.
+ optional uint64 id = 1 [(gogoproto.nullable) = false, (gogoproto.customname) = "ID" ];
+}
+
+// ConfChangeSingle is an individual configuration change operation. Multiple
+// such operations can be carried out atomically via a ConfChangeV2.
+message ConfChangeSingle {
+ optional ConfChangeType type = 1 [(gogoproto.nullable) = false];
+ optional uint64 node_id = 2 [(gogoproto.nullable) = false, (gogoproto.customname) = "NodeID"];
+}
+
+// ConfChangeV2 messages initiate configuration changes. They support both the
+// simple "one at a time" membership change protocol and full Joint Consensus
+// allowing for arbitrary changes in membership.
+//
+// The supplied context is treated as an opaque payload and can be used to
+// attach an action on the state machine to the application of the config change
+// proposal. Note that contrary to Joint Consensus as outlined in the Raft
+// paper[1], configuration changes become active when they are *applied* to the
+// state machine (not when they are appended to the log).
+//
+// The simple protocol can be used whenever only a single change is made.
+//
+// Non-simple changes require the use of Joint Consensus, for which two
+// configuration changes are run. The first configuration change specifies the
+// desired changes and transitions the Raft group into the joint configuration,
+// in which quorum requires a majority of both the pre-changes and post-changes
+// configuration. Joint Consensus avoids entering fragile intermediate
+// configurations that could compromise survivability. For example, without the
+// use of Joint Consensus and running across three availability zones with a
+// replication factor of three, it is not possible to replace a voter without
+// entering an intermediate configuration that does not survive the outage of
+// one availability zone.
+//
+// The provided ConfChangeTransition specifies how (and whether) Joint Consensus
+// is used, and assigns the task of leaving the joint configuration either to
+// Raft or the application. Leaving the joint configuration is accomplished by
+// proposing a ConfChangeV2 with only and optionally the Context field
+// populated.
+//
+// For details on Raft membership changes, see:
+//
+// [1]: https://github.com/ongardie/dissertation/blob/master/online-trim.pdf
+message ConfChangeV2 {
+ optional ConfChangeTransition transition = 1 [(gogoproto.nullable) = false];
+ repeated ConfChangeSingle changes = 2 [(gogoproto.nullable) = false];
+ optional bytes context = 3;
}
diff --git a/vendor/go.etcd.io/etcd/raft/rawnode.go b/vendor/go.etcd.io/etcd/raft/rawnode.go
index d7a272d..90eb694 100644
--- a/vendor/go.etcd.io/etcd/raft/rawnode.go
+++ b/vendor/go.etcd.io/etcd/raft/rawnode.go
@@ -18,6 +18,7 @@
"errors"
pb "go.etcd.io/etcd/raft/raftpb"
+ "go.etcd.io/etcd/raft/tracker"
)
// ErrStepLocalMsg is returned when try to step a local raft message
@@ -36,82 +37,20 @@
prevHardSt pb.HardState
}
-func (rn *RawNode) newReady() Ready {
- return newReady(rn.raft, rn.prevSoftSt, rn.prevHardSt)
-}
-
-func (rn *RawNode) commitReady(rd Ready) {
- if rd.SoftState != nil {
- rn.prevSoftSt = rd.SoftState
- }
- if !IsEmptyHardState(rd.HardState) {
- rn.prevHardSt = rd.HardState
- }
-
- // If entries were applied (or a snapshot), update our cursor for
- // the next Ready. Note that if the current HardState contains a
- // new Commit index, this does not mean that we're also applying
- // all of the new entries due to commit pagination by size.
- if index := rd.appliedCursor(); index > 0 {
- rn.raft.raftLog.appliedTo(index)
- }
-
- if len(rd.Entries) > 0 {
- e := rd.Entries[len(rd.Entries)-1]
- rn.raft.raftLog.stableTo(e.Index, e.Term)
- }
- if !IsEmptySnap(rd.Snapshot) {
- rn.raft.raftLog.stableSnapTo(rd.Snapshot.Metadata.Index)
- }
- if len(rd.ReadStates) != 0 {
- rn.raft.readStates = nil
- }
-}
-
-// NewRawNode returns a new RawNode given configuration and a list of raft peers.
-func NewRawNode(config *Config, peers []Peer) (*RawNode, error) {
- if config.ID == 0 {
- panic("config.ID must not be zero")
- }
+// NewRawNode instantiates a RawNode from the given configuration.
+//
+// See Bootstrap() for bootstrapping an initial state; this replaces the former
+// 'peers' argument to this method (with identical behavior). However, It is
+// recommended that instead of calling Bootstrap, applications bootstrap their
+// state manually by setting up a Storage that has a first index > 1 and which
+// stores the desired ConfState as its InitialState.
+func NewRawNode(config *Config) (*RawNode, error) {
r := newRaft(config)
rn := &RawNode{
raft: r,
}
- lastIndex, err := config.Storage.LastIndex()
- if err != nil {
- panic(err) // TODO(bdarnell)
- }
- // If the log is empty, this is a new RawNode (like StartNode); otherwise it's
- // restoring an existing RawNode (like RestartNode).
- // TODO(bdarnell): rethink RawNode initialization and whether the application needs
- // to be able to tell us when it expects the RawNode to exist.
- if lastIndex == 0 {
- r.becomeFollower(1, None)
- ents := make([]pb.Entry, len(peers))
- for i, peer := range peers {
- cc := pb.ConfChange{Type: pb.ConfChangeAddNode, NodeID: peer.ID, Context: peer.Context}
- data, err := cc.Marshal()
- if err != nil {
- panic("unexpected marshal error")
- }
-
- ents[i] = pb.Entry{Type: pb.EntryConfChange, Term: 1, Index: uint64(i + 1), Data: data}
- }
- r.raftLog.append(ents...)
- r.raftLog.committed = uint64(len(ents))
- for _, peer := range peers {
- r.addNode(peer.ID)
- }
- }
-
- // Set the initial hard and soft states after performing all initialization.
rn.prevSoftSt = r.softState()
- if lastIndex == 0 {
- rn.prevHardSt = emptyState
- } else {
- rn.prevHardSt = r.hardState()
- }
-
+ rn.prevHardSt = r.hardState()
return rn, nil
}
@@ -149,37 +88,20 @@
}})
}
-// ProposeConfChange proposes a config change.
-func (rn *RawNode) ProposeConfChange(cc pb.ConfChange) error {
- data, err := cc.Marshal()
+// ProposeConfChange proposes a config change. See (Node).ProposeConfChange for
+// details.
+func (rn *RawNode) ProposeConfChange(cc pb.ConfChangeI) error {
+ m, err := confChangeToMsg(cc)
if err != nil {
return err
}
- return rn.raft.Step(pb.Message{
- Type: pb.MsgProp,
- Entries: []pb.Entry{
- {Type: pb.EntryConfChange, Data: data},
- },
- })
+ return rn.raft.Step(m)
}
// ApplyConfChange applies a config change to the local node.
-func (rn *RawNode) ApplyConfChange(cc pb.ConfChange) *pb.ConfState {
- if cc.NodeID == None {
- return &pb.ConfState{Nodes: rn.raft.nodes(), Learners: rn.raft.learnerNodes()}
- }
- switch cc.Type {
- case pb.ConfChangeAddNode:
- rn.raft.addNode(cc.NodeID)
- case pb.ConfChangeAddLearnerNode:
- rn.raft.addLearner(cc.NodeID)
- case pb.ConfChangeRemoveNode:
- rn.raft.removeNode(cc.NodeID)
- case pb.ConfChangeUpdateNode:
- default:
- panic("unexpected conf type")
- }
- return &pb.ConfState{Nodes: rn.raft.nodes(), Learners: rn.raft.learnerNodes()}
+func (rn *RawNode) ApplyConfChange(cc pb.ConfChangeI) *pb.ConfState {
+ cs := rn.raft.applyConfChange(cc.AsV2())
+ return &cs
}
// Step advances the state machine using the given message.
@@ -188,20 +110,41 @@
if IsLocalMsg(m.Type) {
return ErrStepLocalMsg
}
- if pr := rn.raft.getProgress(m.From); pr != nil || !IsResponseMsg(m.Type) {
+ if pr := rn.raft.prs.Progress[m.From]; pr != nil || !IsResponseMsg(m.Type) {
return rn.raft.Step(m)
}
return ErrStepPeerNotFound
}
-// Ready returns the current point-in-time state of this RawNode.
+// Ready returns the outstanding work that the application needs to handle. This
+// includes appending and applying entries or a snapshot, updating the HardState,
+// and sending messages. The returned Ready() *must* be handled and subsequently
+// passed back via Advance().
func (rn *RawNode) Ready() Ready {
- rd := rn.newReady()
- rn.raft.msgs = nil
- rn.raft.reduceUncommittedSize(rd.CommittedEntries)
+ rd := rn.readyWithoutAccept()
+ rn.acceptReady(rd)
return rd
}
+// readyWithoutAccept returns a Ready. This is a read-only operation, i.e. there
+// is no obligation that the Ready must be handled.
+func (rn *RawNode) readyWithoutAccept() Ready {
+ return newReady(rn.raft, rn.prevSoftSt, rn.prevHardSt)
+}
+
+// acceptReady is called when the consumer of the RawNode has decided to go
+// ahead and handle a Ready. Nothing must alter the state of the RawNode between
+// this call and the prior call to Ready().
+func (rn *RawNode) acceptReady(rd Ready) {
+ if rd.SoftState != nil {
+ rn.prevSoftSt = rd.SoftState
+ }
+ if len(rd.ReadStates) != 0 {
+ rn.raft.readStates = nil
+ }
+ rn.raft.msgs = nil
+}
+
// HasReady called when RawNode user need to check if any Ready pending.
// Checking logic in this method should be consistent with Ready.containsUpdates().
func (rn *RawNode) HasReady() bool {
@@ -227,21 +170,23 @@
// Advance notifies the RawNode that the application has applied and saved progress in the
// last Ready results.
func (rn *RawNode) Advance(rd Ready) {
- rn.commitReady(rd)
+ if !IsEmptyHardState(rd.HardState) {
+ rn.prevHardSt = rd.HardState
+ }
+ rn.raft.advance(rd)
}
-// Status returns the current status of the given group.
-func (rn *RawNode) Status() *Status {
+// Status returns the current status of the given group. This allocates, see
+// BasicStatus and WithProgress for allocation-friendlier choices.
+func (rn *RawNode) Status() Status {
status := getStatus(rn.raft)
- return &status
+ return status
}
-// StatusWithoutProgress returns a Status without populating the Progress field
-// (and returns the Status as a value to avoid forcing it onto the heap). This
-// is more performant if the Progress is not required. See WithProgress for an
-// allocation-free way to introspect the Progress.
-func (rn *RawNode) StatusWithoutProgress() Status {
- return getStatusWithoutProgress(rn.raft)
+// BasicStatus returns a BasicStatus. Notably this does not contain the
+// Progress map; see WithProgress for an allocation-free way to inspect it.
+func (rn *RawNode) BasicStatus() BasicStatus {
+ return getBasicStatus(rn.raft)
}
// ProgressType indicates the type of replica a Progress corresponds to.
@@ -256,17 +201,16 @@
// WithProgress is a helper to introspect the Progress for this node and its
// peers.
-func (rn *RawNode) WithProgress(visitor func(id uint64, typ ProgressType, pr Progress)) {
- for id, pr := range rn.raft.prs {
- pr := *pr
- pr.ins = nil
- visitor(id, ProgressTypePeer, pr)
- }
- for id, pr := range rn.raft.learnerPrs {
- pr := *pr
- pr.ins = nil
- visitor(id, ProgressTypeLearner, pr)
- }
+func (rn *RawNode) WithProgress(visitor func(id uint64, typ ProgressType, pr tracker.Progress)) {
+ rn.raft.prs.Visit(func(id uint64, pr *tracker.Progress) {
+ typ := ProgressTypePeer
+ if pr.IsLearner {
+ typ = ProgressTypeLearner
+ }
+ p := *pr
+ p.Inflights = nil
+ visitor(id, typ, p)
+ })
}
// ReportUnreachable reports the given node is not reachable for the last send.
diff --git a/vendor/go.etcd.io/etcd/raft/read_only.go b/vendor/go.etcd.io/etcd/raft/read_only.go
index aecc6b2..6987f1b 100644
--- a/vendor/go.etcd.io/etcd/raft/read_only.go
+++ b/vendor/go.etcd.io/etcd/raft/read_only.go
@@ -29,7 +29,11 @@
type readIndexStatus struct {
req pb.Message
index uint64
- acks map[uint64]struct{}
+ // NB: this never records 'false', but it's more convenient to use this
+ // instead of a map[uint64]struct{} due to the API of quorum.VoteResult. If
+ // this becomes performance sensitive enough (doubtful), quorum.VoteResult
+ // can change to an API that is closer to that of CommittedIndex.
+ acks map[uint64]bool
}
type readOnly struct {
@@ -50,26 +54,25 @@
// the read only request.
// `m` is the original read only request message from the local or remote node.
func (ro *readOnly) addRequest(index uint64, m pb.Message) {
- ctx := string(m.Entries[0].Data)
- if _, ok := ro.pendingReadIndex[ctx]; ok {
+ s := string(m.Entries[0].Data)
+ if _, ok := ro.pendingReadIndex[s]; ok {
return
}
- ro.pendingReadIndex[ctx] = &readIndexStatus{index: index, req: m, acks: make(map[uint64]struct{})}
- ro.readIndexQueue = append(ro.readIndexQueue, ctx)
+ ro.pendingReadIndex[s] = &readIndexStatus{index: index, req: m, acks: make(map[uint64]bool)}
+ ro.readIndexQueue = append(ro.readIndexQueue, s)
}
// recvAck notifies the readonly struct that the raft state machine received
// an acknowledgment of the heartbeat that attached with the read only request
// context.
-func (ro *readOnly) recvAck(m pb.Message) int {
- rs, ok := ro.pendingReadIndex[string(m.Context)]
+func (ro *readOnly) recvAck(id uint64, context []byte) map[uint64]bool {
+ rs, ok := ro.pendingReadIndex[string(context)]
if !ok {
- return 0
+ return nil
}
- rs.acks[m.From] = struct{}{}
- // add one to include an ack from local node
- return len(rs.acks) + 1
+ rs.acks[id] = true
+ return rs.acks
}
// advance advances the read only request queue kept by the readonly struct.
diff --git a/vendor/go.etcd.io/etcd/raft/status.go b/vendor/go.etcd.io/etcd/raft/status.go
index 9feca7c..adc6048 100644
--- a/vendor/go.etcd.io/etcd/raft/status.go
+++ b/vendor/go.etcd.io/etcd/raft/status.go
@@ -18,34 +18,44 @@
"fmt"
pb "go.etcd.io/etcd/raft/raftpb"
+ "go.etcd.io/etcd/raft/tracker"
)
+// Status contains information about this Raft peer and its view of the system.
+// The Progress is only populated on the leader.
type Status struct {
+ BasicStatus
+ Config tracker.Config
+ Progress map[uint64]tracker.Progress
+}
+
+// BasicStatus contains basic information about the Raft peer. It does not allocate.
+type BasicStatus struct {
ID uint64
pb.HardState
SoftState
- Applied uint64
- Progress map[uint64]Progress
+ Applied uint64
LeadTransferee uint64
}
-func getProgressCopy(r *raft) map[uint64]Progress {
- prs := make(map[uint64]Progress)
- for id, p := range r.prs {
- prs[id] = *p
- }
+func getProgressCopy(r *raft) map[uint64]tracker.Progress {
+ m := make(map[uint64]tracker.Progress)
+ r.prs.Visit(func(id uint64, pr *tracker.Progress) {
+ var p tracker.Progress
+ p = *pr
+ p.Inflights = pr.Inflights.Clone()
+ pr = nil
- for id, p := range r.learnerPrs {
- prs[id] = *p
- }
- return prs
+ m[id] = p
+ })
+ return m
}
-func getStatusWithoutProgress(r *raft) Status {
- s := Status{
+func getBasicStatus(r *raft) BasicStatus {
+ s := BasicStatus{
ID: r.id,
LeadTransferee: r.leadTransferee,
}
@@ -57,10 +67,12 @@
// getStatus gets a copy of the current raft status.
func getStatus(r *raft) Status {
- s := getStatusWithoutProgress(r)
+ var s Status
+ s.BasicStatus = getBasicStatus(r)
if s.RaftState == StateLeader {
s.Progress = getProgressCopy(r)
}
+ s.Config = r.prs.Config.Clone()
return s
}
diff --git a/vendor/go.etcd.io/etcd/raft/storage.go b/vendor/go.etcd.io/etcd/raft/storage.go
index 14ad686..6be5745 100644
--- a/vendor/go.etcd.io/etcd/raft/storage.go
+++ b/vendor/go.etcd.io/etcd/raft/storage.go
@@ -44,6 +44,8 @@
// become inoperable and refuse to participate in elections; the
// application is responsible for cleanup and recovery in this case.
type Storage interface {
+ // TODO(tbg): split this into two interfaces, LogStorage and StateStorage.
+
// InitialState returns the saved HardState and ConfState information.
InitialState() (pb.HardState, pb.ConfState, error)
// Entries returns a slice of log entries in the range [lo,hi).
diff --git a/vendor/go.etcd.io/etcd/raft/tracker/inflights.go b/vendor/go.etcd.io/etcd/raft/tracker/inflights.go
new file mode 100644
index 0000000..1a05634
--- /dev/null
+++ b/vendor/go.etcd.io/etcd/raft/tracker/inflights.go
@@ -0,0 +1,132 @@
+// Copyright 2019 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tracker
+
+// Inflights limits the number of MsgApp (represented by the largest index
+// contained within) sent to followers but not yet acknowledged by them. Callers
+// use Full() to check whether more messages can be sent, call Add() whenever
+// they are sending a new append, and release "quota" via FreeLE() whenever an
+// ack is received.
+type Inflights struct {
+ // the starting index in the buffer
+ start int
+ // number of inflights in the buffer
+ count int
+
+ // the size of the buffer
+ size int
+
+ // buffer contains the index of the last entry
+ // inside one message.
+ buffer []uint64
+}
+
+// NewInflights sets up an Inflights that allows up to 'size' inflight messages.
+func NewInflights(size int) *Inflights {
+ return &Inflights{
+ size: size,
+ }
+}
+
+// Clone returns an *Inflights that is identical to but shares no memory with
+// the receiver.
+func (in *Inflights) Clone() *Inflights {
+ ins := *in
+ ins.buffer = append([]uint64(nil), in.buffer...)
+ return &ins
+}
+
+// Add notifies the Inflights that a new message with the given index is being
+// dispatched. Full() must be called prior to Add() to verify that there is room
+// for one more message, and consecutive calls to add Add() must provide a
+// monotonic sequence of indexes.
+func (in *Inflights) Add(inflight uint64) {
+ if in.Full() {
+ panic("cannot add into a Full inflights")
+ }
+ next := in.start + in.count
+ size := in.size
+ if next >= size {
+ next -= size
+ }
+ if next >= len(in.buffer) {
+ in.grow()
+ }
+ in.buffer[next] = inflight
+ in.count++
+}
+
+// grow the inflight buffer by doubling up to inflights.size. We grow on demand
+// instead of preallocating to inflights.size to handle systems which have
+// thousands of Raft groups per process.
+func (in *Inflights) grow() {
+ newSize := len(in.buffer) * 2
+ if newSize == 0 {
+ newSize = 1
+ } else if newSize > in.size {
+ newSize = in.size
+ }
+ newBuffer := make([]uint64, newSize)
+ copy(newBuffer, in.buffer)
+ in.buffer = newBuffer
+}
+
+// FreeLE frees the inflights smaller or equal to the given `to` flight.
+func (in *Inflights) FreeLE(to uint64) {
+ if in.count == 0 || to < in.buffer[in.start] {
+ // out of the left side of the window
+ return
+ }
+
+ idx := in.start
+ var i int
+ for i = 0; i < in.count; i++ {
+ if to < in.buffer[idx] { // found the first large inflight
+ break
+ }
+
+ // increase index and maybe rotate
+ size := in.size
+ if idx++; idx >= size {
+ idx -= size
+ }
+ }
+ // free i inflights and set new start index
+ in.count -= i
+ in.start = idx
+ if in.count == 0 {
+ // inflights is empty, reset the start index so that we don't grow the
+ // buffer unnecessarily.
+ in.start = 0
+ }
+}
+
+// FreeFirstOne releases the first inflight. This is a no-op if nothing is
+// inflight.
+func (in *Inflights) FreeFirstOne() { in.FreeLE(in.buffer[in.start]) }
+
+// Full returns true if no more messages can be sent at the moment.
+func (in *Inflights) Full() bool {
+ return in.count == in.size
+}
+
+// Count returns the number of inflight messages.
+func (in *Inflights) Count() int { return in.count }
+
+// reset frees all inflights.
+func (in *Inflights) reset() {
+ in.count = 0
+ in.start = 0
+}
diff --git a/vendor/go.etcd.io/etcd/raft/tracker/progress.go b/vendor/go.etcd.io/etcd/raft/tracker/progress.go
new file mode 100644
index 0000000..62c81f4
--- /dev/null
+++ b/vendor/go.etcd.io/etcd/raft/tracker/progress.go
@@ -0,0 +1,259 @@
+// Copyright 2019 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tracker
+
+import (
+ "fmt"
+ "sort"
+ "strings"
+)
+
+// Progress represents a follower’s progress in the view of the leader. Leader
+// maintains progresses of all followers, and sends entries to the follower
+// based on its progress.
+//
+// NB(tbg): Progress is basically a state machine whose transitions are mostly
+// strewn around `*raft.raft`. Additionally, some fields are only used when in a
+// certain State. All of this isn't ideal.
+type Progress struct {
+ Match, Next uint64
+ // State defines how the leader should interact with the follower.
+ //
+ // When in StateProbe, leader sends at most one replication message
+ // per heartbeat interval. It also probes actual progress of the follower.
+ //
+ // When in StateReplicate, leader optimistically increases next
+ // to the latest entry sent after sending replication message. This is
+ // an optimized state for fast replicating log entries to the follower.
+ //
+ // When in StateSnapshot, leader should have sent out snapshot
+ // before and stops sending any replication message.
+ State StateType
+
+ // PendingSnapshot is used in StateSnapshot.
+ // If there is a pending snapshot, the pendingSnapshot will be set to the
+ // index of the snapshot. If pendingSnapshot is set, the replication process of
+ // this Progress will be paused. raft will not resend snapshot until the pending one
+ // is reported to be failed.
+ PendingSnapshot uint64
+
+ // RecentActive is true if the progress is recently active. Receiving any messages
+ // from the corresponding follower indicates the progress is active.
+ // RecentActive can be reset to false after an election timeout.
+ //
+ // TODO(tbg): the leader should always have this set to true.
+ RecentActive bool
+
+ // ProbeSent is used while this follower is in StateProbe. When ProbeSent is
+ // true, raft should pause sending replication message to this peer until
+ // ProbeSent is reset. See ProbeAcked() and IsPaused().
+ ProbeSent bool
+
+ // Inflights is a sliding window for the inflight messages.
+ // Each inflight message contains one or more log entries.
+ // The max number of entries per message is defined in raft config as MaxSizePerMsg.
+ // Thus inflight effectively limits both the number of inflight messages
+ // and the bandwidth each Progress can use.
+ // When inflights is Full, no more message should be sent.
+ // When a leader sends out a message, the index of the last
+ // entry should be added to inflights. The index MUST be added
+ // into inflights in order.
+ // When a leader receives a reply, the previous inflights should
+ // be freed by calling inflights.FreeLE with the index of the last
+ // received entry.
+ Inflights *Inflights
+
+ // IsLearner is true if this progress is tracked for a learner.
+ IsLearner bool
+}
+
+// ResetState moves the Progress into the specified State, resetting ProbeSent,
+// PendingSnapshot, and Inflights.
+func (pr *Progress) ResetState(state StateType) {
+ pr.ProbeSent = false
+ pr.PendingSnapshot = 0
+ pr.State = state
+ pr.Inflights.reset()
+}
+
+func max(a, b uint64) uint64 {
+ if a > b {
+ return a
+ }
+ return b
+}
+
+func min(a, b uint64) uint64 {
+ if a > b {
+ return b
+ }
+ return a
+}
+
+// ProbeAcked is called when this peer has accepted an append. It resets
+// ProbeSent to signal that additional append messages should be sent without
+// further delay.
+func (pr *Progress) ProbeAcked() {
+ pr.ProbeSent = false
+}
+
+// BecomeProbe transitions into StateProbe. Next is reset to Match+1 or,
+// optionally and if larger, the index of the pending snapshot.
+func (pr *Progress) BecomeProbe() {
+ // If the original state is StateSnapshot, progress knows that
+ // the pending snapshot has been sent to this peer successfully, then
+ // probes from pendingSnapshot + 1.
+ if pr.State == StateSnapshot {
+ pendingSnapshot := pr.PendingSnapshot
+ pr.ResetState(StateProbe)
+ pr.Next = max(pr.Match+1, pendingSnapshot+1)
+ } else {
+ pr.ResetState(StateProbe)
+ pr.Next = pr.Match + 1
+ }
+}
+
+// BecomeReplicate transitions into StateReplicate, resetting Next to Match+1.
+func (pr *Progress) BecomeReplicate() {
+ pr.ResetState(StateReplicate)
+ pr.Next = pr.Match + 1
+}
+
+// BecomeSnapshot moves the Progress to StateSnapshot with the specified pending
+// snapshot index.
+func (pr *Progress) BecomeSnapshot(snapshoti uint64) {
+ pr.ResetState(StateSnapshot)
+ pr.PendingSnapshot = snapshoti
+}
+
+// MaybeUpdate is called when an MsgAppResp arrives from the follower, with the
+// index acked by it. The method returns false if the given n index comes from
+// an outdated message. Otherwise it updates the progress and returns true.
+func (pr *Progress) MaybeUpdate(n uint64) bool {
+ var updated bool
+ if pr.Match < n {
+ pr.Match = n
+ updated = true
+ pr.ProbeAcked()
+ }
+ if pr.Next < n+1 {
+ pr.Next = n + 1
+ }
+ return updated
+}
+
+// OptimisticUpdate signals that appends all the way up to and including index n
+// are in-flight. As a result, Next is increased to n+1.
+func (pr *Progress) OptimisticUpdate(n uint64) { pr.Next = n + 1 }
+
+// MaybeDecrTo adjusts the Progress to the receipt of a MsgApp rejection. The
+// arguments are the index the follower rejected to append to its log, and its
+// last index.
+//
+// Rejections can happen spuriously as messages are sent out of order or
+// duplicated. In such cases, the rejection pertains to an index that the
+// Progress already knows were previously acknowledged, and false is returned
+// without changing the Progress.
+//
+// If the rejection is genuine, Next is lowered sensibly, and the Progress is
+// cleared for sending log entries.
+func (pr *Progress) MaybeDecrTo(rejected, last uint64) bool {
+ if pr.State == StateReplicate {
+ // The rejection must be stale if the progress has matched and "rejected"
+ // is smaller than "match".
+ if rejected <= pr.Match {
+ return false
+ }
+ // Directly decrease next to match + 1.
+ //
+ // TODO(tbg): why not use last if it's larger?
+ pr.Next = pr.Match + 1
+ return true
+ }
+
+ // The rejection must be stale if "rejected" does not match next - 1. This
+ // is because non-replicating followers are probed one entry at a time.
+ if pr.Next-1 != rejected {
+ return false
+ }
+
+ if pr.Next = min(rejected, last+1); pr.Next < 1 {
+ pr.Next = 1
+ }
+ pr.ProbeSent = false
+ return true
+}
+
+// IsPaused returns whether sending log entries to this node has been throttled.
+// This is done when a node has rejected recent MsgApps, is currently waiting
+// for a snapshot, or has reached the MaxInflightMsgs limit. In normal
+// operation, this is false. A throttled node will be contacted less frequently
+// until it has reached a state in which it's able to accept a steady stream of
+// log entries again.
+func (pr *Progress) IsPaused() bool {
+ switch pr.State {
+ case StateProbe:
+ return pr.ProbeSent
+ case StateReplicate:
+ return pr.Inflights.Full()
+ case StateSnapshot:
+ return true
+ default:
+ panic("unexpected state")
+ }
+}
+
+func (pr *Progress) String() string {
+ var buf strings.Builder
+ fmt.Fprintf(&buf, "%s match=%d next=%d", pr.State, pr.Match, pr.Next)
+ if pr.IsLearner {
+ fmt.Fprint(&buf, " learner")
+ }
+ if pr.IsPaused() {
+ fmt.Fprint(&buf, " paused")
+ }
+ if pr.PendingSnapshot > 0 {
+ fmt.Fprintf(&buf, " pendingSnap=%d", pr.PendingSnapshot)
+ }
+ if !pr.RecentActive {
+ fmt.Fprintf(&buf, " inactive")
+ }
+ if n := pr.Inflights.Count(); n > 0 {
+ fmt.Fprintf(&buf, " inflight=%d", n)
+ if pr.Inflights.Full() {
+ fmt.Fprint(&buf, "[full]")
+ }
+ }
+ return buf.String()
+}
+
+// ProgressMap is a map of *Progress.
+type ProgressMap map[uint64]*Progress
+
+// String prints the ProgressMap in sorted key order, one Progress per line.
+func (m ProgressMap) String() string {
+ ids := make([]uint64, 0, len(m))
+ for k := range m {
+ ids = append(ids, k)
+ }
+ sort.Slice(ids, func(i, j int) bool {
+ return ids[i] < ids[j]
+ })
+ var buf strings.Builder
+ for _, id := range ids {
+ fmt.Fprintf(&buf, "%d: %s\n", id, m[id])
+ }
+ return buf.String()
+}
diff --git a/vendor/go.etcd.io/etcd/raft/tracker/state.go b/vendor/go.etcd.io/etcd/raft/tracker/state.go
new file mode 100644
index 0000000..285b4b8
--- /dev/null
+++ b/vendor/go.etcd.io/etcd/raft/tracker/state.go
@@ -0,0 +1,42 @@
+// Copyright 2019 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tracker
+
+// StateType is the state of a tracked follower.
+type StateType uint64
+
+const (
+ // StateProbe indicates a follower whose last index isn't known. Such a
+ // follower is "probed" (i.e. an append sent periodically) to narrow down
+ // its last index. In the ideal (and common) case, only one round of probing
+ // is necessary as the follower will react with a hint. Followers that are
+ // probed over extended periods of time are often offline.
+ StateProbe StateType = iota
+ // StateReplicate is the state steady in which a follower eagerly receives
+ // log entries to append to its log.
+ StateReplicate
+ // StateSnapshot indicates a follower that needs log entries not available
+ // from the leader's Raft log. Such a follower needs a full snapshot to
+ // return to StateReplicate.
+ StateSnapshot
+)
+
+var prstmap = [...]string{
+ "StateProbe",
+ "StateReplicate",
+ "StateSnapshot",
+}
+
+func (st StateType) String() string { return prstmap[uint64(st)] }
diff --git a/vendor/go.etcd.io/etcd/raft/tracker/tracker.go b/vendor/go.etcd.io/etcd/raft/tracker/tracker.go
new file mode 100644
index 0000000..a458114
--- /dev/null
+++ b/vendor/go.etcd.io/etcd/raft/tracker/tracker.go
@@ -0,0 +1,288 @@
+// Copyright 2019 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tracker
+
+import (
+ "fmt"
+ "sort"
+ "strings"
+
+ "go.etcd.io/etcd/raft/quorum"
+ pb "go.etcd.io/etcd/raft/raftpb"
+)
+
+// Config reflects the configuration tracked in a ProgressTracker.
+type Config struct {
+ Voters quorum.JointConfig
+ // AutoLeave is true if the configuration is joint and a transition to the
+ // incoming configuration should be carried out automatically by Raft when
+ // this is possible. If false, the configuration will be joint until the
+ // application initiates the transition manually.
+ AutoLeave bool
+ // Learners is a set of IDs corresponding to the learners active in the
+ // current configuration.
+ //
+ // Invariant: Learners and Voters does not intersect, i.e. if a peer is in
+ // either half of the joint config, it can't be a learner; if it is a
+ // learner it can't be in either half of the joint config. This invariant
+ // simplifies the implementation since it allows peers to have clarity about
+ // its current role without taking into account joint consensus.
+ Learners map[uint64]struct{}
+ // When we turn a voter into a learner during a joint consensus transition,
+ // we cannot add the learner directly when entering the joint state. This is
+ // because this would violate the invariant that the intersection of
+ // voters and learners is empty. For example, assume a Voter is removed and
+ // immediately re-added as a learner (or in other words, it is demoted):
+ //
+ // Initially, the configuration will be
+ //
+ // voters: {1 2 3}
+ // learners: {}
+ //
+ // and we want to demote 3. Entering the joint configuration, we naively get
+ //
+ // voters: {1 2} & {1 2 3}
+ // learners: {3}
+ //
+ // but this violates the invariant (3 is both voter and learner). Instead,
+ // we get
+ //
+ // voters: {1 2} & {1 2 3}
+ // learners: {}
+ // next_learners: {3}
+ //
+ // Where 3 is now still purely a voter, but we are remembering the intention
+ // to make it a learner upon transitioning into the final configuration:
+ //
+ // voters: {1 2}
+ // learners: {3}
+ // next_learners: {}
+ //
+ // Note that next_learners is not used while adding a learner that is not
+ // also a voter in the joint config. In this case, the learner is added
+ // right away when entering the joint configuration, so that it is caught up
+ // as soon as possible.
+ LearnersNext map[uint64]struct{}
+}
+
+func (c Config) String() string {
+ var buf strings.Builder
+ fmt.Fprintf(&buf, "voters=%s", c.Voters)
+ if c.Learners != nil {
+ fmt.Fprintf(&buf, " learners=%s", quorum.MajorityConfig(c.Learners).String())
+ }
+ if c.LearnersNext != nil {
+ fmt.Fprintf(&buf, " learners_next=%s", quorum.MajorityConfig(c.LearnersNext).String())
+ }
+ if c.AutoLeave {
+ fmt.Fprintf(&buf, " autoleave")
+ }
+ return buf.String()
+}
+
+// Clone returns a copy of the Config that shares no memory with the original.
+func (c *Config) Clone() Config {
+ clone := func(m map[uint64]struct{}) map[uint64]struct{} {
+ if m == nil {
+ return nil
+ }
+ mm := make(map[uint64]struct{}, len(m))
+ for k := range m {
+ mm[k] = struct{}{}
+ }
+ return mm
+ }
+ return Config{
+ Voters: quorum.JointConfig{clone(c.Voters[0]), clone(c.Voters[1])},
+ Learners: clone(c.Learners),
+ LearnersNext: clone(c.LearnersNext),
+ }
+}
+
+// ProgressTracker tracks the currently active configuration and the information
+// known about the nodes and learners in it. In particular, it tracks the match
+// index for each peer which in turn allows reasoning about the committed index.
+type ProgressTracker struct {
+ Config
+
+ Progress ProgressMap
+
+ Votes map[uint64]bool
+
+ MaxInflight int
+}
+
+// MakeProgressTracker initializes a ProgressTracker.
+func MakeProgressTracker(maxInflight int) ProgressTracker {
+ p := ProgressTracker{
+ MaxInflight: maxInflight,
+ Config: Config{
+ Voters: quorum.JointConfig{
+ quorum.MajorityConfig{},
+ nil, // only populated when used
+ },
+ Learners: nil, // only populated when used
+ LearnersNext: nil, // only populated when used
+ },
+ Votes: map[uint64]bool{},
+ Progress: map[uint64]*Progress{},
+ }
+ return p
+}
+
+// ConfState returns a ConfState representing the active configuration.
+func (p *ProgressTracker) ConfState() pb.ConfState {
+ return pb.ConfState{
+ Voters: p.Voters[0].Slice(),
+ VotersOutgoing: p.Voters[1].Slice(),
+ Learners: quorum.MajorityConfig(p.Learners).Slice(),
+ LearnersNext: quorum.MajorityConfig(p.LearnersNext).Slice(),
+ AutoLeave: p.AutoLeave,
+ }
+}
+
+// IsSingleton returns true if (and only if) there is only one voting member
+// (i.e. the leader) in the current configuration.
+func (p *ProgressTracker) IsSingleton() bool {
+ return len(p.Voters[0]) == 1 && len(p.Voters[1]) == 0
+}
+
+type matchAckIndexer map[uint64]*Progress
+
+var _ quorum.AckedIndexer = matchAckIndexer(nil)
+
+// AckedIndex implements IndexLookuper.
+func (l matchAckIndexer) AckedIndex(id uint64) (quorum.Index, bool) {
+ pr, ok := l[id]
+ if !ok {
+ return 0, false
+ }
+ return quorum.Index(pr.Match), true
+}
+
+// Committed returns the largest log index known to be committed based on what
+// the voting members of the group have acknowledged.
+func (p *ProgressTracker) Committed() uint64 {
+ return uint64(p.Voters.CommittedIndex(matchAckIndexer(p.Progress)))
+}
+
+func insertionSort(sl []uint64) {
+ a, b := 0, len(sl)
+ for i := a + 1; i < b; i++ {
+ for j := i; j > a && sl[j] < sl[j-1]; j-- {
+ sl[j], sl[j-1] = sl[j-1], sl[j]
+ }
+ }
+}
+
+// Visit invokes the supplied closure for all tracked progresses in stable order.
+func (p *ProgressTracker) Visit(f func(id uint64, pr *Progress)) {
+ n := len(p.Progress)
+ // We need to sort the IDs and don't want to allocate since this is hot code.
+ // The optimization here mirrors that in `(MajorityConfig).CommittedIndex`,
+ // see there for details.
+ var sl [7]uint64
+ ids := sl[:]
+ if len(sl) >= n {
+ ids = sl[:n]
+ } else {
+ ids = make([]uint64, n)
+ }
+ for id := range p.Progress {
+ n--
+ ids[n] = id
+ }
+ insertionSort(ids)
+ for _, id := range ids {
+ f(id, p.Progress[id])
+ }
+}
+
+// QuorumActive returns true if the quorum is active from the view of the local
+// raft state machine. Otherwise, it returns false.
+func (p *ProgressTracker) QuorumActive() bool {
+ votes := map[uint64]bool{}
+ p.Visit(func(id uint64, pr *Progress) {
+ if pr.IsLearner {
+ return
+ }
+ votes[id] = pr.RecentActive
+ })
+
+ return p.Voters.VoteResult(votes) == quorum.VoteWon
+}
+
+// VoterNodes returns a sorted slice of voters.
+func (p *ProgressTracker) VoterNodes() []uint64 {
+ m := p.Voters.IDs()
+ nodes := make([]uint64, 0, len(m))
+ for id := range m {
+ nodes = append(nodes, id)
+ }
+ sort.Slice(nodes, func(i, j int) bool { return nodes[i] < nodes[j] })
+ return nodes
+}
+
+// LearnerNodes returns a sorted slice of learners.
+func (p *ProgressTracker) LearnerNodes() []uint64 {
+ if len(p.Learners) == 0 {
+ return nil
+ }
+ nodes := make([]uint64, 0, len(p.Learners))
+ for id := range p.Learners {
+ nodes = append(nodes, id)
+ }
+ sort.Slice(nodes, func(i, j int) bool { return nodes[i] < nodes[j] })
+ return nodes
+}
+
+// ResetVotes prepares for a new round of vote counting via recordVote.
+func (p *ProgressTracker) ResetVotes() {
+ p.Votes = map[uint64]bool{}
+}
+
+// RecordVote records that the node with the given id voted for this Raft
+// instance if v == true (and declined it otherwise).
+func (p *ProgressTracker) RecordVote(id uint64, v bool) {
+ _, ok := p.Votes[id]
+ if !ok {
+ p.Votes[id] = v
+ }
+}
+
+// TallyVotes returns the number of granted and rejected Votes, and whether the
+// election outcome is known.
+func (p *ProgressTracker) TallyVotes() (granted int, rejected int, _ quorum.VoteResult) {
+ // Make sure to populate granted/rejected correctly even if the Votes slice
+ // contains members no longer part of the configuration. This doesn't really
+ // matter in the way the numbers are used (they're informational), but might
+ // as well get it right.
+ for id, pr := range p.Progress {
+ if pr.IsLearner {
+ continue
+ }
+ v, voted := p.Votes[id]
+ if !voted {
+ continue
+ }
+ if v {
+ granted++
+ } else {
+ rejected++
+ }
+ }
+ result := p.Voters.VoteResult(p.Votes)
+ return granted, rejected, result
+}
diff --git a/vendor/go.etcd.io/etcd/raft/util.go b/vendor/go.etcd.io/etcd/raft/util.go
index c145d26..785cf73 100644
--- a/vendor/go.etcd.io/etcd/raft/util.go
+++ b/vendor/go.etcd.io/etcd/raft/util.go
@@ -17,6 +17,7 @@
import (
"bytes"
"fmt"
+ "strings"
pb "go.etcd.io/etcd/raft/raftpb"
)
@@ -25,13 +26,6 @@
return []byte(fmt.Sprintf("%q", st.String())), nil
}
-// uint64Slice implements sort interface
-type uint64Slice []uint64
-
-func (p uint64Slice) Len() int { return len(p) }
-func (p uint64Slice) Less(i, j int) bool { return p[i] < p[j] }
-func (p uint64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
-
func min(a, b uint64) uint64 {
if a > b {
return b
@@ -67,6 +61,69 @@
}
}
+func DescribeHardState(hs pb.HardState) string {
+ var buf strings.Builder
+ fmt.Fprintf(&buf, "Term:%d", hs.Term)
+ if hs.Vote != 0 {
+ fmt.Fprintf(&buf, " Vote:%d", hs.Vote)
+ }
+ fmt.Fprintf(&buf, " Commit:%d", hs.Commit)
+ return buf.String()
+}
+
+func DescribeSoftState(ss SoftState) string {
+ return fmt.Sprintf("Lead:%d State:%s", ss.Lead, ss.RaftState)
+}
+
+func DescribeConfState(state pb.ConfState) string {
+ return fmt.Sprintf(
+ "Voters:%v VotersOutgoing:%v Learners:%v LearnersNext:%v AutoLeave:%v",
+ state.Voters, state.VotersOutgoing, state.Learners, state.LearnersNext, state.AutoLeave,
+ )
+}
+
+func DescribeSnapshot(snap pb.Snapshot) string {
+ m := snap.Metadata
+ return fmt.Sprintf("Index:%d Term:%d ConfState:%s", m.Index, m.Term, DescribeConfState(m.ConfState))
+}
+
+func DescribeReady(rd Ready, f EntryFormatter) string {
+ var buf strings.Builder
+ if rd.SoftState != nil {
+ fmt.Fprint(&buf, DescribeSoftState(*rd.SoftState))
+ buf.WriteByte('\n')
+ }
+ if !IsEmptyHardState(rd.HardState) {
+ fmt.Fprintf(&buf, "HardState %s", DescribeHardState(rd.HardState))
+ buf.WriteByte('\n')
+ }
+ if len(rd.ReadStates) > 0 {
+ fmt.Fprintf(&buf, "ReadStates %v\n", rd.ReadStates)
+ }
+ if len(rd.Entries) > 0 {
+ buf.WriteString("Entries:\n")
+ fmt.Fprint(&buf, DescribeEntries(rd.Entries, f))
+ }
+ if !IsEmptySnap(rd.Snapshot) {
+ fmt.Fprintf(&buf, "Snapshot %s\n", DescribeSnapshot(rd.Snapshot))
+ }
+ if len(rd.CommittedEntries) > 0 {
+ buf.WriteString("CommittedEntries:\n")
+ fmt.Fprint(&buf, DescribeEntries(rd.CommittedEntries, f))
+ }
+ if len(rd.Messages) > 0 {
+ buf.WriteString("Messages:\n")
+ for _, msg := range rd.Messages {
+ fmt.Fprint(&buf, DescribeMessage(msg, f))
+ buf.WriteByte('\n')
+ }
+ }
+ if buf.Len() > 0 {
+ return fmt.Sprintf("Ready MustSync=%t:\n%s", rd.MustSync, buf.String())
+ }
+ return "<empty Ready>"
+}
+
// EntryFormatter can be implemented by the application to provide human-readable formatting
// of entry data. Nil is a valid EntryFormatter and will use a default format.
type EntryFormatter func([]byte) string
@@ -93,7 +150,7 @@
fmt.Fprintf(&buf, "]")
}
if !IsEmptySnap(m.Snapshot) {
- fmt.Fprintf(&buf, " Snapshot:%v", m.Snapshot)
+ fmt.Fprintf(&buf, " Snapshot: %s", DescribeSnapshot(m.Snapshot))
}
return buf.String()
}
@@ -107,13 +164,39 @@
// DescribeEntry returns a concise human-readable description of an
// Entry for debugging.
func DescribeEntry(e pb.Entry, f EntryFormatter) string {
- var formatted string
- if e.Type == pb.EntryNormal && f != nil {
- formatted = f(e.Data)
- } else {
- formatted = fmt.Sprintf("%q", e.Data)
+ if f == nil {
+ f = func(data []byte) string { return fmt.Sprintf("%q", data) }
}
- return fmt.Sprintf("%d/%d %s %s", e.Term, e.Index, e.Type, formatted)
+
+ formatConfChange := func(cc pb.ConfChangeI) string {
+ // TODO(tbg): give the EntryFormatter a type argument so that it gets
+ // a chance to expose the Context.
+ return pb.ConfChangesToString(cc.AsV2().Changes)
+ }
+
+ var formatted string
+ switch e.Type {
+ case pb.EntryNormal:
+ formatted = f(e.Data)
+ case pb.EntryConfChange:
+ var cc pb.ConfChange
+ if err := cc.Unmarshal(e.Data); err != nil {
+ formatted = err.Error()
+ } else {
+ formatted = formatConfChange(cc)
+ }
+ case pb.EntryConfChangeV2:
+ var cc pb.ConfChangeV2
+ if err := cc.Unmarshal(e.Data); err != nil {
+ formatted = err.Error()
+ } else {
+ formatted = formatConfChange(cc)
+ }
+ }
+ if formatted != "" {
+ formatted = " " + formatted
+ }
+ return fmt.Sprintf("%d/%d %s%s", e.Term, e.Index, e.Type, formatted)
}
// DescribeEntries calls DescribeEntry for each Entry, adding a newline to
@@ -140,3 +223,11 @@
}
return ents[:limit]
}
+
+func assertConfStatesEquivalent(l Logger, cs1, cs2 pb.ConfState) {
+ err := cs1.Equivalent(cs2)
+ if err == nil {
+ return
+ }
+ l.Panic(err)
+}