| // Copyright 2015 The etcd Authors |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| package embed |
| |
| import ( |
| "context" |
| "fmt" |
| "io/ioutil" |
| defaultLog "log" |
| "net" |
| "net/http" |
| "strings" |
| |
| "go.etcd.io/etcd/clientv3/credentials" |
| "go.etcd.io/etcd/etcdserver" |
| "go.etcd.io/etcd/etcdserver/api/v3client" |
| "go.etcd.io/etcd/etcdserver/api/v3election" |
| "go.etcd.io/etcd/etcdserver/api/v3election/v3electionpb" |
| v3electiongw "go.etcd.io/etcd/etcdserver/api/v3election/v3electionpb/gw" |
| "go.etcd.io/etcd/etcdserver/api/v3lock" |
| "go.etcd.io/etcd/etcdserver/api/v3lock/v3lockpb" |
| v3lockgw "go.etcd.io/etcd/etcdserver/api/v3lock/v3lockpb/gw" |
| "go.etcd.io/etcd/etcdserver/api/v3rpc" |
| etcdservergw "go.etcd.io/etcd/etcdserver/etcdserverpb/gw" |
| "go.etcd.io/etcd/pkg/debugutil" |
| "go.etcd.io/etcd/pkg/httputil" |
| "go.etcd.io/etcd/pkg/transport" |
| |
| gw "github.com/grpc-ecosystem/grpc-gateway/runtime" |
| "github.com/soheilhy/cmux" |
| "github.com/tmc/grpc-websocket-proxy/wsproxy" |
| "go.uber.org/zap" |
| "golang.org/x/net/trace" |
| "google.golang.org/grpc" |
| ) |
| |
| type serveCtx struct { |
| lg *zap.Logger |
| l net.Listener |
| addr string |
| network string |
| secure bool |
| insecure bool |
| |
| ctx context.Context |
| cancel context.CancelFunc |
| |
| userHandlers map[string]http.Handler |
| serviceRegister func(*grpc.Server) |
| serversC chan *servers |
| } |
| |
| type servers struct { |
| secure bool |
| grpc *grpc.Server |
| http *http.Server |
| } |
| |
| func newServeCtx(lg *zap.Logger) *serveCtx { |
| ctx, cancel := context.WithCancel(context.Background()) |
| return &serveCtx{ |
| lg: lg, |
| ctx: ctx, |
| cancel: cancel, |
| userHandlers: make(map[string]http.Handler), |
| serversC: make(chan *servers, 2), // in case sctx.insecure,sctx.secure true |
| } |
| } |
| |
| // serve accepts incoming connections on the listener l, |
| // creating a new service goroutine for each. The service goroutines |
| // read requests and then call handler to reply to them. |
| func (sctx *serveCtx) serve( |
| s *etcdserver.EtcdServer, |
| tlsinfo *transport.TLSInfo, |
| handler http.Handler, |
| errHandler func(error), |
| gopts ...grpc.ServerOption) (err error) { |
| logger := defaultLog.New(ioutil.Discard, "etcdhttp", 0) |
| <-s.ReadyNotify() |
| |
| if sctx.lg == nil { |
| plog.Info("ready to serve client requests") |
| } |
| |
| m := cmux.New(sctx.l) |
| v3c := v3client.New(s) |
| servElection := v3election.NewElectionServer(v3c) |
| servLock := v3lock.NewLockServer(v3c) |
| |
| var gs *grpc.Server |
| defer func() { |
| if err != nil && gs != nil { |
| gs.Stop() |
| } |
| }() |
| |
| if sctx.insecure { |
| gs = v3rpc.Server(s, nil, gopts...) |
| v3electionpb.RegisterElectionServer(gs, servElection) |
| v3lockpb.RegisterLockServer(gs, servLock) |
| if sctx.serviceRegister != nil { |
| sctx.serviceRegister(gs) |
| } |
| grpcl := m.Match(cmux.HTTP2()) |
| go func() { errHandler(gs.Serve(grpcl)) }() |
| |
| var gwmux *gw.ServeMux |
| if s.Cfg.EnableGRPCGateway { |
| gwmux, err = sctx.registerGateway([]grpc.DialOption{grpc.WithInsecure()}) |
| if err != nil { |
| return err |
| } |
| } |
| |
| httpmux := sctx.createMux(gwmux, handler) |
| |
| srvhttp := &http.Server{ |
| Handler: createAccessController(sctx.lg, s, httpmux), |
| ErrorLog: logger, // do not log user error |
| } |
| httpl := m.Match(cmux.HTTP1()) |
| go func() { errHandler(srvhttp.Serve(httpl)) }() |
| |
| sctx.serversC <- &servers{grpc: gs, http: srvhttp} |
| if sctx.lg != nil { |
| sctx.lg.Info( |
| "serving client traffic insecurely; this is strongly discouraged!", |
| zap.String("address", sctx.l.Addr().String()), |
| ) |
| } else { |
| plog.Noticef("serving insecure client requests on %s, this is strongly discouraged!", sctx.l.Addr().String()) |
| } |
| } |
| |
| if sctx.secure { |
| tlscfg, tlsErr := tlsinfo.ServerConfig() |
| if tlsErr != nil { |
| return tlsErr |
| } |
| gs = v3rpc.Server(s, tlscfg, gopts...) |
| v3electionpb.RegisterElectionServer(gs, servElection) |
| v3lockpb.RegisterLockServer(gs, servLock) |
| if sctx.serviceRegister != nil { |
| sctx.serviceRegister(gs) |
| } |
| handler = grpcHandlerFunc(gs, handler) |
| |
| var gwmux *gw.ServeMux |
| if s.Cfg.EnableGRPCGateway { |
| dtls := tlscfg.Clone() |
| // trust local server |
| dtls.InsecureSkipVerify = true |
| bundle := credentials.NewBundle(credentials.Config{TLSConfig: dtls}) |
| opts := []grpc.DialOption{grpc.WithTransportCredentials(bundle.TransportCredentials())} |
| gwmux, err = sctx.registerGateway(opts) |
| if err != nil { |
| return err |
| } |
| } |
| |
| var tlsl net.Listener |
| tlsl, err = transport.NewTLSListener(m.Match(cmux.Any()), tlsinfo) |
| if err != nil { |
| return err |
| } |
| // TODO: add debug flag; enable logging when debug flag is set |
| httpmux := sctx.createMux(gwmux, handler) |
| |
| srv := &http.Server{ |
| Handler: createAccessController(sctx.lg, s, httpmux), |
| TLSConfig: tlscfg, |
| ErrorLog: logger, // do not log user error |
| } |
| go func() { errHandler(srv.Serve(tlsl)) }() |
| |
| sctx.serversC <- &servers{secure: true, grpc: gs, http: srv} |
| if sctx.lg != nil { |
| sctx.lg.Info( |
| "serving client traffic securely", |
| zap.String("address", sctx.l.Addr().String()), |
| ) |
| } else { |
| plog.Infof("serving client requests on %s", sctx.l.Addr().String()) |
| } |
| } |
| |
| close(sctx.serversC) |
| return m.Serve() |
| } |
| |
| // grpcHandlerFunc returns an http.Handler that delegates to grpcServer on incoming gRPC |
| // connections or otherHandler otherwise. Given in gRPC docs. |
| func grpcHandlerFunc(grpcServer *grpc.Server, otherHandler http.Handler) http.Handler { |
| if otherHandler == nil { |
| return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| grpcServer.ServeHTTP(w, r) |
| }) |
| } |
| return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| if r.ProtoMajor == 2 && strings.Contains(r.Header.Get("Content-Type"), "application/grpc") { |
| grpcServer.ServeHTTP(w, r) |
| } else { |
| otherHandler.ServeHTTP(w, r) |
| } |
| }) |
| } |
| |
| type registerHandlerFunc func(context.Context, *gw.ServeMux, *grpc.ClientConn) error |
| |
| func (sctx *serveCtx) registerGateway(opts []grpc.DialOption) (*gw.ServeMux, error) { |
| ctx := sctx.ctx |
| |
| addr := sctx.addr |
| if network := sctx.network; network == "unix" { |
| // explicitly define unix network for gRPC socket support |
| addr = fmt.Sprintf("%s://%s", network, addr) |
| } |
| |
| conn, err := grpc.DialContext(ctx, addr, opts...) |
| if err != nil { |
| return nil, err |
| } |
| gwmux := gw.NewServeMux() |
| |
| handlers := []registerHandlerFunc{ |
| etcdservergw.RegisterKVHandler, |
| etcdservergw.RegisterWatchHandler, |
| etcdservergw.RegisterLeaseHandler, |
| etcdservergw.RegisterClusterHandler, |
| etcdservergw.RegisterMaintenanceHandler, |
| etcdservergw.RegisterAuthHandler, |
| v3lockgw.RegisterLockHandler, |
| v3electiongw.RegisterElectionHandler, |
| } |
| for _, h := range handlers { |
| if err := h(ctx, gwmux, conn); err != nil { |
| return nil, err |
| } |
| } |
| go func() { |
| <-ctx.Done() |
| if cerr := conn.Close(); cerr != nil { |
| if sctx.lg != nil { |
| sctx.lg.Warn( |
| "failed to close connection", |
| zap.String("address", sctx.l.Addr().String()), |
| zap.Error(cerr), |
| ) |
| } else { |
| plog.Warningf("failed to close conn to %s: %v", sctx.l.Addr().String(), cerr) |
| } |
| } |
| }() |
| |
| return gwmux, nil |
| } |
| |
| func (sctx *serveCtx) createMux(gwmux *gw.ServeMux, handler http.Handler) *http.ServeMux { |
| httpmux := http.NewServeMux() |
| for path, h := range sctx.userHandlers { |
| httpmux.Handle(path, h) |
| } |
| |
| if gwmux != nil { |
| httpmux.Handle( |
| "/v3/", |
| wsproxy.WebsocketProxy( |
| gwmux, |
| wsproxy.WithRequestMutator( |
| // Default to the POST method for streams |
| func(_ *http.Request, outgoing *http.Request) *http.Request { |
| outgoing.Method = "POST" |
| return outgoing |
| }, |
| ), |
| ), |
| ) |
| } |
| if handler != nil { |
| httpmux.Handle("/", handler) |
| } |
| return httpmux |
| } |
| |
| // createAccessController wraps HTTP multiplexer: |
| // - mutate gRPC gateway request paths |
| // - check hostname whitelist |
| // client HTTP requests goes here first |
| func createAccessController(lg *zap.Logger, s *etcdserver.EtcdServer, mux *http.ServeMux) http.Handler { |
| return &accessController{lg: lg, s: s, mux: mux} |
| } |
| |
| type accessController struct { |
| lg *zap.Logger |
| s *etcdserver.EtcdServer |
| mux *http.ServeMux |
| } |
| |
| func (ac *accessController) ServeHTTP(rw http.ResponseWriter, req *http.Request) { |
| // redirect for backward compatibilities |
| if req != nil && req.URL != nil && strings.HasPrefix(req.URL.Path, "/v3beta/") { |
| req.URL.Path = strings.Replace(req.URL.Path, "/v3beta/", "/v3/", 1) |
| } |
| |
| if req.TLS == nil { // check origin if client connection is not secure |
| host := httputil.GetHostname(req) |
| if !ac.s.AccessController.IsHostWhitelisted(host) { |
| if ac.lg != nil { |
| ac.lg.Warn( |
| "rejecting HTTP request to prevent DNS rebinding attacks", |
| zap.String("host", host), |
| ) |
| } else { |
| plog.Warningf("rejecting HTTP request from %q to prevent DNS rebinding attacks", host) |
| } |
| // TODO: use Go's "http.StatusMisdirectedRequest" (421) |
| // https://github.com/golang/go/commit/4b8a7eafef039af1834ef9bfa879257c4a72b7b5 |
| http.Error(rw, errCVE20185702(host), 421) |
| return |
| } |
| } else if ac.s.Cfg.ClientCertAuthEnabled && ac.s.Cfg.EnableGRPCGateway && |
| ac.s.AuthStore().IsAuthEnabled() && strings.HasPrefix(req.URL.Path, "/v3/") { |
| for _, chains := range req.TLS.VerifiedChains { |
| if len(chains) < 1 { |
| continue |
| } |
| if len(chains[0].Subject.CommonName) != 0 { |
| http.Error(rw, "CommonName of client sending a request against gateway will be ignored and not used as expected", 400) |
| return |
| } |
| } |
| } |
| |
| // Write CORS header. |
| if ac.s.AccessController.OriginAllowed("*") { |
| addCORSHeader(rw, "*") |
| } else if origin := req.Header.Get("Origin"); ac.s.OriginAllowed(origin) { |
| addCORSHeader(rw, origin) |
| } |
| |
| if req.Method == "OPTIONS" { |
| rw.WriteHeader(http.StatusOK) |
| return |
| } |
| |
| ac.mux.ServeHTTP(rw, req) |
| } |
| |
| // addCORSHeader adds the correct cors headers given an origin |
| func addCORSHeader(w http.ResponseWriter, origin string) { |
| w.Header().Add("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, DELETE") |
| w.Header().Add("Access-Control-Allow-Origin", origin) |
| w.Header().Add("Access-Control-Allow-Headers", "accept, content-type, authorization") |
| } |
| |
| // https://github.com/transmission/transmission/pull/468 |
| func errCVE20185702(host string) string { |
| return fmt.Sprintf(` |
| etcd received your request, but the Host header was unrecognized. |
| |
| To fix this, choose one of the following options: |
| - Enable TLS, then any HTTPS request will be allowed. |
| - Add the hostname you want to use to the whitelist in settings. |
| - e.g. etcd --host-whitelist %q |
| |
| This requirement has been added to help prevent "DNS Rebinding" attacks (CVE-2018-5702). |
| `, host) |
| } |
| |
| // WrapCORS wraps existing handler with CORS. |
| // TODO: deprecate this after v2 proxy deprecate |
| func WrapCORS(cors map[string]struct{}, h http.Handler) http.Handler { |
| return &corsHandler{ |
| ac: &etcdserver.AccessController{CORS: cors}, |
| h: h, |
| } |
| } |
| |
| type corsHandler struct { |
| ac *etcdserver.AccessController |
| h http.Handler |
| } |
| |
| func (ch *corsHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { |
| if ch.ac.OriginAllowed("*") { |
| addCORSHeader(rw, "*") |
| } else if origin := req.Header.Get("Origin"); ch.ac.OriginAllowed(origin) { |
| addCORSHeader(rw, origin) |
| } |
| |
| if req.Method == "OPTIONS" { |
| rw.WriteHeader(http.StatusOK) |
| return |
| } |
| |
| ch.h.ServeHTTP(rw, req) |
| } |
| |
| func (sctx *serveCtx) registerUserHandler(s string, h http.Handler) { |
| if sctx.userHandlers[s] != nil { |
| if sctx.lg != nil { |
| sctx.lg.Warn("path is already registered by user handler", zap.String("path", s)) |
| } else { |
| plog.Warningf("path %s already registered by user handler", s) |
| } |
| return |
| } |
| sctx.userHandlers[s] = h |
| } |
| |
| func (sctx *serveCtx) registerPprof() { |
| for p, h := range debugutil.PProfHandlers() { |
| sctx.registerUserHandler(p, h) |
| } |
| } |
| |
| func (sctx *serveCtx) registerTrace() { |
| reqf := func(w http.ResponseWriter, r *http.Request) { trace.Render(w, r, true) } |
| sctx.registerUserHandler("/debug/requests", http.HandlerFunc(reqf)) |
| evf := func(w http.ResponseWriter, r *http.Request) { trace.RenderEvents(w, r, true) } |
| sctx.registerUserHandler("/debug/events", http.HandlerFunc(evf)) |
| } |