diff --git a/LICENSE b/LICENSE index 4e4eb14ac4..9a25639758 100644 --- a/LICENSE +++ b/LICENSE @@ -1,5 +1,6 @@ Skipper is in general licensed under the following Apache Version 2.0 license with the exception -of the pathmux subdirectory which is licensed under MIT license (see notice file below). +of the pathmux subdirectory which is licensed under MIT license and +h2c subdirectory which is licensed under BSD-style license (see notice files below). Copyright 2015 Zalando SE @@ -41,3 +42,34 @@ MODIFICATIONS TO pathmux/tree.go and pathmux/tree_test.go: it can be used to look up arbitrary objects in a Patricia tree. 21.04.2016 - Enabled backtracking in the tree lookup. + + +Notice file for h2c/h2c.go + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/config/config.go b/config/config.go index 563bb89104..2f657ca948 100644 --- a/config/config.go +++ b/config/config.go @@ -218,6 +218,7 @@ type Config struct { ExpectContinueTimeoutBackend time.Duration `yaml:"expect-continue-timeout-backend"` MaxIdleConnsBackend int `yaml:"max-idle-connection-backend"` DisableHTTPKeepalives bool `yaml:"disable-http-keepalives"` + EnableH2CPriorKnowledge bool `yaml:"enable-h2c-prior-knowledge"` // swarm: EnableSwarm bool `yaml:"enable-swarm"` @@ -459,6 +460,7 @@ func NewConfig() *Config { flag.DurationVar(&cfg.ExpectContinueTimeoutBackend, "expect-continue-timeout-backend", 30*time.Second, "sets the HTTP expect continue timeout for backend connections") flag.IntVar(&cfg.MaxIdleConnsBackend, "max-idle-connection-backend", 0, "sets the maximum idle connections for all backend connections") flag.BoolVar(&cfg.DisableHTTPKeepalives, "disable-http-keepalives", false, "forces backend to always create a new connection") + flag.BoolVar(&cfg.EnableH2CPriorKnowledge, "enable-h2c-prior-knowledge", false, "enables HTTP/2 connections over cleartext TCP with Prior Knowledge") // Swarm: flag.BoolVar(&cfg.EnableSwarm, "enable-swarm", false, "enable swarm communication between nodes in a skipper fleet") @@ -744,6 +746,7 @@ func (c *Config) ToOptions() skipper.Options { ExpectContinueTimeoutBackend: c.ExpectContinueTimeoutBackend, MaxIdleConnsBackend: c.MaxIdleConnsBackend, DisableHTTPKeepalives: c.DisableHTTPKeepalives, + EnableH2CPriorKnowledge: c.EnableH2CPriorKnowledge, // swarm: EnableSwarm: c.EnableSwarm, diff --git a/go.mod b/go.mod index 1badcbe7d8..cbb9171ca3 100644 --- a/go.mod +++ b/go.mod @@ -49,10 +49,11 @@ require ( github.com/yuin/gopher-lua v0.0.0-20200603152657-dc2b0ca8b37e go.uber.org/atomic v1.4.0 // indirect golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 - golang.org/x/net v0.0.0-20210415231046-e915ea6b2b7d + golang.org/x/net v0.0.0-20210924151903-3ad01bbaa167 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/sys v0.0.0-20210917161153-d61c044b1678 // indirect + golang.org/x/text v0.3.7 // indirect golang.org/x/tools v0.1.0 // indirect google.golang.org/grpc v1.22.0 // indirect gopkg.in/alecthomas/kingpin.v2 v2.2.6 diff --git a/go.sum b/go.sum index 656eeb1755..3dc443c5bc 100644 --- a/go.sum +++ b/go.sum @@ -387,6 +387,8 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210415231046-e915ea6b2b7d h1:BgJvlyh+UqCUaPlscHJ+PN8GcpfrFdr7NHjd1JL0+Gs= golang.org/x/net v0.0.0-20210415231046-e915ea6b2b7d/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= +golang.org/x/net v0.0.0-20210924151903-3ad01bbaa167 h1:eDd+TJqbgfXruGQ5sJRU7tEtp/58OAx4+Ayjxg4SM+4= +golang.org/x/net v0.0.0-20210924151903-3ad01bbaa167/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= @@ -432,6 +434,7 @@ golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210415045647-66c3f260301c h1:6L+uOeS3OQt/f4eFHXZcTxeZrGCuz+CLElgEBjbcTA4= golang.org/x/sys v0.0.0-20210415045647-66c3f260301c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210917161153-d61c044b1678 h1:J27LZFQBFoihqXoegpscI10HpjZ7B5WQLLKL2FZXQKw= golang.org/x/sys v0.0.0-20210917161153-d61c044b1678/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -442,6 +445,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/h2c/h2c.go b/h2c/h2c.go new file mode 100644 index 0000000000..3dfd226332 --- /dev/null +++ b/h2c/h2c.go @@ -0,0 +1,156 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package h2c implements the unencrypted "h2c" form of HTTP/2. +// +// The h2c protocol is the non-TLS version of HTTP/2 +// +// The implementation is based on the golang.org/x/net/http2/h2c +package h2c + +import ( + "context" + "errors" + "io" + "net" + "net/http" + "strings" + "sync/atomic" + "time" + + log "github.com/sirupsen/logrus" + "golang.org/x/net/http2" +) + +type H2CHandler interface { + // Shutdown gracefully shuts down the handler by waiting + // indefinitely for h2c connections to close. + // If the provided context expires before the shutdown is complete, + // Shutdown returns the context's error. + // + // Shutdown should be called after http.Server Shutdown returns. + Shutdown(context.Context) error +} + +type h2cHandler struct { + handler http.Handler + s1 *http.Server + s2 *http2.Server + conns int64 +} + +// Enable creates an http2.Server s2, wraps http.Server s1 original handler +// with a handler intercepting any h2c traffic and registers s2 +// startGracefulShutdown on s1 Shutdown. It returns H2CHandler to be called +// after s1.Shutdown returns. +// +// If a request is an h2c connection, it is hijacked and redirected to the +// s2.ServeConn along with the original s1 handler. Otherwise the handler just +// forwards requests to the original s1 handler. This works because h2c is +// designed to be parsable as a valid HTTP/1, but ignored by any HTTP server +// that does not handle h2c. Therefore we leverage the HTTP/1 compatible parts +// of the Go http library to parse and recognize h2c requests. +// +// There are two ways to begin an h2c connection (RFC 7540 Section 3.2 and 3.4): +// (1) Starting with Prior Knowledge - this works by starting an h2c connection +// with a string of bytes that is valid HTTP/1, but unlikely to occur in +// practice and (2) Upgrading from HTTP/1 to h2c. +// +// This implementation workarounds several issues of the golang.org/x/net/http2/h2c: +// * drops support for upgrading from HTTP/1 to h2c, see https://github.com/golang/go/issues/38064 +// * implements graceful shutdown, see https://github.com/golang/go/issues/26682 +// * remove closing of the hijacked connection because s2.ServeConn closes it +// * removes buffered connection write +func Enable(s1 *http.Server) H2CHandler { + s2 := &http2.Server{} + h := &h2cHandler{handler: s1.Handler, s1: s1, s2: s2} + + // register s2 startGracefulShutdown on s1 Shutdown + http2.ConfigureServer(s1, s2) + s1.Handler = h + + return h +} + +func (s *h2cHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + // Handle h2c with prior knowledge (RFC 7540 Section 3.4) + if r.Method == "PRI" && len(r.Header) == 0 && r.URL.Path == "*" && r.Proto == "HTTP/2.0" { + conn, err := initH2CWithPriorKnowledge(w) + if err != nil { + return + } + n := atomic.AddInt64(&s.conns, 1) + log.Debugf("h2c start: %d connections", n) + + s.s2.ServeConn(conn, &http2.ServeConnOpts{Handler: s.handler, BaseConfig: s.s1}) + + n = atomic.AddInt64(&s.conns, -1) + log.Debugf("h2c done: %d connections", n) + } else { + s.handler.ServeHTTP(w, r) + } + return +} + +// initH2CWithPriorKnowledge implements creating an h2c connection with prior +// knowledge (Section 3.4) and creates a net.Conn suitable for http2.ServeConn. +// All we have to do is look for the client preface that is supposed to be a part +// of the body, and reforward the client preface on the net.Conn this function +// creates. +func initH2CWithPriorKnowledge(w http.ResponseWriter) (net.Conn, error) { + hijacker, ok := w.(http.Hijacker) + if !ok { + return nil, errors.New("hijack is not supported") + } + conn, rw, err := hijacker.Hijack() + if err != nil { + return nil, err + } + r := rw.Reader + + const expectedBody = "SM\r\n\r\n" + + buf := make([]byte, len(expectedBody)) + n, err := io.ReadFull(r, buf) + if err != nil { + return nil, err + } + + if string(buf[:n]) == expectedBody { + return &h2cConn{ + Conn: conn, + Reader: io.MultiReader(strings.NewReader(http2.ClientPreface), r), + }, nil + } + + conn.Close() + return nil, errors.New("invalid client preface") +} + +func (s *h2cHandler) Shutdown(ctx context.Context) error { + timer := time.NewTicker(500 * time.Millisecond) + defer timer.Stop() + for { + n := atomic.LoadInt64(&s.conns) + log.Debugf("h2c close: %d connections", n) + + if n == 0 { + return nil + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + } + } +} + +type h2cConn struct { + net.Conn + io.Reader +} + +func (c *h2cConn) Read(p []byte) (int, error) { + return c.Reader.Read(p) +} diff --git a/skipper.go b/skipper.go index e032898dd1..e608452cfa 100644 --- a/skipper.go +++ b/skipper.go @@ -33,6 +33,7 @@ import ( "github.com/zalando/skipper/filters/fadein" logfilter "github.com/zalando/skipper/filters/log" ratelimitfilters "github.com/zalando/skipper/filters/ratelimit" + "github.com/zalando/skipper/h2c" "github.com/zalando/skipper/innkeeper" "github.com/zalando/skipper/loadbalancer" "github.com/zalando/skipper/logging" @@ -337,6 +338,9 @@ type Options struct { // a backend to always create a new connection. DisableHTTPKeepalives bool + // Enables HTTP/2 connections over cleartext TCP with Prior Knowledge + EnableH2CPriorKnowledge bool + // Flag indicating to ignore trailing slashes in paths during route // lookup. IgnoreTrailingSlash bool @@ -953,6 +957,10 @@ func (o *Options) tlsConfig() (*tls.Config, error) { return nil, nil } + if o.EnableH2CPriorKnowledge { + return nil, fmt.Errorf("TLS implies no HTTP/2 connections over cleartext TCP") + } + crts := strings.Split(o.CertPathTLS, ",") keys := strings.Split(o.KeyPathTLS, ",") @@ -1065,6 +1073,7 @@ func listenAndServeQuit( sigs = make(chan os.Signal, 1) } + var h2cHandler h2c.H2CHandler go func() { signal.Notify(sigs, syscall.SIGTERM) @@ -1077,6 +1086,11 @@ func listenAndServeQuit( if err := srv.Shutdown(context.Background()); err != nil { log.Errorf("Failed to graceful shutdown: %v", err) } + if h2cHandler != nil { + if err := h2cHandler.Shutdown(context.Background()); err != nil { + log.Errorf("Failed to graceful shutdown h2c: %v", err) + } + } close(idleConnsCH) }() @@ -1090,6 +1104,11 @@ func listenAndServeQuit( } else { log.Infof("TLS settings not found, defaulting to HTTP") + if o.EnableH2CPriorKnowledge { + log.Infof("Enabling HTTP/2 connections over cleartext TCP") + h2cHandler = h2c.Enable(srv) + } + l, err := listen(o, mtr) if err != nil { return err diff --git a/skipper_test.go b/skipper_test.go index 326d2ccfeb..caff5b72e5 100644 --- a/skipper_test.go +++ b/skipper_test.go @@ -5,12 +5,14 @@ import ( "io" "net" "net/http" + neturl "net/url" "os" "syscall" "testing" "time" log "github.com/sirupsen/logrus" + "golang.org/x/net/http2" "github.com/zalando/skipper/dataclients/routestring" "github.com/zalando/skipper/filters" @@ -27,6 +29,45 @@ const ( listenTimeout = 9 * listenDelay ) +type protocol int + +const ( + HTTP protocol = iota + HTTPS + H2C +) + +func (p protocol) scheme() string { + return [...]string{"http", "https", "http"}[p] +} + +func (p protocol) newClient() *http.Client { + switch p { + case HTTP: + return http.DefaultClient + case HTTPS: + return &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + }, + } + case H2C: + return &http.Client{ + Transport: &http2.Transport{ + // allow http scheme + AllowHTTP: true, + // ignore tls.Config and dial unencrypted TCP + DialTLS: func(network, addr string, cfg *tls.Config) (net.Conn, error) { + return net.Dial(network, addr) + }, + }, + } + } + return nil +} + func testListener() bool { for _, a := range os.Args { if a == "listener" { @@ -54,12 +95,16 @@ func waitConn(req func() (*http.Response, error)) (*http.Response, error) { } } -func waitConnGet(url string) (*http.Response, error) { +func waitConnGet(proto protocol, address string) (*http.Response, error) { + u, err := neturl.Parse("scheme://" + address) + if err != nil { + return nil, err + } + u.Scheme = proto.scheme() + client := proto.newClient() + return waitConn(func() (*http.Response, error) { - return (&http.Client{ - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true}}}).Get(url) + return client.Get(u.String()) }) } @@ -113,7 +158,7 @@ func TestOptionsTLSConfig(t *testing.T) { require.Equal(t, []tls.Certificate{cert, cert2}, c.Certificates) } -func TestOptionsTLSConfigInvalidPaths(t *testing.T) { +func TestOptionsTLSConfigInvalid(t *testing.T) { for _, tt := range []struct { name string options *Options @@ -125,6 +170,7 @@ func TestOptionsTLSConfigInvalidPaths(t *testing.T) { {"cert key mismatch", &Options{CertPathTLS: "fixtures/test.crt", KeyPathTLS: "fixtures/test2.key"}}, {"multiple cert key count mismatch", &Options{CertPathTLS: "fixtures/test.crt,fixtures/test2.crt", KeyPathTLS: "fixtures/test.key"}}, {"multiple cert key mismatch", &Options{CertPathTLS: "fixtures/test.crt,fixtures/test2.crt", KeyPathTLS: "fixtures/test2.key,fixtures/test.key"}}, + {"h2c conflicts with tls", &Options{EnableH2CPriorKnowledge: true, CertPathTLS: "fixtures/test.crt", KeyPathTLS: "fixtures/test.key"}}, } { t.Run(tt.name, func(t *testing.T) { _, err := tt.options.tlsConfig() @@ -161,7 +207,7 @@ func TestHTTPSServer(t *testing.T) { defer proxy.Close() go listenAndServe(proxy, &o) - r, err := waitConnGet("https://" + o.Address) + r, err := waitConnGet(HTTPS, o.Address) if r != nil { defer r.Body.Close() } @@ -199,7 +245,7 @@ func TestHTTPServer(t *testing.T) { proxy := proxy.New(rt, proxy.OptionsNone) defer proxy.Close() go listenAndServe(proxy, &o) - r, err := waitConnGet("http://" + o.Address) + r, err := waitConnGet(HTTP, o.Address) if r != nil { defer r.Body.Close() } @@ -215,27 +261,33 @@ func TestHTTPServer(t *testing.T) { } } -func TestHTTPServerShutdown(t *testing.T) { +func TestServerShutdownHTTP(t *testing.T) { o := &Options{} - testServerShutdown(t, o, "http") + testServerShutdown(t, o, HTTP) } -func TestHTTPSServerShutdown(t *testing.T) { +func TestServerShutdownHTTPS(t *testing.T) { o := &Options{ CertPathTLS: "fixtures/test.crt", KeyPathTLS: "fixtures/test.key", } - testServerShutdown(t, o, "https") + testServerShutdown(t, o, HTTPS) +} + +func TestServerShutdownH2C(t *testing.T) { + o := &Options{ + EnableH2CPriorKnowledge: true, + } + testServerShutdown(t, o, H2C) } -func testServerShutdown(t *testing.T, o *Options, scheme string) { +func testServerShutdown(t *testing.T, o *Options, proto protocol) { const shutdownDelay = 1 * time.Second address, err := findAddress() require.NoError(t, err) o.Address, o.WaitForHealthcheckInterval = address, shutdownDelay - testUrl := scheme + "://" + address // simulate a backend that got a request and should be handled correctly dc, err := routestring.New(`r0: * -> latency("3s") -> inlineContent("OK") -> status(200) -> `) @@ -262,7 +314,7 @@ func testServerShutdown(t *testing.T, o *Options, scheme string) { time.Sleep(shutdownDelay / 2) t.Logf("ongoing request passing in before shutdown") - r, err := waitConnGet(testUrl) + r, err := waitConnGet(proto, address) require.NoError(t, err) require.Equal(t, 200, r.StatusCode) @@ -275,7 +327,7 @@ func testServerShutdown(t *testing.T, o *Options, scheme string) { time.Sleep(shutdownDelay / 2) t.Logf("request after shutdown should fail") - _, err = waitConnGet(testUrl) + _, err = waitConnGet(proto, address) require.Error(t, err) }