Skip to content

Commit

Permalink
Adds h2c support
Browse files Browse the repository at this point in the history
Enables HTTP/2 connections over cleartext TCP with Prior Knowledge (RFC 7540 3.4).

The implementation is based on the golang.org/x/net/http2/h2c and workarounds several issues:
* golang/go#38064
* golang/go#26682

See h2c package docs for details.

Signed-off-by: Alexander Yastrebov <[email protected]>
  • Loading branch information
AlexanderYastrebov committed Dec 22, 2021
1 parent 8444aeb commit f2b780b
Show file tree
Hide file tree
Showing 7 changed files with 286 additions and 27 deletions.
34 changes: 33 additions & 1 deletion LICENSE
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
Skipper is in general licensed under the following Apache Version 2.0 license with the exception
of the pathmux subdirectory which is licensed under MIT license (see notice file below).
of the pathmux subdirectory which is licensed under MIT license and
h2c subdirectory which is licensed under BSD-style license (see notice files below).

Copyright 2015 Zalando SE

Expand Down Expand Up @@ -41,3 +42,34 @@ MODIFICATIONS TO pathmux/tree.go and pathmux/tree_test.go:
it can be used to look up arbitrary objects in a Patricia tree.

21.04.2016 - Enabled backtracking in the tree lookup.


Notice file for h2c/h2c.go

Copyright (c) 2009 The Go Authors. All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:

* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
3 changes: 3 additions & 0 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,7 @@ type Config struct {
ExpectContinueTimeoutBackend time.Duration `yaml:"expect-continue-timeout-backend"`
MaxIdleConnsBackend int `yaml:"max-idle-connection-backend"`
DisableHTTPKeepalives bool `yaml:"disable-http-keepalives"`
EnableH2CPriorKnowledge bool `yaml:"enable-h2c-prior-knowledge"`

// swarm:
EnableSwarm bool `yaml:"enable-swarm"`
Expand Down Expand Up @@ -474,6 +475,7 @@ func NewConfig() *Config {
flag.DurationVar(&cfg.ExpectContinueTimeoutBackend, "expect-continue-timeout-backend", 30*time.Second, "sets the HTTP expect continue timeout for backend connections")
flag.IntVar(&cfg.MaxIdleConnsBackend, "max-idle-connection-backend", 0, "sets the maximum idle connections for all backend connections")
flag.BoolVar(&cfg.DisableHTTPKeepalives, "disable-http-keepalives", false, "forces backend to always create a new connection")
flag.BoolVar(&cfg.EnableH2CPriorKnowledge, "enable-h2c-prior-knowledge", false, "enables HTTP/2 connections over cleartext TCP with Prior Knowledge")

// Swarm:
flag.BoolVar(&cfg.EnableSwarm, "enable-swarm", false, "enable swarm communication between nodes in a skipper fleet")
Expand Down Expand Up @@ -805,6 +807,7 @@ func (c *Config) ToOptions() skipper.Options {
ExpectContinueTimeoutBackend: c.ExpectContinueTimeoutBackend,
MaxIdleConnsBackend: c.MaxIdleConnsBackend,
DisableHTTPKeepalives: c.DisableHTTPKeepalives,
EnableH2CPriorKnowledge: c.EnableH2CPriorKnowledge,

// swarm:
EnableSwarm: c.EnableSwarm,
Expand Down
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -47,10 +47,11 @@ require (
github.com/yuin/gopher-lua v0.0.0-20200603152657-dc2b0ca8b37e
go.uber.org/atomic v1.4.0 // indirect
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
golang.org/x/net v0.0.0-20210415231046-e915ea6b2b7d
golang.org/x/net v0.0.0-20211216030914-fe4d6282115f
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/sys v0.0.0-20210917161153-d61c044b1678 // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/tools v0.1.0 // indirect
google.golang.org/grpc v1.22.0 // indirect
gopkg.in/alecthomas/kingpin.v2 v2.2.6
Expand Down Expand Up @@ -97,7 +98,6 @@ require (
github.com/tidwall/pretty v1.2.0 // indirect
github.com/tklauser/numcpus v0.2.2 // indirect
go.opentelemetry.io/otel v0.13.0 // indirect
golang.org/x/text v0.3.6 // indirect
google.golang.org/appengine v1.5.0 // indirect
google.golang.org/genproto v0.0.0-20190530194941-fb225487d101 // indirect
google.golang.org/protobuf v1.23.0 // indirect
Expand Down
11 changes: 5 additions & 6 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -203,8 +203,6 @@ github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY=
github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg=
github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
Expand Down Expand Up @@ -378,8 +376,8 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R
golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210415231046-e915ea6b2b7d h1:BgJvlyh+UqCUaPlscHJ+PN8GcpfrFdr7NHjd1JL0+Gs=
golang.org/x/net v0.0.0-20210415231046-e915ea6b2b7d/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8=
golang.org/x/net v0.0.0-20211216030914-fe4d6282115f h1:hEYJvxw1lSnWIl8X9ofsYMklzaDs90JI2az5YMd4fPM=
golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
Expand Down Expand Up @@ -422,7 +420,7 @@ golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210217105451-b926d437f341/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210917161153-d61c044b1678 h1:J27LZFQBFoihqXoegpscI10HpjZ7B5WQLLKL2FZXQKw=
golang.org/x/sys v0.0.0-20210917161153-d61c044b1678/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
Expand All @@ -431,8 +429,9 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
Expand Down
158 changes: 158 additions & 0 deletions h2c/h2c.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package h2c implements the unencrypted "h2c" form of HTTP/2.
//
// The h2c protocol is the non-TLS version of HTTP/2
//
// The implementation is based on the golang.org/x/net/http2/h2c
package h2c

import (
"context"
"errors"
"io"
"net"
"net/http"
"strings"
"sync/atomic"
"time"

log "github.com/sirupsen/logrus"
"golang.org/x/net/http2"
)

type Handler interface {
// Shutdown gracefully shuts down the underlying HTTP/1 server and
// waits for h2c connections to close.
//
// Returns HTTP/1 server shutdown err or the context's error
// if the provided context expires before the shutdown is complete.
Shutdown(context.Context) error
}

type Options struct{}

type h2cHandler struct {
handler http.Handler
s1 *http.Server
s2 *http2.Server
conns int64
}

// Enable creates an http2.Server s2, wraps http.Server s1 original handler
// with a handler intercepting any h2c traffic and registers s2
// startGracefulShutdown on s1 Shutdown.
// It returns h2c handler that should be called to shutdown s1 and h2c connections.
//
// If a request is an h2c connection, it is hijacked and redirected to the
// s2.ServeConn along with the original s1 handler. Otherwise the handler just
// forwards requests to the original s1 handler. This works because h2c is
// designed to be parsable as a valid HTTP/1, but ignored by any HTTP server
// that does not handle h2c. Therefore we leverage the HTTP/1 compatible parts
// of the Go http library to parse and recognize h2c requests.
//
// There are two ways to begin an h2c connection (RFC 7540 Section 3.2 and 3.4):
// (1) Starting with Prior Knowledge - this works by starting an h2c connection
// with a string of bytes that is valid HTTP/1, but unlikely to occur in
// practice and (2) Upgrading from HTTP/1 to h2c.
//
// This implementation workarounds several issues of the golang.org/x/net/http2/h2c:
// * drops support for upgrading from HTTP/1 to h2c, see https://github.com/golang/go/issues/38064
// * implements graceful shutdown, see https://github.com/golang/go/issues/26682
// * removes closing of the hijacked connection because s2.ServeConn closes it
// * removes buffered connection write
func Enable(s1 *http.Server, reserved *Options) Handler {
s2 := &http2.Server{}
h := &h2cHandler{handler: s1.Handler, s1: s1, s2: s2}

// register s2 startGracefulShutdown on s1 Shutdown
http2.ConfigureServer(s1, s2)
s1.Handler = h

return h
}

func (h *h2cHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// Handle h2c with prior knowledge (RFC 7540 Section 3.4)
if r.Method == "PRI" && len(r.Header) == 0 && r.URL.Path == "*" && r.Proto == "HTTP/2.0" {
conn, err := initH2CWithPriorKnowledge(w)
if err != nil {
return
}
n := atomic.AddInt64(&h.conns, 1)
log.Debugf("h2c start: %d connections", n)

h.s2.ServeConn(conn, &http2.ServeConnOpts{Handler: h.handler, BaseConfig: h.s1})

n = atomic.AddInt64(&h.conns, -1)
log.Debugf("h2c done: %d connections", n)
} else {
h.handler.ServeHTTP(w, r)
}
}

// initH2CWithPriorKnowledge implements creating an h2c connection with prior
// knowledge (Section 3.4) and creates a net.Conn suitable for http2.ServeConn.
// All we have to do is look for the client preface that is supposed to be a part
// of the body, and reforward the client preface on the net.Conn this function
// creates.
func initH2CWithPriorKnowledge(w http.ResponseWriter) (net.Conn, error) {
hijacker, ok := w.(http.Hijacker)
if !ok {
return nil, errors.New("hijack is not supported")
}
conn, rw, err := hijacker.Hijack()
if err != nil {
return nil, err
}
r := rw.Reader

const expectedBody = "SM\r\n\r\n"

buf := make([]byte, len(expectedBody))
n, err := io.ReadFull(r, buf)
if err != nil {
return nil, err
}

if string(buf[:n]) == expectedBody {
return &h2cConn{
Conn: conn,
Reader: io.MultiReader(strings.NewReader(http2.ClientPreface), r),
}, nil
}

conn.Close()
return nil, errors.New("invalid client preface")
}

func (h *h2cHandler) Shutdown(ctx context.Context) error {
serr := h.s1.Shutdown(ctx)

timer := time.NewTicker(500 * time.Millisecond)
defer timer.Stop()
for {
n := atomic.LoadInt64(&h.conns)
log.Debugf("h2c shutdown: %d connections", n)

if n == 0 {
return serr
}
select {
case <-ctx.Done():
return ctx.Err()
case <-timer.C:
}
}
}

type h2cConn struct {
net.Conn
io.Reader
}

func (c *h2cConn) Read(p []byte) (int, error) {
return c.Reader.Read(p)
}
19 changes: 17 additions & 2 deletions skipper.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ import (
"github.com/zalando/skipper/filters/fadein"
logfilter "github.com/zalando/skipper/filters/log"
ratelimitfilters "github.com/zalando/skipper/filters/ratelimit"
"github.com/zalando/skipper/h2c"
"github.com/zalando/skipper/innkeeper"
"github.com/zalando/skipper/loadbalancer"
"github.com/zalando/skipper/logging"
Expand Down Expand Up @@ -343,6 +344,9 @@ type Options struct {
// a backend to always create a new connection.
DisableHTTPKeepalives bool

// Enables HTTP/2 connections over cleartext TCP with Prior Knowledge
EnableH2CPriorKnowledge bool

// Flag indicating to ignore trailing slashes in paths during route
// lookup.
IgnoreTrailingSlash bool
Expand Down Expand Up @@ -981,6 +985,10 @@ func (o *Options) tlsConfig() (*tls.Config, error) {
return nil, nil
}

if o.EnableH2CPriorKnowledge {
return nil, fmt.Errorf("TLS implies no HTTP/2 connections over cleartext TCP")
}

crts := strings.Split(o.CertPathTLS, ",")
keys := strings.Split(o.KeyPathTLS, ",")

Expand Down Expand Up @@ -1064,6 +1072,7 @@ func listenAndServeQuit(
if err != nil {
return err
}
serveTLS := tlsConfig != nil

srv := &http.Server{
Addr: o.Address,
Expand Down Expand Up @@ -1094,6 +1103,12 @@ func listenAndServeQuit(
sigs = make(chan os.Signal, 1)
}

shutdown := srv.Shutdown
if o.EnableH2CPriorKnowledge {
log.Infof("Enabling HTTP/2 connections over cleartext TCP")
shutdown = h2c.Enable(srv, nil).Shutdown
}

go func() {
signal.Notify(sigs, syscall.SIGTERM)

Expand All @@ -1103,15 +1118,15 @@ func listenAndServeQuit(
time.Sleep(o.WaitForHealthcheckInterval)

log.Info("Start shutdown")
if err := srv.Shutdown(context.Background()); err != nil {
if err := shutdown(context.Background()); err != nil {
log.Errorf("Failed to graceful shutdown: %v", err)
}
close(idleConnsCH)
}()

log.Infof("proxy listener on %v", o.Address)

if srv.TLSConfig != nil {
if serveTLS {
if err := srv.ListenAndServeTLS("", ""); err != http.ErrServerClosed {
log.Errorf("ListenAndServeTLS failed: %v", err)
return err
Expand Down
Loading

0 comments on commit f2b780b

Please sign in to comment.