From 93e6c6584c0743bb6618264f5d07646dbdf3e6df Mon Sep 17 00:00:00 2001 From: Komu Wairagu Date: Thu, 30 Jun 2022 10:10:20 +0300 Subject: [PATCH] issues/65: resuse address/port for pprof and redirect servers (#67) fixes: https://github.com/komuw/goweb/issues/65 --- CHANGELOG.md | 1 + server/pprof.go | 12 +++++-- server/pprof_test.go | 4 +-- server/server.go | 84 ++++++++++++++++++++++++++----------------- server/server_test.go | 76 +++++++++++++++++++++++++++++++++++++-- 5 files changed, 139 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9c91c9b..03e40c70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,3 +25,4 @@ Most recent version is listed first. - handle tls: https://github.com/komuw/goweb/pull/58 - expvar metrics: https://github.com/komuw/goweb/pull/64 - fix some races: https://github.com/komuw/goweb/pull/66 +- resuse address/port for pprof and redirect servers: https://github.com/komuw/goweb/pull/67 diff --git a/server/pprof.go b/server/pprof.go index de6d5e64..f2f98675 100644 --- a/server/pprof.go +++ b/server/pprof.go @@ -37,7 +37,7 @@ func startPprofServer() { WithFields(log.F{"pid": os.Getpid()}) port := 6060 - addr := fmt.Sprintf("localhost:%d", port) + addr := fmt.Sprintf("127.0.0.1:%d", port) readHeader, read, write, idle := pprofTimeouts() pprofSrv := &http.Server{ Addr: addr, @@ -53,10 +53,18 @@ func startPprofServer() { } go func() { + cfg := listenerConfig() + l, err := cfg.Listen(ctx, "tcp", pprofSrv.Addr) + if err != nil { + err = gowebErrors.Wrap(err) + logger.Error(err, log.F{"msg": "pprof server, unable to create listener"}) + return + } + logger.Info(log.F{ "msg": fmt.Sprintf("pprof server listening at %s", pprofSrv.Addr), }) - errPprofSrv := pprofSrv.ListenAndServe() + errPprofSrv := pprofSrv.Serve(l) if errPprofSrv != nil { errPprofSrv = gowebErrors.Wrap(errPprofSrv) logger.Error(errPprofSrv, log.F{"msg": "unable to start pprof server"}) diff --git a/server/pprof_test.go b/server/pprof_test.go index 3c1a5e15..59c14178 100644 --- a/server/pprof_test.go +++ b/server/pprof_test.go @@ -19,7 +19,7 @@ func TestPprofServer(t *testing.T) { startPprofServer() // await for the server to start. - time.Sleep((1 * time.Second)) + time.Sleep(1 * time.Second) uri := "/debug/pprof/heap" port := 6060 @@ -35,7 +35,7 @@ func TestPprofServer(t *testing.T) { startPprofServer() // await for the server to start. - time.Sleep((1 * time.Second)) + time.Sleep(1 * time.Second) runhandler := func() { uri := "/debug/pprof/heap" diff --git a/server/server.go b/server/server.go index b2a9b11b..e057f3df 100644 --- a/server/server.go +++ b/server/server.go @@ -273,36 +273,11 @@ func sigHandler( } func serve(ctx context.Context, srv *http.Server, o opts, logger log.Logger) error { - cfg := &net.ListenConfig{Control: func(network, address string, conn syscall.RawConn) error { - return conn.Control(func(descriptor uintptr) { - _ = unix.SetsockoptInt( - int(descriptor), - unix.SOL_SOCKET, - // go vet will complain if we used syscall.SO_REUSEPORT, even though it would work. - // this is because Go considers syscall pkg to be frozen. The same goes for syscall.SetsockoptInt - // so we use x/sys/unix - // see: https://github.com/golang/go/issues/26771 - unix.SO_REUSEPORT, - 1, - ) - _ = unix.SetsockoptInt( - int(descriptor), - unix.SOL_SOCKET, - unix.SO_REUSEADDR, - 1, - ) - }) - }} - l, err := cfg.Listen(ctx, o.network, o.serverAddress) - if err != nil { - return gowebErrors.Wrap(err) - } - if o.certFile != "" { { // HTTP(non-tls) LISTERNER: - httpSrv := &http.Server{ - Addr: fmt.Sprintf("localhost%s", o.httpPort), + redirectSrv := &http.Server{ + Addr: fmt.Sprintf("127.0.0.1%s", o.httpPort), Handler: middleware.HttpsRedirector(srv.Handler, o.port), ReadHeaderTimeout: o.readHeaderTimeout, ReadTimeout: o.readTimeout, @@ -312,19 +287,32 @@ func serve(ctx context.Context, srv *http.Server, o opts, logger log.Logger) err BaseContext: func(net.Listener) context.Context { return ctx }, } go func() { + redirectSrvCfg := listenerConfig() + redirectSrvListener, errL := redirectSrvCfg.Listen(ctx, "tcp", redirectSrv.Addr) + if errL != nil { + errL = gowebErrors.Wrap(errL) + logger.Error(errL, log.F{"msg": "redirect server, unable to create listener"}) + return + } + logger.Info(log.F{ - "msg": fmt.Sprintf("http server listening at %s", o.httpPort), + "msg": fmt.Sprintf("redirect server listening at %s", redirectSrv.Addr), }) - errHttpSrv := httpSrv.ListenAndServe() - if errHttpSrv != nil { - errHttpSrv = gowebErrors.Wrap(errHttpSrv) - logger.Error(errHttpSrv, log.F{"msg": "unable to start http listener for redirects"}) + errRedirectSrv := redirectSrv.Serve(redirectSrvListener) + if errRedirectSrv != nil { + errRedirectSrv = gowebErrors.Wrap(errRedirectSrv) + logger.Error(errRedirectSrv, log.F{"msg": "unable to start redirect server"}) } }() } { // HTTPS(tls) LISTERNER: + cfg := listenerConfig() + l, err := cfg.Listen(ctx, o.network, o.serverAddress) + if err != nil { + return gowebErrors.Wrap(err) + } logger.Info(log.F{ "msg": fmt.Sprintf("https server listening at %s", o.serverAddress), }) @@ -333,6 +321,11 @@ func serve(ctx context.Context, srv *http.Server, o opts, logger log.Logger) err } } } else { + cfg := listenerConfig() + l, err := cfg.Listen(ctx, o.network, o.serverAddress) + if err != nil { + return gowebErrors.Wrap(err) + } logger.Info(log.F{ "msg": fmt.Sprintf("http server listening at %s", o.serverAddress), }) @@ -367,3 +360,30 @@ func drainDuration(o opts) time.Duration { return dur } + +// listenerConfig creates a net listener config that reuses address and port. +// This is essential in order to be able to carry out zero-downtime deploys. +func listenerConfig() *net.ListenConfig { + return &net.ListenConfig{ + Control: func(network, address string, conn syscall.RawConn) error { + return conn.Control(func(descriptor uintptr) { + _ = unix.SetsockoptInt( + int(descriptor), + unix.SOL_SOCKET, + // go vet will complain if we used syscall.SO_REUSEPORT, even though it would work. + // this is because Go considers syscall pkg to be frozen. The same goes for syscall.SetsockoptInt + // so we use x/sys/unix + // see: https://github.com/golang/go/issues/26771 + unix.SO_REUSEPORT, + 1, + ) + _ = unix.SetsockoptInt( + int(descriptor), + unix.SOL_SOCKET, + unix.SO_REUSEADDR, + 1, + ) + }) + }, + } +} diff --git a/server/server_test.go b/server/server_test.go index d37ccc76..42d8d8ce 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -1,6 +1,7 @@ package server import ( + "crypto/tls" "fmt" "io" "math" @@ -160,7 +161,7 @@ func TestServer(t *testing.T) { }() // await for the server to start. - time.Sleep((1 * time.Second)) + time.Sleep(1 * time.Second) res, err := http.Get(fmt.Sprintf("http://127.0.0.1:%d%s", port, uri)) attest.Ok(t, err) @@ -173,6 +174,77 @@ func TestServer(t *testing.T) { attest.Equal(t, string(rb), msg) }) + t.Run("tls", func(t *testing.T) { + t.Parallel() + + if os.Getenv("GITHUB_ACTIONS") != "" { + // server.Run() calls setRlimit() + // and setRlimit() fails in github actions with error: `operation not permitted` + // specifically the call to `unix.Setrlimit()` + return + } + + port := 8081 + uri := "/api" + msg := "hello world" + mux := NewMux( + Routes{ + NewRoute( + uri, + MethodGet, + someServerTestHandler(msg), + middleware.WithOpts("localhost"), + ), + }) + + go func() { + _, _ = CreateDevCertKey() + time.Sleep(1 * time.Second) + err := Run(mux, DefaultTlsOpts()) + attest.Ok(t, err) + }() + + // await for the server to start. + time.Sleep(7 * time.Second) + + tr := &http.Transport{ + // since we are using self-signed certificates, we need to skip verification. + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + client := &http.Client{Transport: tr} + + { + // https server. + res, err := client.Get(fmt.Sprintf( + // note: the https scheme. + "https://127.0.0.1:%d%s", + port, + uri, + )) + attest.Ok(t, err) + + defer res.Body.Close() + rb, err := io.ReadAll(res.Body) + attest.Ok(t, err) + + attest.Equal(t, res.StatusCode, http.StatusOK) + attest.Equal(t, string(rb), msg) + } + + { + // redirect server + res, err := client.Get(fmt.Sprintf("http://127.0.0.1:%d%s", port-1, uri)) + attest.Ok(t, err) + + defer res.Body.Close() + rb, err := io.ReadAll(res.Body) + attest.Ok(t, err) + + attest.Equal(t, res.StatusCode, http.StatusOK) + attest.Equal(t, string(rb), msg) + } + }) + t.Run("concurrency safe", func(t *testing.T) { t.Parallel() @@ -202,7 +274,7 @@ func TestServer(t *testing.T) { }() // await for the server to start. - time.Sleep((1 * time.Second)) + time.Sleep(1 * time.Second) runhandler := func() { res, err := http.Get(fmt.Sprintf("http://127.0.0.1:%d%s", port, uri))