issues/65: resuse address/port for pprof and redirect servers (#67)

fixes: #65
komuw · Jun 30, 2022 · 93e6c65 · 93e6c65
1 parent a7ec7f7
commit 93e6c65
Show file tree

Hide file tree

Showing 5 changed files with 139 additions and 38 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -25,3 +25,4 @@ Most recent version is listed first.
 - handle tls: https://github.com/komuw/goweb/pull/58
 - expvar metrics: https://github.com/komuw/goweb/pull/64
 - fix some races: https://github.com/komuw/goweb/pull/66
+- resuse address/port for pprof and redirect servers: https://github.com/komuw/goweb/pull/67
diff --git a/server/pprof.go b/server/pprof.go
@@ -37,7 +37,7 @@ func startPprofServer() {
 		WithFields(log.F{"pid": os.Getpid()})
 
 	port := 6060
-	addr := fmt.Sprintf("localhost:%d", port)
+	addr := fmt.Sprintf("127.0.0.1:%d", port)
 	readHeader, read, write, idle := pprofTimeouts()
 	pprofSrv := &http.Server{
 		Addr: addr,
@@ -53,10 +53,18 @@ func startPprofServer() {
 	}
 
 	go func() {
+		cfg := listenerConfig()
+		l, err := cfg.Listen(ctx, "tcp", pprofSrv.Addr)
+		if err != nil {
+			err = gowebErrors.Wrap(err)
+			logger.Error(err, log.F{"msg": "pprof server, unable to create listener"})
+			return
+		}
+
 		logger.Info(log.F{
 			"msg": fmt.Sprintf("pprof server listening at %s", pprofSrv.Addr),
 		})
-		errPprofSrv := pprofSrv.ListenAndServe()
+		errPprofSrv := pprofSrv.Serve(l)
 		if errPprofSrv != nil {
 			errPprofSrv = gowebErrors.Wrap(errPprofSrv)
 			logger.Error(errPprofSrv, log.F{"msg": "unable to start pprof server"})

diff --git a/server/pprof_test.go b/server/pprof_test.go
@@ -19,7 +19,7 @@ func TestPprofServer(t *testing.T) {
 		startPprofServer()
 
 		// await for the server to start.
-		time.Sleep((1 * time.Second))
+		time.Sleep(1 * time.Second)
 
 		uri := "/debug/pprof/heap"
 		port := 6060
@@ -35,7 +35,7 @@ func TestPprofServer(t *testing.T) {
 		startPprofServer()
 
 		// await for the server to start.
-		time.Sleep((1 * time.Second))
+		time.Sleep(1 * time.Second)
 
 		runhandler := func() {
 			uri := "/debug/pprof/heap"

diff --git a/server/server.go b/server/server.go
@@ -273,36 +273,11 @@ func sigHandler(
 }
 
 func serve(ctx context.Context, srv *http.Server, o opts, logger log.Logger) error {
-	cfg := &net.ListenConfig{Control: func(network, address string, conn syscall.RawConn) error {
-		return conn.Control(func(descriptor uintptr) {
-			_ = unix.SetsockoptInt(
-				int(descriptor),
-				unix.SOL_SOCKET,
-				// go vet will complain if we used syscall.SO_REUSEPORT, even though it would work.
-				// this is because Go considers syscall pkg to be frozen. The same goes for syscall.SetsockoptInt
-				// so we use x/sys/unix
-				// see: https://github.com/golang/go/issues/26771
-				unix.SO_REUSEPORT,
-				1,
-			)
-			_ = unix.SetsockoptInt(
-				int(descriptor),
-				unix.SOL_SOCKET,
-				unix.SO_REUSEADDR,
-				1,
-			)
-		})
-	}}
-	l, err := cfg.Listen(ctx, o.network, o.serverAddress)
-	if err != nil {
-		return gowebErrors.Wrap(err)
-	}
-
 	if o.certFile != "" {
 		{
 			// HTTP(non-tls) LISTERNER:
-			httpSrv := &http.Server{
-				Addr:              fmt.Sprintf("localhost%s", o.httpPort),
+			redirectSrv := &http.Server{
+				Addr:              fmt.Sprintf("127.0.0.1%s", o.httpPort),
 				Handler:           middleware.HttpsRedirector(srv.Handler, o.port),
 				ReadHeaderTimeout: o.readHeaderTimeout,
 				ReadTimeout:       o.readTimeout,
@@ -312,19 +287,32 @@ func serve(ctx context.Context, srv *http.Server, o opts, logger log.Logger) err
 				BaseContext:       func(net.Listener) context.Context { return ctx },
 			}
 			go func() {
+				redirectSrvCfg := listenerConfig()
+				redirectSrvListener, errL := redirectSrvCfg.Listen(ctx, "tcp", redirectSrv.Addr)
+				if errL != nil {
+					errL = gowebErrors.Wrap(errL)
+					logger.Error(errL, log.F{"msg": "redirect server, unable to create listener"})
+					return
+				}
+
 				logger.Info(log.F{
-					"msg": fmt.Sprintf("http server listening at %s", o.httpPort),
+					"msg": fmt.Sprintf("redirect server listening at %s", redirectSrv.Addr),
 				})
-				errHttpSrv := httpSrv.ListenAndServe()
-				if errHttpSrv != nil {
-					errHttpSrv = gowebErrors.Wrap(errHttpSrv)
-					logger.Error(errHttpSrv, log.F{"msg": "unable to start http listener for redirects"})
+				errRedirectSrv := redirectSrv.Serve(redirectSrvListener)
+				if errRedirectSrv != nil {
+					errRedirectSrv = gowebErrors.Wrap(errRedirectSrv)
+					logger.Error(errRedirectSrv, log.F{"msg": "unable to start redirect server"})
 				}
 			}()
 		}
 
 		{
 			// HTTPS(tls) LISTERNER:
+			cfg := listenerConfig()
+			l, err := cfg.Listen(ctx, o.network, o.serverAddress)
+			if err != nil {
+				return gowebErrors.Wrap(err)
+			}
 			logger.Info(log.F{
 				"msg": fmt.Sprintf("https server listening at %s", o.serverAddress),
 			})
@@ -333,6 +321,11 @@ func serve(ctx context.Context, srv *http.Server, o opts, logger log.Logger) err
 			}
 		}
 	} else {
+		cfg := listenerConfig()
+		l, err := cfg.Listen(ctx, o.network, o.serverAddress)
+		if err != nil {
+			return gowebErrors.Wrap(err)
+		}
 		logger.Info(log.F{
 			"msg": fmt.Sprintf("http server listening at %s", o.serverAddress),
 		})
@@ -367,3 +360,30 @@ func drainDuration(o opts) time.Duration {
 
 	return dur
 }
+
+// listenerConfig creates a net listener config that reuses address and port.
+// This is essential in order to be able to carry out zero-downtime deploys.
+func listenerConfig() *net.ListenConfig {
+	return &net.ListenConfig{
+		Control: func(network, address string, conn syscall.RawConn) error {
+			return conn.Control(func(descriptor uintptr) {
+				_ = unix.SetsockoptInt(
+					int(descriptor),
+					unix.SOL_SOCKET,
+					// go vet will complain if we used syscall.SO_REUSEPORT, even though it would work.
+					// this is because Go considers syscall pkg to be frozen. The same goes for syscall.SetsockoptInt
+					// so we use x/sys/unix
+					// see: https://github.com/golang/go/issues/26771
+					unix.SO_REUSEPORT,
+					1,
+				)
+				_ = unix.SetsockoptInt(
+					int(descriptor),
+					unix.SOL_SOCKET,
+					unix.SO_REUSEADDR,
+					1,
+				)
+			})
+		},
+	}
+}
diff --git a/server/server_test.go b/server/server_test.go
@@ -1,6 +1,7 @@
 package server
 
 import (
+	"crypto/tls"
 	"fmt"
 	"io"
 	"math"
@@ -160,7 +161,7 @@ func TestServer(t *testing.T) {
 		}()
 
 		// await for the server to start.
-		time.Sleep((1 * time.Second))
+		time.Sleep(1 * time.Second)
 
 		res, err := http.Get(fmt.Sprintf("http://127.0.0.1:%d%s", port, uri))
 		attest.Ok(t, err)
@@ -173,6 +174,77 @@ func TestServer(t *testing.T) {
 		attest.Equal(t, string(rb), msg)
 	})
 
+	t.Run("tls", func(t *testing.T) {
+		t.Parallel()
+
+		if os.Getenv("GITHUB_ACTIONS") != "" {
+			// server.Run() calls setRlimit()
+			// and setRlimit() fails in github actions with error: `operation not permitted`
+			// specifically the call to `unix.Setrlimit()`
+			return
+		}
+
+		port := 8081
+		uri := "/api"
+		msg := "hello world"
+		mux := NewMux(
+			Routes{
+				NewRoute(
+					uri,
+					MethodGet,
+					someServerTestHandler(msg),
+					middleware.WithOpts("localhost"),
+				),
+			})
+
+		go func() {
+			_, _ = CreateDevCertKey()
+			time.Sleep(1 * time.Second)
+			err := Run(mux, DefaultTlsOpts())
+			attest.Ok(t, err)
+		}()
+
+		// await for the server to start.
+		time.Sleep(7 * time.Second)
+
+		tr := &http.Transport{
+			// since we are using self-signed certificates, we need to skip verification.
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		}
+		client := &http.Client{Transport: tr}
+
+		{
+			// https server.
+			res, err := client.Get(fmt.Sprintf(
+				// note: the https scheme.
+				"https://127.0.0.1:%d%s",
+				port,
+				uri,
+			))
+			attest.Ok(t, err)
+
+			defer res.Body.Close()
+			rb, err := io.ReadAll(res.Body)
+			attest.Ok(t, err)
+
+			attest.Equal(t, res.StatusCode, http.StatusOK)
+			attest.Equal(t, string(rb), msg)
+		}
+
+		{
+			// redirect server
+			res, err := client.Get(fmt.Sprintf("http://127.0.0.1:%d%s", port-1, uri))
+			attest.Ok(t, err)
+
+			defer res.Body.Close()
+			rb, err := io.ReadAll(res.Body)
+			attest.Ok(t, err)
+
+			attest.Equal(t, res.StatusCode, http.StatusOK)
+			attest.Equal(t, string(rb), msg)
+		}
+	})
+
 	t.Run("concurrency safe", func(t *testing.T) {
 		t.Parallel()
 
@@ -202,7 +274,7 @@ func TestServer(t *testing.T) {
 		}()
 
 		// await for the server to start.
-		time.Sleep((1 * time.Second))
+		time.Sleep(1 * time.Second)
 
 		runhandler := func() {
 			res, err := http.Get(fmt.Sprintf("http://127.0.0.1:%d%s", port, uri))