Implements HTTPS graceful shutdown (#1866)

Fixes #1865

Signed-off-by: Alexander Yastrebov <[email protected]>

fixtures/gencert.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+if [[ $# -eq 0 ]] ; then
+  echo 'Certificate name required'
+  exit 1
+fi
+
+openssl req -x509 -sha256 -nodes -newkey rsa:2048 -keyout "$1.key" -out "$1.crt" \
+  -days 3650 \
+  -subj "/C=DE/ST=Berlin/O=Zalando SE/OU=Technology/CN=do-not-trust-test-data"

fixtures/test2.crt

@@ -0,0 +1,22 @@
+-----BEGIN CERTIFICATE-----
+MIIDszCCApugAwIBAgIUfXes2jmw6GNLTEObvIsMrlx4NX0wDQYJKoZIhvcNAQEL
+BQAwaTELMAkGA1UEBhMCREUxDzANBgNVBAgMBkJlcmxpbjETMBEGA1UECgwKWmFs
+YW5kbyBTRTETMBEGA1UECwwKVGVjaG5vbG9neTEfMB0GA1UEAwwWZG8tbm90LXRy
+dXN0LXRlc3QtZGF0YTAeFw0yMTA5MjQxMTQ1MjBaFw0zMTA5MjIxMTQ1MjBaMGkx
+CzAJBgNVBAYTAkRFMQ8wDQYDVQQIDAZCZXJsaW4xEzARBgNVBAoMClphbGFuZG8g
+U0UxEzARBgNVBAsMClRlY2hub2xvZ3kxHzAdBgNVBAMMFmRvLW5vdC10cnVzdC10
+ZXN0LWRhdGEwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDLvAr/DUEe
+2T+TzPjoT/Q32892rNXz3p/M4RopdqJWLEZwzASh8Z7FiLBiGOIsWrGhWluwqQCy
+nrpulZ/sQOqi8KYjyhsvlRV68L9wczX7GZScxnMCfvsEAJJD0E9GqwtOal+yCmvx
+59/5V7sLD6JzrPZWoUL4qEEXk1CoGCjSHGjHUPvVqX1oZmyzYKh73oiUKQ//DqYR
+9zhUaGO01R1QvNISMqFMsdYLW9SsatuRiyriAfTsslbIg3jQ+l/n7Y7YztJvbrbe
+gB6cMqrignXskNtlqZnEGbJgiX8Ko7m4Y7ttmOMcawLqhwWxzvKj9iBERL6irxqr
+Q15mpV915O4vAgMBAAGjUzBRMB0GA1UdDgQWBBSJv18vO9LQKJsCkTiIZ0vhI2xb
+KjAfBgNVHSMEGDAWgBSJv18vO9LQKJsCkTiIZ0vhI2xbKjAPBgNVHRMBAf8EBTAD
+AQH/MA0GCSqGSIb3DQEBCwUAA4IBAQC0o9QFpmTe0YC5sfRhJcjmejGggS4IZB5U
+KQ6/S+Xk/W4UitQCo7iQot+ZF5HOmaGX6FtpUCSx1JeSWxAkzbl8LK1dCO6YNXJ6
+2EzkPthcf5EJkJ9rpCRL5dAoPSXr1EJut646jlR/hwlxOEWCOr7zYQK136hc+HFA
+6No4tshp2l4mT+OfR3wn8Ae1hH+tyuWzXGlBdUlm/2OtsUwHjHKeiv/ZnQ63K+gg
+nxdkUvQL07O0Avz9ttz9yZ9jYAi00zjotG7e+buSbarGBTuXXxnlG4+ISclQikJ4
+m/ApwEDTocYpOJgsdn+bysY/F2nntrAQ1Sof2E6N5ww9sR8taGe3
+-----END CERTIFICATE-----

fixtures/test2.key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDLvAr/DUEe2T+T
+zPjoT/Q32892rNXz3p/M4RopdqJWLEZwzASh8Z7FiLBiGOIsWrGhWluwqQCynrpu
+lZ/sQOqi8KYjyhsvlRV68L9wczX7GZScxnMCfvsEAJJD0E9GqwtOal+yCmvx59/5
+V7sLD6JzrPZWoUL4qEEXk1CoGCjSHGjHUPvVqX1oZmyzYKh73oiUKQ//DqYR9zhU
+aGO01R1QvNISMqFMsdYLW9SsatuRiyriAfTsslbIg3jQ+l/n7Y7YztJvbrbegB6c
+MqrignXskNtlqZnEGbJgiX8Ko7m4Y7ttmOMcawLqhwWxzvKj9iBERL6irxqrQ15m
+pV915O4vAgMBAAECggEATQPKjFuwUD8Dn5WOShNfWHZJWK1BO6zeb45wW1gzSav2
+/NDCt40k3bssIgkSBn5KQ5pqqr9YOi1ygDcjeyWXDP03cLQHztbmhdDYLWP/9enX
+meQSudDShtLId8YZEbe60Gu5vQ3ffFSRACq/1BCW8m9ht6HCNUk1Qfo4NTLcy3+x
+F8gI65jGTPiZzJ7eHz7576KBcgeArv14DSm+jPjkBiKWzrUb9sV8elaps5j6yIjL
+glMimWhlTuajt/9d8QD/qN+uR67OYcWtflHnRI4FSRrwyPfN80tK3a9WjpuBKlFZ
+MwLaNU3gCw2x9VW4xVOKyFhuQzj7uHUn+vUCWjr5UQKBgQDm5XXUzRu5dRx6n5sR
+P3TrIs8UocKOuaNRgp3+aYjhF578szGP9lLG/he58w7irTYEZxvdwCRSE7HADwQv
+GUZmyyVtwSuO4yE0xVOVtf02SRJnn7akAWy/CL4V87RbpFGYDXdNtN/0600VaYxO
+UxKjw/K9yeCoAZv9Q+fPXKlO7QKBgQDh4paieu1ZvwNcRX1FFYwtq5YHTelL4AD/
+CNQXZLlIp2i4Z0rcrfaIovb5sh5sJ+WXH5YJ0VIzgNe6tuTKMU6Cc45UF4Hb0ZTx
+sVjAqZp+6KnajOHBdkc0w5c0Hv6UpAKPdZUzsKlb5r0kT4nRPB1/Ov3CUAqJi/JQ
+FZtYgz1yCwKBgAxXcX/pYrT8BIStaU13tdknqCfzKYIVfBxMPgOuQmm9qHrbXSfT
+w8LtK/l9e2s0VPHRTRUCQy677MFWTCP0VuYBr8N5Esn1a/31Gi2jZ6ByMXCmgc2s
+YdKoNfjYaOiJFO9qsNjPdTUTKrCdTqmVGSb1v1DTrJVuWJcl/QsBae9VAoGAbLHM
+KoNck0MHKu+FSCkGOzPGDd2/1XMFB7QH2vns7rkf+xw5Ode8OiOxFJZRbVoFcKMS
+X8cJ9x6YsJAxp9nyHXPdmTl2k4BWW7crLgpu/YKXuULxn1Z7DTjRGZOQjZYeZUn/
+cdAgrshpW3+qobR7vS11znsVlvpwr3i2N/FvL+ECgYEAwcvBK5wtd70YweHZ4Rm8
+P8iUsjeVV9LuBT0/0jL46Q48rGxMnHK+hUxXPCp/wl4BrML/lVsTvPZh+KlYQz4C
+yk/f2FB2aC8dCFXcsGswVXC0WQGStogkXf56nr05b8Av0LoQ4/REG0NDSoYaApVF
+U1EucR/02DUrc+LE0j44D9o=
+-----END PRIVATE KEY-----

skipper.go

@@ -944,8 +944,35 @@ func initLog(o Options) error {
 	return nil
 }
 
-func (o *Options) isHTTPS() bool {
-	return (o.ProxyTLS != nil) || (o.CertPathTLS != "" && o.KeyPathTLS != "")
+func (o *Options) tlsConfig() (*tls.Config, error) {
+	if o.ProxyTLS != nil {
+		return o.ProxyTLS, nil
+	}
+
+	if o.CertPathTLS == "" && o.KeyPathTLS == "" {
+		return nil, nil
+	}
+
+	crts := strings.Split(o.CertPathTLS, ",")
+	keys := strings.Split(o.KeyPathTLS, ",")
+
+	if len(crts) != len(keys) {
+		return nil, fmt.Errorf("number of certificates does not match number of keys")
+	}
+
+	config := &tls.Config{
+		MinVersion: o.TLSMinVersion,
+	}
+
+	for i := 0; i < len(crts); i++ {
+		crt, key := crts[i], keys[i]
+		keypair, err := tls.LoadX509KeyPair(crt, key)
+		if err != nil {
+			return nil, fmt.Errorf("failed to load X509 keypair from %s and %s: %w", crt, key, err)
+		}
+		config.Certificates = append(config.Certificates, keypair)
+	}
+	return config, nil
 }
 
 func listen(o *Options, mtr metrics.Metrics) (net.Listener, error) {
@@ -1005,11 +1032,14 @@ func listenAndServeQuit(
 	idleConnsCH chan struct{},
 	mtr metrics.Metrics,
 ) error {
-	// create the access log handler
-	log.Infof("proxy listener on %v", o.Address)
+	tlsConfig, err := o.tlsConfig()
+	if err != nil {
+		return err
+	}
 
 	srv := &http.Server{
 		Addr:              o.Address,
+		TLSConfig:         tlsConfig,
 		Handler:           proxy,
 		ReadTimeout:       o.ReadTimeoutServer,
 		ReadHeaderTimeout: o.ReadHeaderTimeoutServer,
@@ -1025,35 +1055,6 @@ func listenAndServeQuit(
 		}
 	}
 
-	if o.isHTTPS() {
-		if o.ProxyTLS != nil {
-			srv.TLSConfig = o.ProxyTLS
-			o.CertPathTLS = ""
-			o.KeyPathTLS = ""
-		} else if strings.Index(o.CertPathTLS, ",") > 0 && strings.Index(o.KeyPathTLS, ",") > 0 {
-			tlsCfg := &tls.Config{
-				MinVersion: o.TLSMinVersion,
-			}
-			crts := strings.Split(o.CertPathTLS, ",")
-			keys := strings.Split(o.KeyPathTLS, ",")
-			if len(crts) != len(keys) {
-				log.Fatalf("number of certs does not match number of keys")
-			}
-			for i, crt := range crts {
-				kp, err := tls.LoadX509KeyPair(crt, keys[i])
-				if err != nil {
-					log.Fatalf("Failed to load X509 keypair from %s/%s: %v", crt, keys[i], err)
-				}
-				tlsCfg.Certificates = append(tlsCfg.Certificates, kp)
-			}
-			o.CertPathTLS = ""
-			o.KeyPathTLS = ""
-			srv.TLSConfig = tlsCfg
-		}
-		return srv.ListenAndServeTLS(o.CertPathTLS, o.KeyPathTLS)
-	}
-	log.Infof("TLS settings not found, defaulting to HTTP")
-
 	// making idleConnsCH and sigs optional parameters is required to be able to tear down a server
 	// from the tests
 	if idleConnsCH == nil {
@@ -1079,14 +1080,25 @@ func listenAndServeQuit(
 		close(idleConnsCH)
 	}()
 
-	l, err := listen(o, mtr)
-	if err != nil {
-		return err
-	}
+	log.Infof("proxy listener on %v", o.Address)
 
-	if err := srv.Serve(l); err != nil && err != http.ErrServerClosed {
-		log.Errorf("Failed to start to ListenAndServe: %v", err)
-		return err
+	if srv.TLSConfig != nil {
+		if err := srv.ListenAndServeTLS("", ""); err != http.ErrServerClosed {
+			log.Errorf("ListenAndServeTLS failed: %v", err)
+			return err
+		}
+	} else {
+		log.Infof("TLS settings not found, defaulting to HTTP")
+
+		l, err := listen(o, mtr)
+		if err != nil {
+			return err
+		}
+
+		if err := srv.Serve(l); err != http.ErrServerClosed {
+			log.Errorf("Serve failed: %v", err)
+			return err
+		}
 	}
 
 	<-idleConnsCH