bootstrap: support for Envoy xDS certificate rotation

Added new flag --resources-dir to bootstrap command for supporting path-based
SDS resources to define xDS certificate and key.  When using the flag,
following changes take place:

- Bootstrap file will contain references to SDS resource files instead of
  direct cert/key paths.
- SDS resource file is written into resources dir for xDS client cert and key
- SDS resource file is written into resources dir for xDS trusted CA cert

With this change there is no need to restart Envoy anymore when certificates
are rotated.  Envoy will monitor and automatically reload the certificate and
key files without causing an interruption to traffic.

Any recent version of Envoy supports the above configuration, but Envoy 1.14.1
or later is required for automatic certificate reload.

Signed-off-by: Tero Saarni <[email protected]>

cmd/contour/bootstrap.go

@@ -14,56 +14,25 @@
 package main
 
 import (
-	"io"
-	"os"
-
-	"github.com/golang/protobuf/jsonpb"
 	"github.com/projectcontour/contour/internal/envoy"
 	kingpin "gopkg.in/alecthomas/kingpin.v2"
 )
 
 // registerBootstrap registers the bootstrap subcommand and flags
 // with the Application provided.
-func registerBootstrap(app *kingpin.Application) (*kingpin.CmdClause, *bootstrapContext) {
-	var ctx bootstrapContext
+func registerBootstrap(app *kingpin.Application) (*kingpin.CmdClause, *envoy.BootstrapConfig) {
+	var config envoy.BootstrapConfig
 
 	bootstrap := app.Command("bootstrap", "Generate bootstrap configuration.")
-	bootstrap.Arg("path", "Configuration file ('-' for standard output).").Required().StringVar(&ctx.path)
-	bootstrap.Flag("admin-address", "Envoy admin interface address.").StringVar(&ctx.config.AdminAddress)
-	bootstrap.Flag("admin-port", "Envoy admin interface port.").IntVar(&ctx.config.AdminPort)
-	bootstrap.Flag("xds-address", "xDS gRPC API address.").StringVar(&ctx.config.XDSAddress)
-	bootstrap.Flag("xds-port", "xDS gRPC API port.").IntVar(&ctx.config.XDSGRPCPort)
-	bootstrap.Flag("envoy-cafile", "gRPC CA Filename for Envoy to load.").Envar("ENVOY_CAFILE").StringVar(&ctx.config.GrpcCABundle)
-	bootstrap.Flag("envoy-cert-file", "gRPC Client cert filename for Envoy to load.").Envar("ENVOY_CERT_FILE").StringVar(&ctx.config.GrpcClientCert)
-	bootstrap.Flag("envoy-key-file", "gRPC Client key filename for Envoy to load.").Envar("ENVOY_KEY_FILE").StringVar(&ctx.config.GrpcClientKey)
-	bootstrap.Flag("namespace", "The namespace the Envoy container will run in.").Envar("CONTOUR_NAMESPACE").Default("projectcontour").StringVar(&ctx.config.Namespace)
-	return bootstrap, &ctx
-}
-
-type bootstrapContext struct {
-	config envoy.BootstrapConfig
-	path   string
-}
-
-// doBootstrap writes an Envoy bootstrap configuration file to the supplied path.
-func doBootstrap(ctx *bootstrapContext) {
-	var out io.Writer
-
-	switch ctx.path {
-	case "-":
-		out = os.Stdout
-	default:
-		f, err := os.Create(ctx.path)
-		check(err)
-
-		out = f
-
-		defer func() {
-			check(f.Close())
-		}()
-	}
-
-	m := &jsonpb.Marshaler{OrigName: true}
-
-	check(m.Marshal(out, envoy.Bootstrap(&ctx.config)))
+	bootstrap.Arg("path", "Configuration file ('-' for standard output).").Required().StringVar(&config.Path)
+	bootstrap.Flag("resources-dir", "Directory where configuration files will be written to.").StringVar(&config.ResourcesDir)
+	bootstrap.Flag("admin-address", "Envoy admin interface address.").StringVar(&config.AdminAddress)
+	bootstrap.Flag("admin-port", "Envoy admin interface port.").IntVar(&config.AdminPort)
+	bootstrap.Flag("xds-address", "xDS gRPC API address.").StringVar(&config.XDSAddress)
+	bootstrap.Flag("xds-port", "xDS gRPC API port.").IntVar(&config.XDSGRPCPort)
+	bootstrap.Flag("envoy-cafile", "gRPC CA Filename for Envoy to load.").Envar("ENVOY_CAFILE").StringVar(&config.GrpcCABundle)
+	bootstrap.Flag("envoy-cert-file", "gRPC Client cert filename for Envoy to load.").Envar("ENVOY_CERT_FILE").StringVar(&config.GrpcClientCert)
+	bootstrap.Flag("envoy-key-file", "gRPC Client key filename for Envoy to load.").Envar("ENVOY_KEY_FILE").StringVar(&config.GrpcClientKey)
+	bootstrap.Flag("namespace", "The namespace the Envoy container will run in.").Envar("CONTOUR_NAMESPACE").Default("projectcontour").StringVar(&config.Namespace)
+	return bootstrap, &config
 }

cmd/contour/contour.go

@@ -19,6 +19,7 @@ import (
 
 	resource "github.com/envoyproxy/go-control-plane/pkg/resource/v2"
 	"github.com/projectcontour/contour/internal/build"
+	"github.com/projectcontour/contour/internal/envoy"
 	"github.com/sirupsen/logrus"
 	kingpin "gopkg.in/alecthomas/kingpin.v2"
 	"k8s.io/klog"
@@ -36,8 +37,8 @@ func main() {
 	log := logrus.StandardLogger()
 	app := kingpin.New("contour", "Contour Kubernetes ingress controller.")
 
-	envoy := app.Command("envoy", "Sub-command for envoy actions.")
-	shutdownManager, shutdownManagerCtx := registerShutdownManager(envoy, log)
+	envoyCmd := app.Command("envoy", "Sub-command for envoy actions.")
+	shutdownManager, shutdownManagerCtx := registerShutdownManager(envoyCmd, log)
 
 	bootstrap, bootstrapCtx := registerBootstrap(app)
 	certgenApp, certgenConfig := registerCertGen(app)
@@ -69,7 +70,7 @@ func main() {
 	case shutdownManager.FullCommand():
 		doShutdownManager(shutdownManagerCtx)
 	case bootstrap.FullCommand():
-		doBootstrap(bootstrapCtx)
+		check(envoy.WriteBootstrap(bootstrapCtx))
 	case certgenApp.FullCommand():
 		doCertgen(certgenConfig)
 	case cds.FullCommand():

internal/envoy/bootstrap.go

@@ -16,7 +16,9 @@
 package envoy
 
 import (
-	"log"
+	"fmt"
+	"os"
+	"path"
 	"strconv"
 	"strings"
 	"time"
@@ -25,19 +27,99 @@ import (
 	envoy_api_v2_auth "github.com/envoyproxy/go-control-plane/envoy/api/v2/auth"
 	clusterv2 "github.com/envoyproxy/go-control-plane/envoy/api/v2/cluster"
 	envoy_api_v2_core "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
-	bootstrap "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v2"
+	envoy_api_bootstrap "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v2"
 	matcher "github.com/envoyproxy/go-control-plane/envoy/type/matcher"
+	"github.com/golang/protobuf/jsonpb"
+	"github.com/golang/protobuf/proto"
+	"github.com/golang/protobuf/ptypes/any"
 	"github.com/projectcontour/contour/internal/protobuf"
 )
 
-// Bootstrap creates a new v2 Bootstrap configuration.
-func Bootstrap(c *BootstrapConfig) *bootstrap.Bootstrap {
-	b := &bootstrap.Bootstrap{
-		DynamicResources: &bootstrap.Bootstrap_DynamicResources{
+// sdsResourcesSubdirectory stores the subdirectory name where SDS path resources are stored to.
+const sdsResourcesSubdirectory = "sds"
+
+// sdsTLSCertificateFile stores the path to the SDS resource with Envoy's
+// client certificate and key for XDS gRPC connection.
+const sdsTLSCertificateFile = "xds-tls-certicate.json"
+
+// sdsValidationContextFile stores the path to the SDS resource with
+// CA certificates for Envoy to use for the XDS gRPC connection.
+const sdsValidationContextFile = "xds-validation-context.json"
+
+// ConfigFiles is a map from configuration filename to configuration file content.
+type ConfigFiles map[string]proto.Message
+
+// WriteBootstrap writes bootstrap configuration to files.
+func WriteBootstrap(c *BootstrapConfig) error {
+	if c.ResourcesDir != "" {
+		if err := os.MkdirAll(path.Join(c.ResourcesDir, "sds"), 0755); err != nil {
+			return err
+		}
+	}
+
+	// Create Envoy bootstrap config and associated resource files.
+	configFiles, err := bootstrap(c)
+	if err != nil {
+		return err
+	}
+
+	// Write all configuration files out to filesystem.
+	for path, config := range configFiles {
+		if err := writeConfig(path, config); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// bootstrap creates a new v2 bootstrap configuration and associated resource files.
+func bootstrap(c *BootstrapConfig) (ConfigFiles, error) {
+	configFiles := make(ConfigFiles)
+
+	b := bootstrapConfig(c)
+
+	if c.GrpcClientCert != "" || c.GrpcClientKey != "" || c.GrpcCABundle != "" {
+		// If one of the two TLS options is not empty, they all must be not empty
+		if !(c.GrpcClientCert != "" && c.GrpcClientKey != "" && c.GrpcCABundle != "") {
+			return nil, fmt.Errorf("you must supply all three TLS parameters - --envoy-cafile, --envoy-cert-file, --envoy-key-file, or none of them")
+		}
+
+		var tls *envoy_api_v2_core.TransportSocket
+
+		if c.ResourcesDir != "" {
+			// xDS certificate rotation is supported by Envoy by using SDS path based resource files.
+			// These files are JSON representation of the SDS protobuf messages that normally get sent over the xDS connection,
+			// but for xDS connection itself, bootstrapping is done by storing the SDS resources in a local filesystem.
+			// Envoy will monitor and reload the resource files and the certificate and key files referred from the SDS resources.
+			//
+			// Two files are written to ResourcesDir:
+			// - SDS resource for xDS client certificate and key for authenticating Envoy towards Contour.
+			// - SDS resource for trusted CA certificate for validating Contour server certificate.
+			sdsTLSCertificatePath := path.Join(c.ResourcesDir, sdsResourcesSubdirectory, sdsTLSCertificateFile)
+			sdsValidationContextPath := path.Join(c.ResourcesDir, sdsResourcesSubdirectory, sdsValidationContextFile)
+			configFiles[sdsTLSCertificatePath] = tlsCertificateSdsSecretConfig(c)
+			configFiles[sdsValidationContextPath] = validationContextSdsSecretConfig(c)
+			tls = UpstreamTLSTransportSocket(upstreamSdsTLSContext(sdsTLSCertificatePath, sdsValidationContextPath))
+		} else {
+			// For backwards compatibility, the old behavior is to use direct certificate and key file paths in bootstrap config.
+			// Envoy does not support rotation of xDS certificate files in this case.
+			tls = UpstreamTLSTransportSocket(upstreamFileTLSContext(c))
+		}
+		b.StaticResources.Clusters[0].TransportSocket = tls
+	}
+	configFiles[c.Path] = b
+
+	return configFiles, nil
+}
+
+func bootstrapConfig(c *BootstrapConfig) *envoy_api_bootstrap.Bootstrap {
+	return &envoy_api_bootstrap.Bootstrap{
+		DynamicResources: &envoy_api_bootstrap.Bootstrap_DynamicResources{
 			LdsConfig: ConfigSource("contour"),
 			CdsConfig: ConfigSource("contour"),
 		},
-		StaticResources: &bootstrap.Bootstrap_StaticResources{
+		StaticResources: &envoy_api_bootstrap.Bootstrap_StaticResources{
 			Clusters: []*api.Cluster{{
 				Name:                 "contour",
 				AltStatName:          strings.Join([]string{c.Namespace, "contour", strconv.Itoa(c.xdsGRPCPort())}, "_"),
@@ -87,45 +169,33 @@ func Bootstrap(c *BootstrapConfig) *bootstrap.Bootstrap {
 				},
 			}},
 		},
-		Admin: &bootstrap.Admin{
+		Admin: &envoy_api_bootstrap.Admin{
 			AccessLogPath: c.adminAccessLogPath(),
 			Address:       SocketAddress(c.adminAddress(), c.adminPort()),
 		},
 	}
-
-	if c.GrpcClientCert != "" || c.GrpcClientKey != "" || c.GrpcCABundle != "" {
-		// If one of the two TLS options is not empty, they all must be not empty
-		if !(c.GrpcClientCert != "" && c.GrpcClientKey != "" && c.GrpcCABundle != "") {
-			log.Fatal("You must supply all three TLS parameters - --envoy-cafile, --envoy-cert-file, --envoy-key-file, or none of them.")
-		}
-		b.StaticResources.Clusters[0].TransportSocket = UpstreamTLSTransportSocket(
-			upstreamFileTLSContext(c.GrpcCABundle, c.GrpcClientCert, c.GrpcClientKey),
-		)
-	}
-
-	return b
 }
 
-func upstreamFileTLSContext(cafile, certfile, keyfile string) *envoy_api_v2_auth.UpstreamTlsContext {
+func upstreamFileTLSContext(c *BootstrapConfig) *envoy_api_v2_auth.UpstreamTlsContext {
 	context := &envoy_api_v2_auth.UpstreamTlsContext{
 		CommonTlsContext: &envoy_api_v2_auth.CommonTlsContext{
 			TlsCertificates: []*envoy_api_v2_auth.TlsCertificate{{
 				CertificateChain: &envoy_api_v2_core.DataSource{
 					Specifier: &envoy_api_v2_core.DataSource_Filename{
-						Filename: certfile,
+						Filename: c.GrpcClientCert,
 					},
 				},
 				PrivateKey: &envoy_api_v2_core.DataSource{
 					Specifier: &envoy_api_v2_core.DataSource_Filename{
-						Filename: keyfile,
+						Filename: c.GrpcClientKey,
 					},
 				},
 			}},
 			ValidationContextType: &envoy_api_v2_auth.CommonTlsContext_ValidationContext{
 				ValidationContext: &envoy_api_v2_auth.CertificateValidationContext{
 					TrustedCa: &envoy_api_v2_core.DataSource{
 						Specifier: &envoy_api_v2_core.DataSource_Filename{
-							Filename: cafile,
+							Filename: c.GrpcCABundle,
 						},
 					},
 					// TODO(youngnick): Does there need to be a flag wired down to here?
@@ -141,6 +211,78 @@ func upstreamFileTLSContext(cafile, certfile, keyfile string) *envoy_api_v2_auth
 	return context
 }
 
+func upstreamSdsTLSContext(certificateSdsFile, validationSdsFile string) *envoy_api_v2_auth.UpstreamTlsContext {
+	context := &envoy_api_v2_auth.UpstreamTlsContext{
+		CommonTlsContext: &envoy_api_v2_auth.CommonTlsContext{
+			TlsCertificateSdsSecretConfigs: []*envoy_api_v2_auth.SdsSecretConfig{{
+				SdsConfig: &envoy_api_v2_core.ConfigSource{
+					ConfigSourceSpecifier: &envoy_api_v2_core.ConfigSource_Path{
+						Path: certificateSdsFile,
+					},
+				},
+			}},
+			ValidationContextType: &envoy_api_v2_auth.CommonTlsContext_ValidationContextSdsSecretConfig{
+				ValidationContextSdsSecretConfig: &envoy_api_v2_auth.SdsSecretConfig{
+					SdsConfig: &envoy_api_v2_core.ConfigSource{
+						ConfigSourceSpecifier: &envoy_api_v2_core.ConfigSource_Path{
+							Path: validationSdsFile,
+						},
+					},
+				},
+			},
+		},
+	}
+	return context
+}
+
+// tlsCertificateSdsSecretConfig creates DiscoveryResponse with file based SDS resource
+// including paths to TLS certificates and key
+func tlsCertificateSdsSecretConfig(c *BootstrapConfig) *api.DiscoveryResponse {
+	secret := &envoy_api_v2_auth.Secret{
+		Type: &envoy_api_v2_auth.Secret_TlsCertificate{
+			TlsCertificate: &envoy_api_v2_auth.TlsCertificate{
+				CertificateChain: &envoy_api_v2_core.DataSource{
+					Specifier: &envoy_api_v2_core.DataSource_Filename{
+						Filename: c.GrpcClientCert,
+					},
+				},
+				PrivateKey: &envoy_api_v2_core.DataSource{
+					Specifier: &envoy_api_v2_core.DataSource_Filename{
+						Filename: c.GrpcClientKey,
+					},
+				},
+			},
+		},
+	}
+	return &api.DiscoveryResponse{
+		Resources: []*any.Any{toAny(secret)},
+	}
+}
+
+// validationContextSdsSecretConfig creates DiscoveryResponse with file based SDS resource
+// including path to CA certificate bundle
+func validationContextSdsSecretConfig(c *BootstrapConfig) *api.DiscoveryResponse {
+	secret := &envoy_api_v2_auth.Secret{
+		Type: &envoy_api_v2_auth.Secret_ValidationContext{
+			ValidationContext: &envoy_api_v2_auth.CertificateValidationContext{
+				TrustedCa: &envoy_api_v2_core.DataSource{
+					Specifier: &envoy_api_v2_core.DataSource_Filename{
+						Filename: c.GrpcCABundle,
+					},
+				},
+				MatchSubjectAltNames: []*matcher.StringMatcher{{
+					MatchPattern: &matcher.StringMatcher_Exact{
+						Exact: "contour",
+					}},
+				},
+			},
+		},
+	}
+	return &api.DiscoveryResponse{
+		Resources: []*any.Any{toAny(secret)},
+	}
+}
+
 // BootstrapConfig holds configuration values for a v2.Bootstrap.
 type BootstrapConfig struct {
 	// AdminAccessLogPath is the path to write the access log for the administration server.
@@ -176,6 +318,12 @@ type BootstrapConfig struct {
 
 	// GrpcClientKey is the filename that contains a client key for secure gRPC with TLS.
 	GrpcClientKey string
+
+	// Path is the filename for the bootstrap configuration file to be created.
+	Path string
+
+	// ResourcesDir is the directory where out of line Envoy resources can be placed.
+	ResourcesDir string
 }
 
 func (c *BootstrapConfig) xdsAddress() string   { return stringOrDefault(c.XDSAddress, "127.0.0.1") }
@@ -199,3 +347,22 @@ func intOrDefault(i, def int) int {
 	}
 	return i
 }
+
+func writeConfig(filename string, config proto.Message) (err error) {
+	var out *os.File
+
+	if filename == "-" {
+		out = os.Stdout
+	} else {
+		out, err = os.Create(filename)
+		if err != nil {
+			return
+		}
+		defer func() {
+			err = out.Close()
+		}()
+	}
+
+	m := &jsonpb.Marshaler{OrigName: true}
+	return m.Marshal(out, config)
+}