bootstrap: support for Envoy xDS certificate rotation (#2333)

Added new flag --resources-dir to bootstrap command for supporting
path-based SDS resources to define xDS certificate and key.  When
using the flag, the following changes take place:

- Bootstrap file will contain references to SDS resource files instead of
  direct cert/key paths.
- SDS resource file is written into resources dir for xDS client cert and key
- SDS resource file is written into resources dir for xDS trusted CA cert

With this change there is no need to restart Envoy anymore when certificates
are rotated.  Envoy will monitor and automatically reload the certificate and
key files without causing an interruption to traffic.

Any recent version of Envoy supports the above configuration, but Envoy 1.14.1
or later is required for automatic certificate reload.

Signed-off-by: Tero Saarni <[email protected]>

cmd/contour/bootstrap.go

@@ -14,56 +14,25 @@
 package main
 
 import (
-	"io"
-	"os"
-
-	"github.com/golang/protobuf/jsonpb"
 	"github.com/projectcontour/contour/internal/envoy"
 	kingpin "gopkg.in/alecthomas/kingpin.v2"
 )
 
 // registerBootstrap registers the bootstrap subcommand and flags
 // with the Application provided.
-func registerBootstrap(app *kingpin.Application) (*kingpin.CmdClause, *bootstrapContext) {
-	var ctx bootstrapContext
+func registerBootstrap(app *kingpin.Application) (*kingpin.CmdClause, *envoy.BootstrapConfig) {
+	var config envoy.BootstrapConfig
 
 	bootstrap := app.Command("bootstrap", "Generate bootstrap configuration.")
-	bootstrap.Arg("path", "Configuration file ('-' for standard output).").Required().StringVar(&ctx.path)
-	bootstrap.Flag("admin-address", "Envoy admin interface address.").StringVar(&ctx.config.AdminAddress)
-	bootstrap.Flag("admin-port", "Envoy admin interface port.").IntVar(&ctx.config.AdminPort)
-	bootstrap.Flag("xds-address", "xDS gRPC API address.").StringVar(&ctx.config.XDSAddress)
-	bootstrap.Flag("xds-port", "xDS gRPC API port.").IntVar(&ctx.config.XDSGRPCPort)
-	bootstrap.Flag("envoy-cafile", "gRPC CA Filename for Envoy to load.").Envar("ENVOY_CAFILE").StringVar(&ctx.config.GrpcCABundle)
-	bootstrap.Flag("envoy-cert-file", "gRPC Client cert filename for Envoy to load.").Envar("ENVOY_CERT_FILE").StringVar(&ctx.config.GrpcClientCert)
-	bootstrap.Flag("envoy-key-file", "gRPC Client key filename for Envoy to load.").Envar("ENVOY_KEY_FILE").StringVar(&ctx.config.GrpcClientKey)
-	bootstrap.Flag("namespace", "The namespace the Envoy container will run in.").Envar("CONTOUR_NAMESPACE").Default("projectcontour").StringVar(&ctx.config.Namespace)
-	return bootstrap, &ctx
-}
-
-type bootstrapContext struct {
-	config envoy.BootstrapConfig
-	path   string
-}
-
-// doBootstrap writes an Envoy bootstrap configuration file to the supplied path.
-func doBootstrap(ctx *bootstrapContext) {
-	var out io.Writer
-
-	switch ctx.path {
-	case "-":
-		out = os.Stdout
-	default:
-		f, err := os.Create(ctx.path)
-		check(err)
-
-		out = f
-
-		defer func() {
-			check(f.Close())
-		}()
-	}
-
-	m := &jsonpb.Marshaler{OrigName: true}
-
-	check(m.Marshal(out, envoy.Bootstrap(&ctx.config)))
+	bootstrap.Arg("path", "Configuration file ('-' for standard output).").Required().StringVar(&config.Path)
+	bootstrap.Flag("resources-dir", "Directory where configuration files will be written to.").StringVar(&config.ResourcesDir)
+	bootstrap.Flag("admin-address", "Envoy admin interface address.").StringVar(&config.AdminAddress)
+	bootstrap.Flag("admin-port", "Envoy admin interface port.").IntVar(&config.AdminPort)
+	bootstrap.Flag("xds-address", "xDS gRPC API address.").StringVar(&config.XDSAddress)
+	bootstrap.Flag("xds-port", "xDS gRPC API port.").IntVar(&config.XDSGRPCPort)
+	bootstrap.Flag("envoy-cafile", "gRPC CA Filename for Envoy to load.").Envar("ENVOY_CAFILE").StringVar(&config.GrpcCABundle)
+	bootstrap.Flag("envoy-cert-file", "gRPC Client cert filename for Envoy to load.").Envar("ENVOY_CERT_FILE").StringVar(&config.GrpcClientCert)
+	bootstrap.Flag("envoy-key-file", "gRPC Client key filename for Envoy to load.").Envar("ENVOY_KEY_FILE").StringVar(&config.GrpcClientKey)
+	bootstrap.Flag("namespace", "The namespace the Envoy container will run in.").Envar("CONTOUR_NAMESPACE").Default("projectcontour").StringVar(&config.Namespace)
+	return bootstrap, &config
 }

cmd/contour/contour.go

@@ -19,6 +19,7 @@ import (
 
 	resource "github.com/envoyproxy/go-control-plane/pkg/resource/v2"
 	"github.com/projectcontour/contour/internal/build"
+	"github.com/projectcontour/contour/internal/envoy"
 	"github.com/sirupsen/logrus"
 	kingpin "gopkg.in/alecthomas/kingpin.v2"
 	"k8s.io/klog"
@@ -36,8 +37,8 @@ func main() {
 	log := logrus.StandardLogger()
 	app := kingpin.New("contour", "Contour Kubernetes ingress controller.")
 
-	envoy := app.Command("envoy", "Sub-command for envoy actions.")
-	shutdownManager, shutdownManagerCtx := registerShutdownManager(envoy, log)
+	envoyCmd := app.Command("envoy", "Sub-command for envoy actions.")
+	shutdownManager, shutdownManagerCtx := registerShutdownManager(envoyCmd, log)
 
 	bootstrap, bootstrapCtx := registerBootstrap(app)
 	certgenApp, certgenConfig := registerCertGen(app)
@@ -69,7 +70,7 @@ func main() {
 	case shutdownManager.FullCommand():
 		doShutdownManager(shutdownManagerCtx)
 	case bootstrap.FullCommand():
-		doBootstrap(bootstrapCtx)
+		check(envoy.WriteBootstrap(bootstrapCtx))
 	case certgenApp.FullCommand():
 		doCertgen(certgenConfig)
 	case cds.FullCommand():