From f7c8e890a97611f8eff7826db245776d7aa5b2e6 Mon Sep 17 00:00:00 2001 From: Tero Saarni Date: Fri, 15 May 2020 10:44:00 +0300 Subject: [PATCH] bootstrap: support for Envoy xDS certificate rotation (#2333) Added new flag --resources-dir to bootstrap command for supporting path-based SDS resources to define xDS certificate and key. When using the flag, the following changes take place: - Bootstrap file will contain references to SDS resource files instead of direct cert/key paths. - SDS resource file is written into resources dir for xDS client cert and key - SDS resource file is written into resources dir for xDS trusted CA cert With this change there is no need to restart Envoy anymore when certificates are rotated. Envoy will monitor and automatically reload the certificate and key files without causing an interruption to traffic. Any recent version of Envoy supports the above configuration, but Envoy 1.14.1 or later is required for automatic certificate reload. Signed-off-by: Tero Saarni --- cmd/contour/bootstrap.go | 57 ++---- cmd/contour/contour.go | 7 +- internal/envoy/bootstrap.go | 244 +++++++++++++++++++--- internal/envoy/bootstrap_test.go | 334 ++++++++++++++++++++++++++----- 4 files changed, 520 insertions(+), 122 deletions(-) diff --git a/cmd/contour/bootstrap.go b/cmd/contour/bootstrap.go index 67f07eee5d5..66a178a96bd 100644 --- a/cmd/contour/bootstrap.go +++ b/cmd/contour/bootstrap.go @@ -14,56 +14,25 @@ package main import ( - "io" - "os" - - "github.com/golang/protobuf/jsonpb" "github.com/projectcontour/contour/internal/envoy" kingpin "gopkg.in/alecthomas/kingpin.v2" ) // registerBootstrap registers the bootstrap subcommand and flags // with the Application provided. -func registerBootstrap(app *kingpin.Application) (*kingpin.CmdClause, *bootstrapContext) { - var ctx bootstrapContext +func registerBootstrap(app *kingpin.Application) (*kingpin.CmdClause, *envoy.BootstrapConfig) { + var config envoy.BootstrapConfig bootstrap := app.Command("bootstrap", "Generate bootstrap configuration.") - bootstrap.Arg("path", "Configuration file ('-' for standard output).").Required().StringVar(&ctx.path) - bootstrap.Flag("admin-address", "Envoy admin interface address.").StringVar(&ctx.config.AdminAddress) - bootstrap.Flag("admin-port", "Envoy admin interface port.").IntVar(&ctx.config.AdminPort) - bootstrap.Flag("xds-address", "xDS gRPC API address.").StringVar(&ctx.config.XDSAddress) - bootstrap.Flag("xds-port", "xDS gRPC API port.").IntVar(&ctx.config.XDSGRPCPort) - bootstrap.Flag("envoy-cafile", "gRPC CA Filename for Envoy to load.").Envar("ENVOY_CAFILE").StringVar(&ctx.config.GrpcCABundle) - bootstrap.Flag("envoy-cert-file", "gRPC Client cert filename for Envoy to load.").Envar("ENVOY_CERT_FILE").StringVar(&ctx.config.GrpcClientCert) - bootstrap.Flag("envoy-key-file", "gRPC Client key filename for Envoy to load.").Envar("ENVOY_KEY_FILE").StringVar(&ctx.config.GrpcClientKey) - bootstrap.Flag("namespace", "The namespace the Envoy container will run in.").Envar("CONTOUR_NAMESPACE").Default("projectcontour").StringVar(&ctx.config.Namespace) - return bootstrap, &ctx -} - -type bootstrapContext struct { - config envoy.BootstrapConfig - path string -} - -// doBootstrap writes an Envoy bootstrap configuration file to the supplied path. -func doBootstrap(ctx *bootstrapContext) { - var out io.Writer - - switch ctx.path { - case "-": - out = os.Stdout - default: - f, err := os.Create(ctx.path) - check(err) - - out = f - - defer func() { - check(f.Close()) - }() - } - - m := &jsonpb.Marshaler{OrigName: true} - - check(m.Marshal(out, envoy.Bootstrap(&ctx.config))) + bootstrap.Arg("path", "Configuration file ('-' for standard output).").Required().StringVar(&config.Path) + bootstrap.Flag("resources-dir", "Directory where configuration files will be written to.").StringVar(&config.ResourcesDir) + bootstrap.Flag("admin-address", "Envoy admin interface address.").StringVar(&config.AdminAddress) + bootstrap.Flag("admin-port", "Envoy admin interface port.").IntVar(&config.AdminPort) + bootstrap.Flag("xds-address", "xDS gRPC API address.").StringVar(&config.XDSAddress) + bootstrap.Flag("xds-port", "xDS gRPC API port.").IntVar(&config.XDSGRPCPort) + bootstrap.Flag("envoy-cafile", "gRPC CA Filename for Envoy to load.").Envar("ENVOY_CAFILE").StringVar(&config.GrpcCABundle) + bootstrap.Flag("envoy-cert-file", "gRPC Client cert filename for Envoy to load.").Envar("ENVOY_CERT_FILE").StringVar(&config.GrpcClientCert) + bootstrap.Flag("envoy-key-file", "gRPC Client key filename for Envoy to load.").Envar("ENVOY_KEY_FILE").StringVar(&config.GrpcClientKey) + bootstrap.Flag("namespace", "The namespace the Envoy container will run in.").Envar("CONTOUR_NAMESPACE").Default("projectcontour").StringVar(&config.Namespace) + return bootstrap, &config } diff --git a/cmd/contour/contour.go b/cmd/contour/contour.go index 72fb520d374..33233725d95 100644 --- a/cmd/contour/contour.go +++ b/cmd/contour/contour.go @@ -19,6 +19,7 @@ import ( resource "github.com/envoyproxy/go-control-plane/pkg/resource/v2" "github.com/projectcontour/contour/internal/build" + "github.com/projectcontour/contour/internal/envoy" "github.com/sirupsen/logrus" kingpin "gopkg.in/alecthomas/kingpin.v2" "k8s.io/klog" @@ -36,8 +37,8 @@ func main() { log := logrus.StandardLogger() app := kingpin.New("contour", "Contour Kubernetes ingress controller.") - envoy := app.Command("envoy", "Sub-command for envoy actions.") - shutdownManager, shutdownManagerCtx := registerShutdownManager(envoy, log) + envoyCmd := app.Command("envoy", "Sub-command for envoy actions.") + shutdownManager, shutdownManagerCtx := registerShutdownManager(envoyCmd, log) bootstrap, bootstrapCtx := registerBootstrap(app) certgenApp, certgenConfig := registerCertGen(app) @@ -69,7 +70,7 @@ func main() { case shutdownManager.FullCommand(): doShutdownManager(shutdownManagerCtx) case bootstrap.FullCommand(): - doBootstrap(bootstrapCtx) + check(envoy.WriteBootstrap(bootstrapCtx)) case certgenApp.FullCommand(): doCertgen(certgenConfig) case cds.FullCommand(): diff --git a/internal/envoy/bootstrap.go b/internal/envoy/bootstrap.go index 7bee3251e98..6724ac9f25b 100644 --- a/internal/envoy/bootstrap.go +++ b/internal/envoy/bootstrap.go @@ -16,7 +16,9 @@ package envoy import ( - "log" + "fmt" + "os" + "path" "strconv" "strings" "time" @@ -25,19 +27,126 @@ import ( envoy_api_v2_auth "github.com/envoyproxy/go-control-plane/envoy/api/v2/auth" clusterv2 "github.com/envoyproxy/go-control-plane/envoy/api/v2/cluster" envoy_api_v2_core "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" - bootstrap "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v2" + envoy_api_bootstrap "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v2" matcher "github.com/envoyproxy/go-control-plane/envoy/type/matcher" + "github.com/golang/protobuf/jsonpb" + "github.com/golang/protobuf/proto" + "github.com/golang/protobuf/ptypes/any" "github.com/projectcontour/contour/internal/protobuf" ) -// Bootstrap creates a new v2 Bootstrap configuration. -func Bootstrap(c *BootstrapConfig) *bootstrap.Bootstrap { - b := &bootstrap.Bootstrap{ - DynamicResources: &bootstrap.Bootstrap_DynamicResources{ +// sdsResourcesSubdirectory stores the subdirectory name where SDS path resources are stored to. +const sdsResourcesSubdirectory = "sds" + +// sdsTLSCertificateFile stores the path to the SDS resource with Envoy's +// client certificate and key for XDS gRPC connection. +const sdsTLSCertificateFile = "xds-tls-certicate.json" + +// sdsValidationContextFile stores the path to the SDS resource with +// CA certificates for Envoy to use for the XDS gRPC connection. +const sdsValidationContextFile = "xds-validation-context.json" + +// ConfigFiles is a map from configuration filename to configuration file content. +type ConfigFiles map[string]proto.Message + +// WriteBootstrap writes bootstrap configuration to files. +func WriteBootstrap(c *BootstrapConfig) error { + // Create Envoy bootstrap config and associated resource files. + steps, err := bootstrap(c) + if err != nil { + return err + } + + if c.ResourcesDir != "" { + if err := os.MkdirAll(path.Join(c.ResourcesDir, "sds"), 0755); err != nil { + return err + } + } + + // Write all configuration files out to filesystem. + for _, step := range steps { + if err := writeConfig(step(c)); err != nil { + return err + } + } + + return nil +} + +type bootstrapf func(*BootstrapConfig) (string, proto.Message) + +// bootstrap creates a new v2 bootstrap configuration and associated resource files. +func bootstrap(c *BootstrapConfig) ([]bootstrapf, error) { + steps := []bootstrapf{} + + if c.GrpcClientCert == "" && c.GrpcClientKey == "" && c.GrpcCABundle == "" { + steps = append(steps, + func(*BootstrapConfig) (string, proto.Message) { + return c.Path, bootstrapConfig(c) + }) + + return steps, nil + } + + // If one of the two TLS options is not empty, they all must be not empty. + if c.GrpcClientCert == "" || c.GrpcClientKey == "" || c.GrpcCABundle == "" { + return nil, fmt.Errorf("you must supply all TLS parameters - %q, %q, %q, or none of them", + "--envoy-cafile", "--envoy-cert-file", "--envoy-key-file") + } + + if c.ResourcesDir == "" { + // For backwards compatibility, the old behavior + // is to use direct certificate and key file paths in + // bootstrap config. Envoy does not support rotation + // of xDS certificate files in this case. + + steps = append(steps, + func(*BootstrapConfig) (string, proto.Message) { + b := bootstrapConfig(c) + b.StaticResources.Clusters[0].TransportSocket = UpstreamTLSTransportSocket( + upstreamFileTLSContext(c)) + return c.Path, b + }) + + return steps, nil + } + + // xDS certificate rotation is supported by Envoy by using SDS path based resource files. + // These files are JSON representation of the SDS protobuf messages that normally get sent over the xDS connection, + // but for xDS connection itself, bootstrapping is done by storing the SDS resources in a local filesystem. + // Envoy will monitor and reload the resource files and the certificate and key files referred from the SDS resources. + // + // Two files are written to ResourcesDir: + // - SDS resource for xDS client certificate and key for authenticating Envoy towards Contour. + // - SDS resource for trusted CA certificate for validating Contour server certificate. + sdsTLSCertificatePath := path.Join(c.ResourcesDir, sdsResourcesSubdirectory, sdsTLSCertificateFile) + sdsValidationContextPath := path.Join(c.ResourcesDir, sdsResourcesSubdirectory, sdsValidationContextFile) + + steps = append(steps, + func(*BootstrapConfig) (string, proto.Message) { + return sdsTLSCertificatePath, tlsCertificateSdsSecretConfig(c) + }, + func(*BootstrapConfig) (string, proto.Message) { + return sdsValidationContextPath, validationContextSdsSecretConfig(c) + }, + func(*BootstrapConfig) (string, proto.Message) { + b := bootstrapConfig(c) + b.StaticResources.Clusters[0].TransportSocket = UpstreamTLSTransportSocket( + upstreamSdsTLSContext(sdsTLSCertificatePath, sdsValidationContextPath)) + return c.Path, b + }, + ) + + return steps, nil +} + +func bootstrapConfig(c *BootstrapConfig) *envoy_api_bootstrap.Bootstrap { + return &envoy_api_bootstrap.Bootstrap{ + DynamicResources: &envoy_api_bootstrap.Bootstrap_DynamicResources{ LdsConfig: ConfigSource("contour"), CdsConfig: ConfigSource("contour"), }, - StaticResources: &bootstrap.Bootstrap_StaticResources{ + StaticResources: &envoy_api_bootstrap.Bootstrap_StaticResources{ Clusters: []*api.Cluster{{ Name: "contour", AltStatName: strings.Join([]string{c.Namespace, "contour", strconv.Itoa(c.xdsGRPCPort())}, "_"), @@ -87,37 +196,25 @@ func Bootstrap(c *BootstrapConfig) *bootstrap.Bootstrap { }, }}, }, - Admin: &bootstrap.Admin{ + Admin: &envoy_api_bootstrap.Admin{ AccessLogPath: c.adminAccessLogPath(), Address: SocketAddress(c.adminAddress(), c.adminPort()), }, } - - if c.GrpcClientCert != "" || c.GrpcClientKey != "" || c.GrpcCABundle != "" { - // If one of the two TLS options is not empty, they all must be not empty - if !(c.GrpcClientCert != "" && c.GrpcClientKey != "" && c.GrpcCABundle != "") { - log.Fatal("You must supply all three TLS parameters - --envoy-cafile, --envoy-cert-file, --envoy-key-file, or none of them.") - } - b.StaticResources.Clusters[0].TransportSocket = UpstreamTLSTransportSocket( - upstreamFileTLSContext(c.GrpcCABundle, c.GrpcClientCert, c.GrpcClientKey), - ) - } - - return b } -func upstreamFileTLSContext(cafile, certfile, keyfile string) *envoy_api_v2_auth.UpstreamTlsContext { +func upstreamFileTLSContext(c *BootstrapConfig) *envoy_api_v2_auth.UpstreamTlsContext { context := &envoy_api_v2_auth.UpstreamTlsContext{ CommonTlsContext: &envoy_api_v2_auth.CommonTlsContext{ TlsCertificates: []*envoy_api_v2_auth.TlsCertificate{{ CertificateChain: &envoy_api_v2_core.DataSource{ Specifier: &envoy_api_v2_core.DataSource_Filename{ - Filename: certfile, + Filename: c.GrpcClientCert, }, }, PrivateKey: &envoy_api_v2_core.DataSource{ Specifier: &envoy_api_v2_core.DataSource_Filename{ - Filename: keyfile, + Filename: c.GrpcClientKey, }, }, }}, @@ -125,7 +222,7 @@ func upstreamFileTLSContext(cafile, certfile, keyfile string) *envoy_api_v2_auth ValidationContext: &envoy_api_v2_auth.CertificateValidationContext{ TrustedCa: &envoy_api_v2_core.DataSource{ Specifier: &envoy_api_v2_core.DataSource_Filename{ - Filename: cafile, + Filename: c.GrpcCABundle, }, }, // TODO(youngnick): Does there need to be a flag wired down to here? @@ -141,6 +238,80 @@ func upstreamFileTLSContext(cafile, certfile, keyfile string) *envoy_api_v2_auth return context } +func upstreamSdsTLSContext(certificateSdsFile, validationSdsFile string) *envoy_api_v2_auth.UpstreamTlsContext { + context := &envoy_api_v2_auth.UpstreamTlsContext{ + CommonTlsContext: &envoy_api_v2_auth.CommonTlsContext{ + TlsCertificateSdsSecretConfigs: []*envoy_api_v2_auth.SdsSecretConfig{{ + SdsConfig: &envoy_api_v2_core.ConfigSource{ + ConfigSourceSpecifier: &envoy_api_v2_core.ConfigSource_Path{ + Path: certificateSdsFile, + }, + }, + }}, + ValidationContextType: &envoy_api_v2_auth.CommonTlsContext_ValidationContextSdsSecretConfig{ + ValidationContextSdsSecretConfig: &envoy_api_v2_auth.SdsSecretConfig{ + SdsConfig: &envoy_api_v2_core.ConfigSource{ + ConfigSourceSpecifier: &envoy_api_v2_core.ConfigSource_Path{ + Path: validationSdsFile, + }, + }, + }, + }, + }, + } + return context +} + +// tlsCertificateSdsSecretConfig creates DiscoveryResponse with file based SDS resource +// including paths to TLS certificates and key +func tlsCertificateSdsSecretConfig(c *BootstrapConfig) *api.DiscoveryResponse { + secret := &envoy_api_v2_auth.Secret{ + Type: &envoy_api_v2_auth.Secret_TlsCertificate{ + TlsCertificate: &envoy_api_v2_auth.TlsCertificate{ + CertificateChain: &envoy_api_v2_core.DataSource{ + Specifier: &envoy_api_v2_core.DataSource_Filename{ + Filename: c.GrpcClientCert, + }, + }, + PrivateKey: &envoy_api_v2_core.DataSource{ + Specifier: &envoy_api_v2_core.DataSource_Filename{ + Filename: c.GrpcClientKey, + }, + }, + }, + }, + } + + return &api.DiscoveryResponse{ + Resources: []*any.Any{toAny(secret)}, + } +} + +// validationContextSdsSecretConfig creates DiscoveryResponse with file based SDS resource +// including path to CA certificate bundle +func validationContextSdsSecretConfig(c *BootstrapConfig) *api.DiscoveryResponse { + secret := &envoy_api_v2_auth.Secret{ + Type: &envoy_api_v2_auth.Secret_ValidationContext{ + ValidationContext: &envoy_api_v2_auth.CertificateValidationContext{ + TrustedCa: &envoy_api_v2_core.DataSource{ + Specifier: &envoy_api_v2_core.DataSource_Filename{ + Filename: c.GrpcCABundle, + }, + }, + MatchSubjectAltNames: []*matcher.StringMatcher{{ + MatchPattern: &matcher.StringMatcher_Exact{ + Exact: "contour", + }}, + }, + }, + }, + } + + return &api.DiscoveryResponse{ + Resources: []*any.Any{toAny(secret)}, + } +} + // BootstrapConfig holds configuration values for a v2.Bootstrap. type BootstrapConfig struct { // AdminAccessLogPath is the path to write the access log for the administration server. @@ -176,6 +347,12 @@ type BootstrapConfig struct { // GrpcClientKey is the filename that contains a client key for secure gRPC with TLS. GrpcClientKey string + + // Path is the filename for the bootstrap configuration file to be created. + Path string + + // ResourcesDir is the directory where out of line Envoy resources can be placed. + ResourcesDir string } func (c *BootstrapConfig) xdsAddress() string { return stringOrDefault(c.XDSAddress, "127.0.0.1") } @@ -199,3 +376,22 @@ func intOrDefault(i, def int) int { } return i } + +func writeConfig(filename string, config proto.Message) (err error) { + var out *os.File + + if filename == "-" { + out = os.Stdout + } else { + out, err = os.Create(filename) + if err != nil { + return + } + defer func() { + err = out.Close() + }() + } + + m := &jsonpb.Marshaler{OrigName: true} + return m.Marshal(out, config) +} diff --git a/internal/envoy/bootstrap_test.go b/internal/envoy/bootstrap_test.go index 9e25142afa6..e872d64fa7b 100644 --- a/internal/envoy/bootstrap_test.go +++ b/internal/envoy/bootstrap_test.go @@ -14,9 +14,11 @@ package envoy import ( + "path" "testing" - bootstrap "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v2" + api "github.com/envoyproxy/go-control-plane/envoy/api/v2" + envoy_api_bootstrap "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v2" "github.com/golang/protobuf/jsonpb" "github.com/golang/protobuf/proto" "github.com/projectcontour/contour/internal/assert" @@ -24,12 +26,17 @@ import ( func TestBootstrap(t *testing.T) { tests := map[string]struct { - config BootstrapConfig - want string + config BootstrapConfig + wantedBootstrapConfig string + wantedTLSCertificateConfig string + wantedValidationContextConfig string + wantedError bool }{ "default configuration": { - config: BootstrapConfig{Namespace: "testing-ns"}, - want: `{ + config: BootstrapConfig{ + Path: "envoy.json", + Namespace: "testing-ns"}, + wantedBootstrapConfig: `{ "static_resources": { "clusters": [ { @@ -89,8 +96,8 @@ func TestBootstrap(t *testing.T) { "connect_timeout": "0.250s", "load_assignment": { "cluster_name": "service-stats", - "endpoints": [ - { + "endpoints": [ + { "lb_endpoints": [ { "endpoint": { @@ -98,11 +105,11 @@ func TestBootstrap(t *testing.T) { "socket_address": { "address": "127.0.0.1", "port_value": 9001 - } - } + } + } } - } - ] + } + ] } ] } @@ -148,11 +155,12 @@ func TestBootstrap(t *testing.T) { }, "--admin-address=8.8.8.8 --admin-port=9200": { config: BootstrapConfig{ + Path: "envoy.json", AdminAddress: "8.8.8.8", AdminPort: 9200, Namespace: "testing-ns", }, - want: `{ + wantedBootstrapConfig: `{ "static_resources": { "clusters": [ { @@ -212,8 +220,8 @@ func TestBootstrap(t *testing.T) { "connect_timeout": "0.250s", "load_assignment": { "cluster_name": "service-stats", - "endpoints": [ - { + "endpoints": [ + { "lb_endpoints": [ { "endpoint": { @@ -221,11 +229,11 @@ func TestBootstrap(t *testing.T) { "socket_address": { "address": "8.8.8.8", "port_value": 9200 - } - } + } + } } - } - ] + } + ] } ] } @@ -271,10 +279,11 @@ func TestBootstrap(t *testing.T) { }, "AdminAccessLogPath": { // TODO(dfc) doesn't appear to be exposed via contour bootstrap config: BootstrapConfig{ + Path: "envoy.json", AdminAccessLogPath: "/var/log/admin.log", Namespace: "testing-ns", }, - want: `{ + wantedBootstrapConfig: `{ "static_resources": { "clusters": [ { @@ -334,8 +343,8 @@ func TestBootstrap(t *testing.T) { "connect_timeout": "0.250s", "load_assignment": { "cluster_name": "service-stats", - "endpoints": [ - { + "endpoints": [ + { "lb_endpoints": [ { "endpoint": { @@ -343,11 +352,11 @@ func TestBootstrap(t *testing.T) { "socket_address": { "address": "127.0.0.1", "port_value": 9001 - } - } + } + } } - } - ] + } + ] } ] } @@ -393,11 +402,12 @@ func TestBootstrap(t *testing.T) { }, "--xds-address=8.8.8.8 --xds-port=9200": { config: BootstrapConfig{ + Path: "envoy.json", XDSAddress: "8.8.8.8", XDSGRPCPort: 9200, Namespace: "testing-ns", }, - want: `{ + wantedBootstrapConfig: `{ "static_resources": { "clusters": [ { @@ -457,8 +467,8 @@ func TestBootstrap(t *testing.T) { "connect_timeout": "0.250s", "load_assignment": { "cluster_name": "service-stats", - "endpoints": [ - { + "endpoints": [ + { "lb_endpoints": [ { "endpoint": { @@ -466,11 +476,11 @@ func TestBootstrap(t *testing.T) { "socket_address": { "address": "127.0.0.1", "port_value": 9001 - } - } + } + } } - } - ] + } + ] } ] } @@ -516,9 +526,10 @@ func TestBootstrap(t *testing.T) { }, "--stats-address=8.8.8.8 --stats-port=9200": { config: BootstrapConfig{ + Path: "envoy.json", Namespace: "testing-ns", }, - want: `{ + wantedBootstrapConfig: `{ "static_resources": { "clusters": [ { @@ -578,8 +589,8 @@ func TestBootstrap(t *testing.T) { "connect_timeout": "0.250s", "load_assignment": { "cluster_name": "service-stats", - "endpoints": [ - { + "endpoints": [ + { "lb_endpoints": [ { "endpoint": { @@ -587,11 +598,11 @@ func TestBootstrap(t *testing.T) { "socket_address": { "address": "127.0.0.1", "port_value": 9001 - } - } + } + } } - } - ] + } + ] } ] } @@ -637,12 +648,13 @@ func TestBootstrap(t *testing.T) { }, "--envoy-cafile=CA.cert --envoy-client-cert=client.cert --envoy-client-key=client.key": { config: BootstrapConfig{ + Path: "envoy.json", Namespace: "testing-ns", GrpcCABundle: "CA.cert", GrpcClientCert: "client.cert", GrpcClientKey: "client.key", }, - want: `{ + wantedBootstrapConfig: `{ "static_resources": { "clusters": [ { @@ -730,8 +742,8 @@ func TestBootstrap(t *testing.T) { "connect_timeout": "0.250s", "load_assignment": { "cluster_name": "service-stats", - "endpoints": [ - { + "endpoints": [ + { "lb_endpoints": [ { "endpoint": { @@ -739,11 +751,11 @@ func TestBootstrap(t *testing.T) { "socket_address": { "address": "127.0.0.1", "port_value": 9001 - } - } + } + } } - } - ] + } + ] } ] } @@ -787,13 +799,233 @@ func TestBootstrap(t *testing.T) { } }`, }, - } + "--resources-dir tmp --envoy-cafile=CA.cert --envoy-client-cert=client.cert --envoy-client-key=client.key": { + config: BootstrapConfig{ + Path: "envoy.json", + Namespace: "testing-ns", + ResourcesDir: "resources", + GrpcCABundle: "CA.cert", + GrpcClientCert: "client.cert", + GrpcClientKey: "client.key", + }, + wantedBootstrapConfig: `{ + "static_resources": { + "clusters": [ + { + "name": "contour", + "alt_stat_name": "testing-ns_contour_8001", + "type": "STRICT_DNS", + "connect_timeout": "5s", + "load_assignment": { + "cluster_name": "contour", + "endpoints": [ + { + "lb_endpoints": [ + { + "endpoint": { + "address": { + "socket_address": { + "address": "127.0.0.1", + "port_value": 8001 + } + } + } + } + ] + } + ] + }, + "circuit_breakers": { + "thresholds": [ + { + "priority": "HIGH", + "max_connections": 100000, + "max_pending_requests": 100000, + "max_requests": 60000000, + "max_retries": 50 + }, + { + "max_connections": 100000, + "max_pending_requests": 100000, + "max_requests": 60000000, + "max_retries": 50 + } + ] + }, + "http2_protocol_options": {}, + "transport_socket": { + "name": "envoy.transport_sockets.tls", + "typed_config": { + "@type": "type.googleapis.com/envoy.api.v2.auth.UpstreamTlsContext", + "common_tls_context": { + "tls_certificate_sds_secret_configs": [ + { + "sds_config": { + "path": "resources/sds/xds-tls-certicate.json" + } + } + ], + "validation_context_sds_secret_config": { + "sds_config": { + "path": "resources/sds/xds-validation-context.json" + } + } + } + } + }, + "upstream_connection_options": { + "tcp_keepalive": { + "keepalive_probes": 3, + "keepalive_time": 30, + "keepalive_interval": 5 + } + } + }, + { + "name": "service-stats", + "alt_stat_name": "testing-ns_service-stats_9001", + "type": "LOGICAL_DNS", + "connect_timeout": "0.250s", + "load_assignment": { + "cluster_name": "service-stats", + "endpoints": [ + { + "lb_endpoints": [ + { + "endpoint": { + "address": { + "socket_address": { + "address": "127.0.0.1", + "port_value": 9001 + } + } + } + } + ] + } + ] + } + } + ] + }, + "dynamic_resources": { + "lds_config": { + "api_config_source": { + "api_type": "GRPC", + "grpc_services": [ + { + "envoy_grpc": { + "cluster_name": "contour" + } + } + ] + } + }, + "cds_config": { + "api_config_source": { + "api_type": "GRPC", + "grpc_services": [ + { + "envoy_grpc": { + "cluster_name": "contour" + } + } + ] + } + } + }, + "admin": { + "access_log_path": "/dev/null", + "address": { + "socket_address": { + "address": "127.0.0.1", + "port_value": 9001 + } + } + } + }`, + wantedTLSCertificateConfig: `{ + "resources": [ + { + "@type": "type.googleapis.com/envoy.api.v2.auth.Secret", + "tls_certificate": { + "certificate_chain": { + "filename": "client.cert" + }, + "private_key": { + "filename": "client.key" + } + } + } + ] + }`, + wantedValidationContextConfig: `{ + "resources": [ + { + "@type": "type.googleapis.com/envoy.api.v2.auth.Secret", + "validation_context": { + "trusted_ca": { + "filename": "CA.cert" + }, + "match_subject_alt_names": [ + { + "exact": "contour" + } + ] + } + } + ] + }`, + }, + "return error when not providing all certificate related parameters": { + config: BootstrapConfig{ + Path: "envoy.json", + Namespace: "testing-ns", + ResourcesDir: "resources", + GrpcClientCert: "client.cert", + GrpcClientKey: "client.key", + }, + wantedError: true, + }} + for name, tc := range tests { t.Run(name, func(t *testing.T) { - got := Bootstrap(&tc.config) - want := new(bootstrap.Bootstrap) - unmarshal(t, tc.want, want) - assert.Equal(t, want, got) + steps, gotError := bootstrap(&tc.config) + assert.Equal(t, gotError != nil, tc.wantedError) + + gotConfigs := map[string]proto.Message{} + for _, step := range steps { + path, config := step(&tc.config) + gotConfigs[path] = config + } + + sdsTLSCertificatePath := path.Join(tc.config.ResourcesDir, sdsResourcesSubdirectory, sdsTLSCertificateFile) + sdsValidationContextPath := path.Join(tc.config.ResourcesDir, sdsResourcesSubdirectory, sdsValidationContextFile) + + if tc.wantedBootstrapConfig != "" { + want := new(envoy_api_bootstrap.Bootstrap) + unmarshal(t, tc.wantedBootstrapConfig, want) + assert.Equal(t, want, gotConfigs[tc.config.Path]) + delete(gotConfigs, tc.config.Path) + } + + if tc.wantedTLSCertificateConfig != "" { + want := new(api.DiscoveryResponse) + unmarshal(t, tc.wantedTLSCertificateConfig, want) + assert.Equal(t, want, gotConfigs[sdsTLSCertificatePath]) + delete(gotConfigs, sdsTLSCertificatePath) + } + + if tc.wantedValidationContextConfig != "" { + want := new(api.DiscoveryResponse) + unmarshal(t, tc.wantedValidationContextConfig, want) + assert.Equal(t, want, gotConfigs[sdsValidationContextPath]) + delete(gotConfigs, sdsValidationContextPath) + } + + if len(gotConfigs) > 0 { + t.Fatalf("got more configs than wanted: %s", gotConfigs) + } }) } }