Skip to content

Commit

Permalink
bootstrap: use path-based SDS resources for xDS cert and key
Browse files Browse the repository at this point in the history 
Replaced direct certificate/key path with path-based SDS resources in Envoy
bootstrap configuration. This allows certificate rotation: Envoy will watch and
reload the certificates and key associated to the SDS resources without need
to restart Envoy.

Signed-off-by: Tero Saarni <[email protected]>
  • Loading branch information
@tsaarni
tsaarni committed Apr 12, 2020
1 parent 6c444d4 commit c2abce8
Show file tree
Hide file tree
Showing 2 changed files with 159 additions and 47 deletions.
58 changes: 16 additions & 42 deletions cmd/contour/bootstrap.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,56 +14,30 @@
package main

import (
"io"
"os"

"github.com/golang/protobuf/jsonpb"
"github.com/projectcontour/contour/internal/envoy"
kingpin "gopkg.in/alecthomas/kingpin.v2"
)

// registerBootstrap registers the bootstrap subcommand and flags
// with the Application provided.
func registerBootstrap(app *kingpin.Application) (*kingpin.CmdClause, *bootstrapContext) {
var ctx bootstrapContext
func registerBootstrap(app *kingpin.Application) (*kingpin.CmdClause, *envoy.BootstrapConfig) {
var config envoy.BootstrapConfig

bootstrap := app.Command("bootstrap", "Generate bootstrap configuration.")
bootstrap.Arg("path", "Configuration file ('-' for standard output).").Required().StringVar(&ctx.path)
bootstrap.Flag("admin-address", "Envoy admin interface address.").StringVar(&ctx.config.AdminAddress)
bootstrap.Flag("admin-port", "Envoy admin interface port.").IntVar(&ctx.config.AdminPort)
bootstrap.Flag("xds-address", "xDS gRPC API address.").StringVar(&ctx.config.XDSAddress)
bootstrap.Flag("xds-port", "xDS gRPC API port.").IntVar(&ctx.config.XDSGRPCPort)
bootstrap.Flag("envoy-cafile", "gRPC CA Filename for Envoy to load.").Envar("ENVOY_CAFILE").StringVar(&ctx.config.GrpcCABundle)
bootstrap.Flag("envoy-cert-file", "gRPC Client cert filename for Envoy to load.").Envar("ENVOY_CERT_FILE").StringVar(&ctx.config.GrpcClientCert)
bootstrap.Flag("envoy-key-file", "gRPC Client key filename for Envoy to load.").Envar("ENVOY_KEY_FILE").StringVar(&ctx.config.GrpcClientKey)
bootstrap.Flag("namespace", "The namespace the Envoy container will run in.").Envar("CONTOUR_NAMESPACE").Default("projectcontour").StringVar(&ctx.config.Namespace)
return bootstrap, &ctx
}

type bootstrapContext struct {
config envoy.BootstrapConfig
path string
bootstrap.Arg("path", "Configuration file ('-' for standard output).").Required().StringVar(&config.Path)
bootstrap.Arg("envoy-resources-dir", "Directory where configuration files will be written to.").StringVar(&config.ResourcesDir)
bootstrap.Flag("admin-address", "Envoy admin interface address.").StringVar(&config.AdminAddress)
bootstrap.Flag("admin-port", "Envoy admin interface port.").IntVar(&config.AdminPort)
bootstrap.Flag("xds-address", "xDS gRPC API address.").StringVar(&config.XDSAddress)
bootstrap.Flag("xds-port", "xDS gRPC API port.").IntVar(&config.XDSGRPCPort)
bootstrap.Flag("envoy-cafile", "gRPC CA Filename for Envoy to load.").Envar("ENVOY_CAFILE").StringVar(&config.GrpcCABundle)
bootstrap.Flag("envoy-cert-file", "gRPC Client cert filename for Envoy to load.").Envar("ENVOY_CERT_FILE").StringVar(&config.GrpcClientCert)
bootstrap.Flag("envoy-key-file", "gRPC Client key filename for Envoy to load.").Envar("ENVOY_KEY_FILE").StringVar(&config.GrpcClientKey)
bootstrap.Flag("namespace", "The namespace the Envoy container will run in.").Envar("CONTOUR_NAMESPACE").Default("projectcontour").StringVar(&config.Namespace)
return bootstrap, &config
}

// doBootstrap writes an Envoy bootstrap configuration file to the supplied path.
func doBootstrap(ctx *bootstrapContext) {
var out io.Writer

switch ctx.path {
case "-":
out = os.Stdout
default:
f, err := os.Create(ctx.path)
check(err)

out = f

defer func() {
check(f.Close())
}()
}

m := &jsonpb.Marshaler{OrigName: true}

check(m.Marshal(out, envoy.Bootstrap(&ctx.config)))
// doBootstrap writes an Envoy bootstrap configuration files to the supplied paths.
func doBootstrap(config *envoy.BootstrapConfig) {
envoy.Bootstrap(config)
}
148 changes: 143 additions & 5 deletions internal/envoy/bootstrap.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,10 @@
package envoy

import (
"fmt"
"log"
"os"
"path"
"strconv"
"strings"
"time"
Expand All @@ -27,11 +30,25 @@ import (
envoy_api_v2_core "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
bootstrap "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v2"
matcher "github.com/envoyproxy/go-control-plane/envoy/type/matcher"
"github.com/gogo/protobuf/proto"
"github.com/golang/protobuf/jsonpb"
"github.com/golang/protobuf/ptypes/any"
"github.com/projectcontour/contour/internal/protobuf"
)

// bootstrapConfigFile stores the path to Envoy's bootstrap configuration.
const bootstrapConfigFile = "envoy.json"

// sdsTLSCertificateFile stores the path to the SDS resource with Envoy's
// client certificate and key for XDS gRPC connection.
const sdsTLSCertificateFile = "xds-tls-certicate.json"

// sdsValidationContextFile stores the path to the SDS resource with
// CA certificates for Envoy to use for the XDS gRPC connection.
const sdsValidationContextFile = "xds-validation-context.json"

// Bootstrap creates a new v2 Bootstrap configuration.
func Bootstrap(c *BootstrapConfig) *bootstrap.Bootstrap {
func Bootstrap(c *BootstrapConfig) {
b := &bootstrap.Bootstrap{
DynamicResources: &bootstrap.Bootstrap_DynamicResources{
LdsConfig: ConfigSource("contour"),
Expand Down Expand Up @@ -98,12 +115,27 @@ func Bootstrap(c *BootstrapConfig) *bootstrap.Bootstrap {
if !(c.GrpcClientCert != "" && c.GrpcClientKey != "" && c.GrpcCABundle != "") {
log.Fatal("You must supply all three TLS parameters - --envoy-cafile, --envoy-cert-file, --envoy-key-file, or none of them.")
}
b.StaticResources.Clusters[0].TransportSocket = UpstreamTLSTransportSocket(
upstreamFileTLSContext(c.GrpcCABundle, c.GrpcClientCert, c.GrpcClientKey),
)

var tls *envoy_api_v2_core.TransportSocket

if c.ResourcesDir != "" {
// Create separate SDS resource files to refer to certificate and key files.
// SDS is needed to support certificate rotation for xDS.
err := os.MkdirAll(path.Join(c.ResourcesDir, "sds"), os.ModePerm)
check(err)
writeConfig(path.Join(c.ResourcesDir, "sds", sdsTLSCertificateFile), tlsCertificateSdsSecretConfig(c.GrpcClientCert, c.GrpcClientKey))
writeConfig(path.Join(c.ResourcesDir, "sds", sdsValidationContextFile), validationContextSdsSecretConfig(c.GrpcCABundle))
tls = UpstreamTLSTransportSocket(upstreamSdsTLSContext(path.Join(c.Path, sdsTLSCertificateFile), path.Join(c.Path, sdsValidationContextFile)))
} else {
// Old behavior is to use direct certificate and key file paths in bootstrap config.
// Envoy does not support rotation of xDS certificate files in this case.
tls = UpstreamTLSTransportSocket(upstreamFileTLSContext(c.GrpcCABundle, c.GrpcClientCert, c.GrpcClientKey))
}
_ = tls
//b.StaticResources.Clusters[0].TransportSocket = tls
}

return b
writeConfig(path.Join(c.Path, bootstrapConfigFile), b)
}

func upstreamFileTLSContext(cafile, certfile, keyfile string) *envoy_api_v2_auth.UpstreamTlsContext {
Expand Down Expand Up @@ -141,6 +173,81 @@ func upstreamFileTLSContext(cafile, certfile, keyfile string) *envoy_api_v2_auth
return context
}

func upstreamSdsTLSContext(certificateSdsFile, validationSdsFile string) *envoy_api_v2_auth.UpstreamTlsContext {
context := &envoy_api_v2_auth.UpstreamTlsContext{
CommonTlsContext: &envoy_api_v2_auth.CommonTlsContext{
TlsCertificateSdsSecretConfigs: []*envoy_api_v2_auth.SdsSecretConfig{{
SdsConfig: &envoy_api_v2_core.ConfigSource{
ConfigSourceSpecifier: &envoy_api_v2_core.ConfigSource_Path{
Path: certificateSdsFile,
},
},
}},
ValidationContextType: &envoy_api_v2_auth.CommonTlsContext_ValidationContextSdsSecretConfig{
ValidationContextSdsSecretConfig: &envoy_api_v2_auth.SdsSecretConfig{
SdsConfig: &envoy_api_v2_core.ConfigSource{
ConfigSourceSpecifier: &envoy_api_v2_core.ConfigSource_Path{
Path: validationSdsFile,
},
},
},
},
},
}
return context
}

func discoveryResponse(secret *envoy_api_v2_auth.Secret) *api.DiscoveryResponse {
response := &api.DiscoveryResponse{
Resources: []*any.Any{toAny(secret)},
}
return response
}

// tlsCertificateSdsSecretConfig creates DiscoveryResponse with file based SDS resource
// including paths to TLS certificates and key
func tlsCertificateSdsSecretConfig(certfile, keyfile string) *api.DiscoveryResponse {
secret := &envoy_api_v2_auth.Secret{
Type: &envoy_api_v2_auth.Secret_TlsCertificate{
TlsCertificate: &envoy_api_v2_auth.TlsCertificate{
CertificateChain: &envoy_api_v2_core.DataSource{
Specifier: &envoy_api_v2_core.DataSource_Filename{
Filename: certfile,
},
},
PrivateKey: &envoy_api_v2_core.DataSource{
Specifier: &envoy_api_v2_core.DataSource_Filename{
Filename: keyfile,
},
},
},
},
}
return discoveryResponse(secret)
}

// validationContextSdsSecretConfig creates DiscoveryResponse with file based SDS resource
// including path to CA certificate bundle
func validationContextSdsSecretConfig(cafile string) *api.DiscoveryResponse {
secret := &envoy_api_v2_auth.Secret{
Type: &envoy_api_v2_auth.Secret_ValidationContext{
ValidationContext: &envoy_api_v2_auth.CertificateValidationContext{
TrustedCa: &envoy_api_v2_core.DataSource{
Specifier: &envoy_api_v2_core.DataSource_Filename{
Filename: cafile,
},
},
MatchSubjectAltNames: []*matcher.StringMatcher{{
MatchPattern: &matcher.StringMatcher_Exact{
Exact: "contour",
}},
},
},
},
}
return discoveryResponse(secret)
}

// BootstrapConfig holds configuration values for a v2.Bootstrap.
type BootstrapConfig struct {
// AdminAccessLogPath is the path to write the access log for the administration server.
Expand Down Expand Up @@ -176,6 +283,12 @@ type BootstrapConfig struct {

// GrpcClientKey is the filename that contains a client key for secure gRPC with TLS.
GrpcClientKey string

// Path is the filename for the bootstrap configuration file to be created.
Path string

// ResourcesDir is the directory where out of line Envoy resources can be placed.
ResourcesDir string
}

func (c *BootstrapConfig) xdsAddress() string { return stringOrDefault(c.XDSAddress, "127.0.0.1") }
Expand All @@ -199,3 +312,28 @@ func intOrDefault(i, def int) int {
}
return i
}

func writeConfig(filename string, config proto.Message) {
var out *os.File

if filename == "-" {
out = os.Stdout
} else {
var err error
out, err = os.Create(filename)
check(err)
defer func() {
check(out.Close())
}()
}

m := &jsonpb.Marshaler{OrigName: true}
check(m.Marshal(out, config))
}

func check(err error) {
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}

0 comments on commit c2abce8

Please sign in to comment.