Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

connect: set consul TLS options on envoy bootstrap #7602

Merged
merged 1 commit into from
Apr 3, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
107 changes: 92 additions & 15 deletions client/allocrunner/taskrunner/envoybootstrap_hook.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,54 @@ import (
agentconsul "github.com/hashicorp/nomad/command/agent/consul"
"github.com/hashicorp/nomad/helper"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/hashicorp/nomad/nomad/structs/config"
"github.com/pkg/errors"
)

const envoyBootstrapHookName = "envoy_bootstrap"

type envoyBootstrapConsulConfig struct {
HTTPAddr string // required
Auth string // optional, env CONSUL_HTTP_AUTH
SSL string // optional, env CONSUL_HTTP_SSL
VerifySSL string // optional, env CONSUL_HTTP_SSL_VERIFY
CAFile string // optional, arg -ca-file
CertFile string // optional, arg -client-cert
KeyFile string // optional, arg -client-key
// CAPath (dir) not supported by Nomad's config object
}

type envoyBootstrapHookConfig struct {
alloc *structs.Allocation
consulHTTPAddr string
logger hclog.Logger
consul envoyBootstrapConsulConfig
alloc *structs.Allocation
logger hclog.Logger
}

func decodeTriState(b *bool) string {
switch {
case b == nil:
return ""
case *b:
return "true"
default:
return "false"
}
}

func newEnvoyBootstrapHookConfig(alloc *structs.Allocation, consul *config.ConsulConfig, logger hclog.Logger) *envoyBootstrapHookConfig {
return &envoyBootstrapHookConfig{
alloc: alloc,
logger: logger,
consul: envoyBootstrapConsulConfig{
HTTPAddr: consul.Addr,
Auth: consul.Auth,
SSL: decodeTriState(consul.EnableSSL),
VerifySSL: decodeTriState(consul.VerifySSL),
CAFile: consul.CAFile,
CertFile: consul.CertFile,
KeyFile: consul.KeyFile,
},
}
}

const (
Expand All @@ -40,18 +79,19 @@ type envoyBootstrapHook struct {

// Bootstrapping Envoy requires talking directly to Consul to generate
// the bootstrap.json config. Runtime Envoy configuration is done via
// Consul's gRPC endpoint.
consulHTTPAddr string
// Consul's gRPC endpoint. There are many security parameters to configure
// before contacting Consul.
consulConfig envoyBootstrapConsulConfig

// logger is used to log things
logger hclog.Logger
}

func newEnvoyBootstrapHook(c *envoyBootstrapHookConfig) *envoyBootstrapHook {
return &envoyBootstrapHook{
alloc: c.alloc,
consulHTTPAddr: c.consulHTTPAddr,
logger: c.logger.Named(envoyBootstrapHookName),
alloc: c.alloc,
consulConfig: c.consul,
logger: c.logger.Named(envoyBootstrapHookName),
}
}

Expand Down Expand Up @@ -113,19 +153,23 @@ func (h *envoyBootstrapHook) Prestart(ctx context.Context, req *interfaces.TaskP
}
h.logger.Debug("check for SI token for task", "task", req.Task.Name, "exists", siToken != "")

bootstrapArgs := envoyBootstrapArgs{
bootstrapBuilder := envoyBootstrapArgs{
consulConfig: h.consulConfig,
sidecarFor: id,
grpcAddr: grpcAddr,
consulHTTPAddr: h.consulHTTPAddr,
envoyAdminBind: envoyAdminBind,
siToken: siToken,
}.args()
}

bootstrapArgs := bootstrapBuilder.args()
bootstrapEnv := bootstrapBuilder.env(os.Environ())

// Since Consul services are registered asynchronously with this task
// hook running, retry a small number of times with backoff.
for tries := 3; ; tries-- {

cmd := exec.CommandContext(ctx, "consul", bootstrapArgs...)
cmd.Env = bootstrapEnv

// Redirect output to secrets/envoy_bootstrap.json
fd, err := os.Create(bootstrapFilePath)
Expand Down Expand Up @@ -225,10 +269,10 @@ func (h *envoyBootstrapHook) execute(cmd *exec.Cmd) (string, error) {
// along to the exec invocation of consul which will then generate the bootstrap
// configuration file for envoy.
type envoyBootstrapArgs struct {
consulConfig envoyBootstrapConsulConfig
sidecarFor string
grpcAddr string
envoyAdminBind string
consulHTTPAddr string
siToken string
}

Expand All @@ -239,17 +283,50 @@ func (e envoyBootstrapArgs) args() []string {
"connect",
"envoy",
"-grpc-addr", e.grpcAddr,
"-http-addr", e.consulHTTPAddr,
"-http-addr", e.consulConfig.HTTPAddr,
"-admin-bind", e.envoyAdminBind,
"-bootstrap",
"-sidecar-for", e.sidecarFor,
}
if e.siToken != "" {
arguments = append(arguments, "-token", e.siToken)

if v := e.siToken; v != "" {
arguments = append(arguments, "-token", v)
}

if v := e.consulConfig.CAFile; v != "" {
arguments = append(arguments, "-ca-file", v)
}

if v := e.consulConfig.CertFile; v != "" {
arguments = append(arguments, "-client-cert", v)
}

if v := e.consulConfig.KeyFile; v != "" {
arguments = append(arguments, "-client-key", v)
}

return arguments
}

// env creates the context of environment variables to be used when exec-ing
// the consul command for generating the envoy bootstrap config. It is expected
// the value of os.Environ() is passed in to be appended to. Because these are
// appended at the end of what will be passed into Cmd.Env, they will override
// any pre-existing values (i.e. what the Nomad agent was launched with).
// https://golang.org/pkg/os/exec/#Cmd
func (e envoyBootstrapArgs) env(env []string) []string {
if v := e.consulConfig.Auth; v != "" {
env = append(env, fmt.Sprintf("%s=%s", "CONSUL_HTTP_AUTH", v))
}
if v := e.consulConfig.SSL; v != "" {
env = append(env, fmt.Sprintf("%s=%s", "CONSUL_HTTP_SSL", v))
}
if v := e.consulConfig.VerifySSL; v != "" {
env = append(env, fmt.Sprintf("%s=%s", "CONSUL_HTTP_SSL_VERIFY", v))
}
return env
}

// maybeLoadSIToken reads the SI token saved to disk in the secrets directory
// by the service identities prestart hook. This envoy bootstrap hook blocks
// until the sids hook completes, so if the SI token is required to exist (i.e.
Expand Down
Loading