Vault Secret Manager #patch (flyteorg#343)

* Start adding Vaul Secret manager Signed-off-by: Tim Bauer <[email protected]> * Auto-update enumer Signed-off-by: Tim Bauer <[email protected]> * Make verbose Signed-off-by: Tim Bauer <[email protected]> * Revert to print Signed-off-by: Tim Bauer <[email protected]> * Mark debug statements Signed-off-by: Tim Bauer <[email protected]> * Remove prints, simplify vault Signed-off-by: Tim Bauer <[email protected]> * Test format env var, print more Signed-off-by: Tim Bauer <[email protected]> * Check annotations Signed-off-by: Tim Bauer <[email protected]> * Try to retrieve annotations Signed-off-by: Tim Bauer <[email protected]> * Attempt append annotation Signed-off-by: Tim Bauer <[email protected]> * Test annotation injection Signed-off-by: Tim Bauer <[email protected]> * Pre-populate only Signed-off-by: Tim Bauer <[email protected]> * Utils func for vault secret annotation Signed-off-by: Tim Bauer <[email protected]> * Add shorter id to avoid 63 char limit Signed-off-by: Tim Bauer <[email protected]> * Rm print Signed-off-by: Tim Bauer <[email protected]> * Set vault role from config Signed-off-by: Tim Bauer <[email protected]> * Add tests Signed-off-by: Tim Bauer <[email protected]> * Name coreIdl import Signed-off-by: Tim Bauer <[email protected]> * Rm duplicate import Signed-off-by: Tim Bauer <[email protected]> * Update documentation and naming Signed-off-by: Tim Bauer <[email protected]> * Update pkg/webhook/utils.go Co-authored-by: Haytham Abuelfutuh <[email protected]> Signed-off-by: Tim Bauer <[email protected]> * Update pkg/webhook/vault_secret_manager.go There is a [workaround](https://www.vaultproject.io/docs/platform/k8s/injector/examples#environment-variable-example) which involves mounting a template formatted file that contains `export API_KEY="{{ .Data.data.api_key }}"` and then sourcing this file as an extra step. But unless the user takes this extra sourcing step, this is still file mounting. So I would go with this Error message since the user should be warned that the expected result from requesting Env var will not be achieved with this. Co-authored-by: Haytham Abuelfutuh <[email protected]> Signed-off-by: Tim Bauer <[email protected]> * Update pkg/webhook/vault_secret_manager.go Co-authored-by: Haytham Abuelfutuh <[email protected]> Signed-off-by: Tim Bauer <[email protected]> * Fix naming and indent Signed-off-by: Tim Bauer <[email protected]> * Add handling of different kv version and test Signed-off-by: Tim Bauer <[email protected]> * Remove print Signed-off-by: Tim Bauer <[email protected]> * Add enumer for KV version Signed-off-by: Tim Bauer <[email protected]> * Correct kvversion type Signed-off-by: Tim Bauer <[email protected]> * Rm newlines from vault secret template Signed-off-by: Tim Bauer <[email protected]> * Apply suggestions from code review Co-authored-by: Ketan Umare <[email protected]> Signed-off-by: Tim Bauer <[email protected]> * Add docstring Signed-off-by: Tim Bauer <[email protected]> Co-authored-by: Haytham Abuelfutuh <[email protected]> Co-authored-by: Ketan Umare <[email protected]>
eapolinario · Oct 21, 2021 · 9e83dee · 9e83dee
1 parent 7b09010
commit 9e83dee
Show file tree

Hide file tree

Showing 10 changed files with 437 additions and 15 deletions.
diff --git a/flytepropeller/pkg/webhook/config/config.go b/flytepropeller/pkg/webhook/config/config.go
@@ -7,6 +7,7 @@ import (
 )
 
 //go:generate enumer --type=SecretManagerType --trimprefix=SecretManagerType -json -yaml
+//go:generate enumer --type=KVVersion --trimprefix=KVVersion -json -yaml
 //go:generate pflags Config --default-var=DefaultConfig
 
 var (
@@ -30,6 +31,10 @@ var (
 				},
 			},
 		},
+		VaultSecretManagerConfig: VaultSecretManagerConfig{
+			Role:      "flyte",
+			KVVersion: KVVersion2,
+		},
 	}
 
 	configSection = config.MustRegisterSection("webhook", DefaultConfig)
@@ -49,23 +54,42 @@ const (
 	// SecretManagerTypeAWS defines a secret manager webhook that injects a side car to pull secrets from AWS Secret
 	// Manager and mount them to a local file system (in memory) and share that mount with other containers in the pod.
 	SecretManagerTypeAWS
+
+	// SecretManagerTypeVault defines a secret manager webhook that pulls secrets from Hashicorp Vault.
+	SecretManagerTypeVault
+)
+
+// Defines with KV Engine Version to use with VaultSecretManager - https://www.vaultproject.io/docs/secrets/kv#kv-secrets-engine
+type KVVersion int
+
+const (
+	// KV v1 refers to unversioned secrets
+	KVVersion1 KVVersion = iota
+	// KV v2 refers to versioned secrets
+	KVVersion2
 )
 
 type Config struct {
-	MetricsPrefix          string                 `json:"metrics-prefix" pflag:",An optional prefix for all published metrics."`
-	CertDir                string                 `json:"certDir" pflag:",Certificate directory to use to write generated certs. Defaults to /etc/webhook/certs/"`
-	ListenPort             int                    `json:"listenPort" pflag:",The port to use to listen to webhook calls. Defaults to 9443"`
-	ServiceName            string                 `json:"serviceName" pflag:",The name of the webhook service."`
-	SecretName             string                 `json:"secretName" pflag:",Secret name to write generated certs to."`
-	SecretManagerType      SecretManagerType      `json:"secretManagerType" pflag:"-,Secret manager type to use if secrets are not found in global secrets."`
-	AWSSecretManagerConfig AWSSecretManagerConfig `json:"awsSecretManager" pflag:",AWS Secret Manager config."`
+	MetricsPrefix            string                   `json:"metrics-prefix" pflag:",An optional prefix for all published metrics."`
+	CertDir                  string                   `json:"certDir" pflag:",Certificate directory to use to write generated certs. Defaults to /etc/webhook/certs/"`
+	ListenPort               int                      `json:"listenPort" pflag:",The port to use to listen to webhook calls. Defaults to 9443"`
+	ServiceName              string                   `json:"serviceName" pflag:",The name of the webhook service."`
+	SecretName               string                   `json:"secretName" pflag:",Secret name to write generated certs to."`
+	SecretManagerType        SecretManagerType        `json:"secretManagerType" pflag:"-,Secret manager type to use if secrets are not found in global secrets."`
+	AWSSecretManagerConfig   AWSSecretManagerConfig   `json:"awsSecretManager" pflag:",AWS Secret Manager config."`
+	VaultSecretManagerConfig VaultSecretManagerConfig `json:"vaultSecretManager" pflag:",Vault Secret Manager config."`
 }
 
 type AWSSecretManagerConfig struct {
 	SidecarImage string                      `json:"sidecarImage" pflag:",Specifies the sidecar docker image to use"`
 	Resources    corev1.ResourceRequirements `json:"resources" pflag:"-,Specifies resource requirements for the init container."`
 }
 
+type VaultSecretManagerConfig struct {
+	Role      string    `json:"role" pflag:",Specifies the vault role to use"`
+	KVVersion KVVersion `json:"kvVersion" pflag:"-,The KV Engine Version. Defaults to 2. Use 1 for unversioned secrets. Refer to - https://www.vaultproject.io/docs/secrets/kv#kv-secrets-engine."`
+}
+
 func GetConfig() *Config {
 	return configSection.GetConfig().(*Config)
 }
diff --git a/flytepropeller/pkg/webhook/config/config_flags.go b/flytepropeller/pkg/webhook/config/config_flags.go
diff --git a/flytepropeller/pkg/webhook/config/config_flags_test.go b/flytepropeller/pkg/webhook/config/config_flags_test.go
diff --git a/flytepropeller/pkg/webhook/config/kvversion_enumer.go b/flytepropeller/pkg/webhook/config/kvversion_enumer.go
diff --git a/flytepropeller/pkg/webhook/config/secretmanagertype_enumer.go b/flytepropeller/pkg/webhook/config/secretmanagertype_enumer.go
diff --git a/flytepropeller/pkg/webhook/k8s_secrets_test.go b/flytepropeller/pkg/webhook/k8s_secrets_test.go
@@ -6,7 +6,6 @@ import (
 
 	"github.com/go-test/deep"
 
-	"github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core"
 	coreIdl "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core"
 	corev1 "k8s.io/api/core/v1"
 )
@@ -179,7 +178,7 @@ func TestK8sSecretInjector_Inject(t *testing.T) {
 
 	ctx := context.Background()
 	type args struct {
-		secret *core.Secret
+		secret *coreIdl.Secret
 		p      *corev1.Pod
 	}
 	tests := []struct {

diff --git a/flytepropeller/pkg/webhook/secrets.go b/flytepropeller/pkg/webhook/secrets.go
@@ -75,6 +75,7 @@ func NewSecretsMutator(cfg *config.Config, _ promutils.Scope) *SecretsMutator {
 			NewGlobalSecrets(secretmanager.NewFileEnvSecretManager(secretmanager.GetConfig())),
 			NewK8sSecretsInjector(),
 			NewAWSSecretManagerInjector(cfg.AWSSecretManagerConfig),
+			NewVaultSecretManagerInjector(cfg.VaultSecretManagerConfig),
 		},
 	}
 }
diff --git a/flytepropeller/pkg/webhook/utils.go b/flytepropeller/pkg/webhook/utils.go
@@ -1,14 +1,17 @@
 package webhook
 
 import (
+	"fmt"
 	"path/filepath"
 	"strings"
 
 	"github.com/flyteorg/flyteplugins/go/tasks/pluginmachinery/encoding"
+	"github.com/flyteorg/flytepropeller/pkg/webhook/config"
 
 	"github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core"
 
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/uuid"
 )
 
 func hasEnvVar(envVars []corev1.EnvVar, envVarKey string) bool {
@@ -115,3 +118,31 @@ func AppendVolume(volumes []corev1.Volume, volume corev1.Volume) []corev1.Volume
 
 	return append(volumes, volume)
 }
+
+func CreateVaultAnnotationsForSecret(secret *core.Secret, kvversion config.KVVersion) (map[string]string, error) {
+	// Creates three grouped annotations "agent-inject-secret", "agent-inject-file" and "agent-inject-template"
+	// for a given secret request and KV engine version. The annotations respectively handle: 1. retrieving the
+	// secret from a vault path specified in secret.Group, 2. storing it in a file named after secret.Group/secret.Key
+	// and 3. creating a template that retrieves only secret.Key from the multiple k:v pairs present in a vault secret.
+	id := string(uuid.NewUUID())
+
+	// Set the consul template language query depending on the KV Secrets Engine version.
+	// Version 1 stores plain k:v pairs under .Data, version 2 supports versioned secrets
+	// and wraps the k:v pairs into an additional subfield.
+	var query string
+	if kvversion == config.KVVersion1 {
+		query = ".Data"
+	} else if kvversion == config.KVVersion2 {
+		query = ".Data.data"
+	} else {
+		err := fmt.Errorf("unsupported KV Version %v, supported versions are 1 and 2", kvversion)
+		return nil, err
+	}
+	template := fmt.Sprintf(`{{- with secret "%s" -}}{{ %s.%s }}{{- end -}}`, secret.Group, query, secret.Key)
+	secretVaultAnnotations := map[string]string{
+		fmt.Sprintf("vault.hashicorp.com/agent-inject-secret-%s", id):   secret.Group,
+		fmt.Sprintf("vault.hashicorp.com/agent-inject-file-%s", id):     fmt.Sprintf("%s/%s", secret.Group, secret.Key),
+		fmt.Sprintf("vault.hashicorp.com/agent-inject-template-%s", id): template,
+	}
+	return secretVaultAnnotations, nil
+}
diff --git a/flytepropeller/pkg/webhook/vault_secret_manager.go b/flytepropeller/pkg/webhook/vault_secret_manager.go
@@ -0,0 +1,95 @@
+package webhook
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	coreIdl "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core"
+	"github.com/flyteorg/flyteplugins/go/tasks/pluginmachinery/utils"
+	"github.com/flyteorg/flytepropeller/pkg/webhook/config"
+	"github.com/flyteorg/flytestdlib/logger"
+	corev1 "k8s.io/api/core/v1"
+)
+
+var (
+	VaultSecretPathPrefix = []string{string(os.PathSeparator), "etc", "flyte", "secrets"}
+)
+
+// VaultSecretManagerInjector allows injecting of secrets into pods by leveraging an existing deployment of Vault Agent
+// Vault Agent functions as an additional webhook that is triggered through annotations and then retrieves and mounts
+// the requested secrets from Vault. This injector parses a secret Request into vault annotations, interpreting the secret
+// Group as the vault secret path and the secret Key as the key for which to extract a value from a Vault secret.
+// It supports adding multiple secrets. (The common annotations will simply be overwritten if added several times)
+// Note that you need to configure the Vault role that this injector will try to use and add Vault policies for
+// the service account and namespaces that your workflows run under.
+// Files will be mounted at /etc/flyte/secrets/<SecretGroup>/<SecretKey>
+type VaultSecretManagerInjector struct {
+	cfg config.VaultSecretManagerConfig
+}
+
+func (i VaultSecretManagerInjector) Type() config.SecretManagerType {
+	return config.SecretManagerTypeVault
+}
+
+func (i VaultSecretManagerInjector) Inject(ctx context.Context, secret *coreIdl.Secret, p *corev1.Pod) (newP *corev1.Pod, injected bool, err error) {
+	if len(secret.Group) == 0 || len(secret.Key) == 0 {
+		return nil, false, fmt.Errorf("Vault Secrets Webhook requires both key and group to be set. "+
+			"Secret: [%v]", secret)
+	}
+
+	switch secret.MountRequirement {
+	case coreIdl.Secret_ANY:
+		fallthrough
+	case coreIdl.Secret_FILE:
+		// Set environment variable to let the container know where to find the mounted files.
+		defaultDirEnvVar := corev1.EnvVar{
+			Name:  SecretPathDefaultDirEnvVar,
+			Value: filepath.Join(VaultSecretPathPrefix...),
+		}
+
+		p.Spec.InitContainers = AppendEnvVars(p.Spec.InitContainers, defaultDirEnvVar)
+		p.Spec.Containers = AppendEnvVars(p.Spec.Containers, defaultDirEnvVar)
+
+		// Sets an empty prefix to let the containers know the file names will match the secret keys as-is.
+		prefixEnvVar := corev1.EnvVar{
+			Name:  SecretPathFilePrefixEnvVar,
+			Value: "",
+		}
+
+		p.Spec.InitContainers = AppendEnvVars(p.Spec.InitContainers, prefixEnvVar)
+		p.Spec.Containers = AppendEnvVars(p.Spec.Containers, prefixEnvVar)
+
+		commonVaultAnnotations := map[string]string{
+			"vault.hashicorp.com/agent-inject":            "true",
+			"vault.hashicorp.com/secret-volume-path":      filepath.Join(VaultSecretPathPrefix...),
+			"vault.hashicorp.com/role":                    i.cfg.Role,
+			"vault.hashicorp.com/agent-pre-populate-only": "true",
+		}
+
+		secretVaultAnnotations, err := CreateVaultAnnotationsForSecret(secret, i.cfg.KVVersion)
+		// Creating annotations can break with an unsupported KVVersion
+		if err != nil {
+			return p, false, err
+		}
+
+		p.ObjectMeta.Annotations = utils.UnionMaps(p.ObjectMeta.Annotations, commonVaultAnnotations)
+		p.ObjectMeta.Annotations = utils.UnionMaps(p.ObjectMeta.Annotations, secretVaultAnnotations)
+
+	case coreIdl.Secret_ENV_VAR:
+		return p, false, fmt.Errorf("Env_Var is not a supported mount requirement for Vault Secret Manager")
+	default:
+		err := fmt.Errorf("unrecognized mount requirement [%v] for secret [%v]", secret.MountRequirement.String(), secret.Key)
+		logger.Error(ctx, err)
+		return p, false, err
+	}
+
+	return p, true, nil
+}
+
+func NewVaultSecretManagerInjector(cfg config.VaultSecretManagerConfig) VaultSecretManagerInjector {
+	return VaultSecretManagerInjector{
+		cfg: cfg,
+	}
+}