Add support for Enterprise License to be configured in Vault (#1032)

* Validate that users must set either both or neither enterpriseLicense.secretName and secretKey

* Add ability to configure enterprise license tocome from vault

* Add ability to configure enterprise license for client snapshot agent to come from vault

* Add ability to configure enterprise license for client daemonset to come from vault

* Updating the comments in help for enterprise license to mention it can also come from Vault.

* Update Changelog for Enterprise License on Vault

* Removing redundant tests in server-acl-init-job that test for existance of either both or neither enterprise license secretKey and secretName.

* Do not mount volume or volume mounts for license when using vault

* Removing redundant broken tests related to entperiseLicense not having either secretKey or secretName when the other is supplied

* Adding test for that both global.enterpriseLicense.secretName and secretKey are provided when one of them is provided.

* Adding acceptance test for Enterprise License on vault

* Fixing acceptance test for Enterprise License on vault

* Fixing ENTERPRISE_LICENSE setting enterprise-license-job for Enterprise License on vault

* Fixing formatting

* Fixing unit tests

* Changing enterprise license logic in vault acceptance test to be conditional based on -enable-enterprise

* Making helm values for ent license conditional in vault accpentance test.  Adding failure logic to client-daemonset.yaml if secretName or secretKey is missing if one of them is already supplied.

* checking on only enterpriseLicense.secretName (without secretKey) since chart will fail when both are not supplied.

* Fixing broken test by allowing test suite config to be passed in so that we can conditionally add consul-enterprise licence policy to the Vault Auth Roles.

* Adding license config in vault to TestVault_WANFederationViaGateways

* Adding failure tests to client-statefulset when only one of global.enterpriseLicense.secretKey or secretName is supplied.

CHANGELOG.md

@@ -3,6 +3,7 @@ IMPROVEMENTS:
 * Helm
   * Vault: Allow passing arbitrary annotations to the vault agent. [[GH-1015](https://github.com/hashicorp/consul-k8s/pull/1015)]
   * Vault: Add support for customized IP and DNS SANs for server cert in Vault. [[GH-1020](https://github.com/hashicorp/consul-k8s/pull/1020)]
+  * Vault: Add support for Enterprise License to be configured in Vault. [[GH-1032](https://github.com/hashicorp/consul-k8s/pull/1032)]
 
 BUG FIXES:
 * API Gateway

acceptance/tests/vault/helpers.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"testing"
 
+	"github.com/hashicorp/consul-k8s/acceptance/framework/config"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/logger"
 	"github.com/hashicorp/go-uuid"
 	vapi "github.com/hashicorp/vault/api"
@@ -22,7 +23,12 @@ path "consul/data/secret/replication" {
   capabilities = ["read", "update"]
 }`
 
-	// connectCAPolicyTemplate allows Consul to bootstrap all certificates for the service mesh in Vault.
+	enterpriseLicensePolicy = `
+path "consul/data/secret/enterpriselicense" {
+  capabilities = ["read"]
+}`
+
+	// connectCAPolicy allows Consul to bootstrap all certificates for the service mesh in Vault.
 	// Adapted from https://www.consul.io/docs/connect/ca/vault#consul-managed-pki-paths.
 	connectCAPolicyTemplate = `
 path "/sys/mounts" {
@@ -93,11 +99,32 @@ func configureGossipVaultSecret(t *testing.T, vaultClient *vapi.Client) string {
 	return gossipKey
 }
 
+// configureEnterpriseLicenseVaultSecret stores it in vault as a secret and configures a policy to access it.
+func configureEnterpriseLicenseVaultSecret(t *testing.T, vaultClient *vapi.Client, cfg *config.TestConfig) {
+	// Create the enterprise license secret.
+	logger.Log(t, "Creating the Enterprise License secret")
+	params := map[string]interface{}{
+		"data": map[string]interface{}{
+			"enterpriselicense": cfg.EnterpriseLicense,
+		},
+	}
+	_, err := vaultClient.Logical().Write("consul/data/secret/enterpriselicense", params)
+	require.NoError(t, err)
+
+	// Create the Vault Policy for the consul-enterpriselicense.
+	err = vaultClient.Sys().PutPolicy("consul-enterpriselicense", enterpriseLicensePolicy)
+	require.NoError(t, err)
+}
+
 // configureKubernetesAuthRoles configures roles for the Kubernetes auth method
 // that will be used by the test Helm chart installation.
-func configureKubernetesAuthRoles(t *testing.T, vaultClient *vapi.Client, consulReleaseName, ns, authPath, datacenter string) {
+func configureKubernetesAuthRoles(t *testing.T, vaultClient *vapi.Client, consulReleaseName, ns, authPath, datacenter string, cfg *config.TestConfig) {
 	consulClientServiceAccountName := fmt.Sprintf("%s-consul-client", consulReleaseName)
 	consulServerServiceAccountName := fmt.Sprintf("%s-consul-server", consulReleaseName)
+	sharedPolicies := "consul-gossip"
+	if cfg.EnableEnterprise {
+		sharedPolicies += ",consul-enterpriselicense"
+	}
 
 	// Create the Auth Roles for consul-server and consul-client.
 	// Auth roles bind policies to Kubernetes service accounts, which
@@ -109,7 +136,7 @@ func configureKubernetesAuthRoles(t *testing.T, vaultClient *vapi.Client, consul
 	params := map[string]interface{}{
 		"bound_service_account_names":      consulClientServiceAccountName,
 		"bound_service_account_namespaces": ns,
-		"policies":                         "consul-gossip",
+		"policies":                         sharedPolicies,
 		"ttl":                              "24h",
 	}
 	_, err := vaultClient.Logical().Write(fmt.Sprintf("auth/%s/role/consul-client", authPath), params)
@@ -118,7 +145,7 @@ func configureKubernetesAuthRoles(t *testing.T, vaultClient *vapi.Client, consul
 	params = map[string]interface{}{
 		"bound_service_account_names":      consulServerServiceAccountName,
 		"bound_service_account_namespaces": ns,
-		"policies":                         fmt.Sprintf("consul-gossip,connect-ca-%s,consul-server-%s,consul-replication-token", datacenter, datacenter),
+		"policies":                         fmt.Sprintf(sharedPolicies+",connect-ca-%s,consul-server-%s,consul-replication-token", datacenter, datacenter),
 		"ttl":                              "24h",
 	}
 	_, err = vaultClient.Logical().Write(fmt.Sprintf("auth/%s/role/consul-server", authPath), params)

acceptance/tests/vault/vault_test.go

@@ -31,8 +31,11 @@ func TestVault(t *testing.T) {
 	gossipKey := configureGossipVaultSecret(t, vaultClient)
 
 	createConnectCAPolicy(t, vaultClient, "dc1")
+	if cfg.EnableEnterprise {
+		configureEnterpriseLicenseVaultSecret(t, vaultClient, cfg)
+	}
 
-	configureKubernetesAuthRoles(t, vaultClient, consulReleaseName, ns, "kubernetes", "dc1")
+	configureKubernetesAuthRoles(t, vaultClient, consulReleaseName, ns, "kubernetes", "dc1", cfg)
 
 	configurePKICA(t, vaultClient)
 	certPath := configurePKICertificates(t, vaultClient, consulReleaseName, ns, "dc1")
@@ -82,6 +85,12 @@ func TestVault(t *testing.T) {
 		"syncCatalog.toConsul": "false",
 		"syncCatalog.toK8S":    "false",
 	}
+
+	if cfg.EnableEnterprise {
+		consulHelmValues["global.enterpriseLicense.secretName"] = "consul/data/secret/enterpriselicense"
+		consulHelmValues["global.enterpriseLicense.secretKey"] = "enterpriselicense"
+	}
+
 	logger.Log(t, "Installing Consul")
 	consulCluster := consul.NewHelmCluster(t, consulHelmValues, ctx, cfg, consulReleaseName)
 	consulCluster.Create(t)
@@ -100,6 +109,14 @@ func TestVault(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, caConfig.Provider, "vault")
 
+	if cfg.EnableEnterprise {
+		// Validate that the enterprise license is set correctly.
+		logger.Log(t, "Validating the enterprise license has been set correctly.")
+		license, licenseErr := consulClient.Operator().LicenseGet(nil)
+		require.NoError(t, licenseErr)
+		require.True(t, license.Valid)
+	}
+
 	// Deploy two services and check that they can talk to each other.
 	logger.Log(t, "creating static-server and static-client deployments")
 	k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-server-inject")

acceptance/tests/vault/vault_wan_fed_test.go

@@ -68,7 +68,11 @@ func TestVault_WANFederationViaGateways(t *testing.T) {
 
 	configureGossipVaultSecret(t, vaultClient)
 
-	configureKubernetesAuthRoles(t, vaultClient, consulReleaseName, ns, "kubernetes", "dc1")
+	if cfg.EnableEnterprise {
+		configureEnterpriseLicenseVaultSecret(t, vaultClient, cfg)
+	}
+
+	configureKubernetesAuthRoles(t, vaultClient, consulReleaseName, ns, "kubernetes", "dc1", cfg)
 
 	// Configure Vault Kubernetes auth method for the secondary datacenter.
 	{
@@ -109,7 +113,7 @@ func TestVault_WANFederationViaGateways(t *testing.T) {
 		secondaryVaultCluster.ConfigureAuthMethod(t, vaultClient, "kubernetes-dc2", k8sAuthMethodHost, authMethodRBACName, ns)
 	}
 
-	configureKubernetesAuthRoles(t, vaultClient, consulReleaseName, ns, "kubernetes-dc2", "dc2")
+	configureKubernetesAuthRoles(t, vaultClient, consulReleaseName, ns, "kubernetes-dc2", "dc2", cfg)
 
 	// Generate a CA and create PKI roles for the primary and secondary Consul servers.
 	configurePKICA(t, vaultClient)
@@ -180,6 +184,11 @@ func TestVault_WANFederationViaGateways(t *testing.T) {
 		"global.secretsBackend.vault.connectCA.intermediatePKIPath": "dc1/connect_inter",
 	}
 
+	if cfg.EnableEnterprise {
+		primaryConsulHelmValues["global.enterpriseLicense.secretName"] = "consul/data/secret/enterpriselicense"
+		primaryConsulHelmValues["global.enterpriseLicense.secretKey"] = "enterpriselicense"
+	}
+
 	if cfg.UseKind {
 		primaryConsulHelmValues["meshGateway.service.type"] = "NodePort"
 		primaryConsulHelmValues["meshGateway.service.nodePort"] = "30000"
@@ -238,6 +247,11 @@ func TestVault_WANFederationViaGateways(t *testing.T) {
 		"global.secretsBackend.vault.connectCA.additionalConfig":    fmt.Sprintf(`"{"connect": [{"ca_config": [{"tls_server_name": "%s-vault"}]}]}"`, vaultReleaseName),
 	}
 
+	if cfg.EnableEnterprise {
+		secondaryConsulHelmValues["global.enterpriseLicense.secretName"] = "consul/data/secret/enterpriselicense"
+		secondaryConsulHelmValues["global.enterpriseLicense.secretKey"] = "enterpriselicense"
+	}
+
 	if cfg.UseKind {
 		secondaryConsulHelmValues["meshGateway.service.type"] = "NodePort"
 		secondaryConsulHelmValues["meshGateway.service.nodePort"] = "30000"

charts/consul/templates/_helpers.tpl

@@ -15,7 +15,7 @@ as well as the global.name setting.
 {{- end -}}
 {{- end -}}
 
-{{- define "consul.vaultGossipTemplate" -}}
+{{- define "consul.vaultSecretTemplate" -}}
  |
             {{ "{{" }}- with secret "{{ .secretName }}" -{{ "}}" }}
             {{ "{{" }}- {{ printf ".Data.data.%s" .secretKey }} -{{ "}}" }}

charts/consul/templates/client-daemonset.yaml

@@ -7,6 +7,8 @@
 {{- if (and (and .Values.global.secretsBackend.vault.enabled .Values.global.tls.enabled) (not .Values.global.tls.enableAutoEncrypt)) }}{{ fail "global.tls.enableAutoEncrypt must be true if global.secretsBackend.vault.enabled=true and global.tls.enabled=true" }}{{ end -}}
 {{- if (and (and .Values.global.secretsBackend.vault.enabled .Values.global.tls.enabled) (not .Values.global.secretsBackend.vault.consulCARole)) }}{{ fail "global.secretsBackend.vault.consulCARole must be provided if global.secretsBackend.vault.enabled=true and global.tls.enabled=true" }}{{ end -}}
 {{- if and .Values.global.federation.enabled .Values.global.adminPartitions.enabled }}{{ fail "If global.federation.enabled is true, global.adminPartitions.enabled must be false because they are mutually exclusive" }}{{ end }}
+{{- if (and .Values.global.enterpriseLicense.secretName (not .Values.global.enterpriseLicense.secretKey)) }}{{fail "enterpriseLicense.secretKey and secretName must both be specified." }}{{ end -}}
+{{- if (and (not .Values.global.enterpriseLicense.secretName) .Values.global.enterpriseLicense.secretKey) }}{{fail "enterpriseLicense.secretKey and secretName must both be specified." }}{{ end -}}
 # DaemonSet to run the Consul clients on every node.
 apiVersion: apps/v1
 kind: DaemonSet
@@ -53,7 +55,7 @@ spec:
         {{- if .Values.global.gossipEncryption.secretName }}
         {{- with .Values.global.gossipEncryption }}
         "vault.hashicorp.com/agent-inject-secret-gossip.txt": {{ .secretName }}
-        "vault.hashicorp.com/agent-inject-template-gossip.txt": {{ template "consul.vaultGossipTemplate" . }}
+        "vault.hashicorp.com/agent-inject-template-gossip.txt": {{ template "consul.vaultSecretTemplate" . }}
         {{- end }}
         {{- end }}
         {{- if .Values.global.tls.enabled }}
@@ -63,6 +65,12 @@ spec:
         {{- if .Values.global.secretsBackend.vault.agentAnnotations }}
         {{ tpl .Values.global.secretsBackend.vault.agentAnnotations . | nindent 8 | trim }}
         {{- end }}
+        {{- if .Values.global.enterpriseLicense.secretName }}
+        {{- with .Values.global.enterpriseLicense }}
+        "vault.hashicorp.com/agent-inject-secret-enterpriselicense.txt": "{{ .secretName }}"
+        "vault.hashicorp.com/agent-inject-template-enterpriselicense.txt": {{ template "consul.vaultSecretTemplate" . }}
+        {{- end }}
+        {{- end }}
         {{- end }}
         "consul.hashicorp.com/connect-inject": "false"
         "consul.hashicorp.com/config-checksum": {{ include (print $.Template.BasePath "/client-config-configmap.yaml") . | sha256sum }}
@@ -159,7 +167,7 @@ spec:
         - name: aclconfig
           emptyDir: {}
         {{- else }}
-        {{- if (and .Values.global.enterpriseLicense.secretName .Values.global.enterpriseLicense.secretKey .Values.global.enterpriseLicense.enableLicenseAutoload) }}
+        {{- if (and .Values.global.enterpriseLicense.secretName .Values.global.enterpriseLicense.secretKey .Values.global.enterpriseLicense.enableLicenseAutoload (not .Values.global.secretsBackend.vault.enabled)) }}
         - name: consul-license
           secret:
             secretName: {{ .Values.global.enterpriseLicense.secretName }}
@@ -213,7 +221,11 @@ spec:
             {{- end }}
             {{- if (and .Values.global.enterpriseLicense.secretName .Values.global.enterpriseLicense.secretKey .Values.global.enterpriseLicense.enableLicenseAutoload (not .Values.global.acls.manageSystemACLs)) }}
             - name: CONSUL_LICENSE_PATH
+              {{- if  .Values.global.secretsBackend.vault.enabled }}
+              value: /vault/secrets/enterpriselicense.txt
+              {{- else }}
               value: /consul/license/{{ .Values.global.enterpriseLicense.secretKey }}
+              {{- end }}
             {{- end }}
             {{- if .Values.global.tls.enabled }}
             - name: CONSUL_HTTP_ADDR
@@ -348,7 +360,7 @@ spec:
             - name: aclconfig
               mountPath: /consul/aclconfig
             {{- else }}
-            {{- if (and .Values.global.enterpriseLicense.secretName .Values.global.enterpriseLicense.secretKey .Values.global.enterpriseLicense.enableLicenseAutoload) }}
+            {{- if (and .Values.global.enterpriseLicense.secretName .Values.global.enterpriseLicense.secretKey .Values.global.enterpriseLicense.enableLicenseAutoload (not .Values.global.secretsBackend.vault.enabled)) }}
             - name: consul-license
               mountPath: /consul/license
               readOnly: true

charts/consul/templates/client-snapshot-agent-deployment.yaml

@@ -28,7 +28,8 @@ spec:
         component: client-snapshot-agent
       annotations:
         "consul.hashicorp.com/connect-inject": "false"
-        {{- if (and .Values.global.secretsBackend.vault.enabled .Values.global.tls.enabled) }}
+        {{- if .Values.global.secretsBackend.vault.enabled }}
+        {{- if .Values.global.tls.enabled }}
         "vault.hashicorp.com/agent-init-first": "true"
         "vault.hashicorp.com/agent-inject": "true"
         "vault.hashicorp.com/role": {{ .Values.global.secretsBackend.vault.consulCARole }}
@@ -40,7 +41,14 @@ spec:
         {{- end }}
         {{- if .Values.global.secretsBackend.vault.agentAnnotations }}
         {{ tpl .Values.global.secretsBackend.vault.agentAnnotations . | nindent 8 | trim }}
+        {{- end }}        
+        {{- end }}
+        {{- if .Values.global.enterpriseLicense.secretName }}
+        {{- with .Values.global.enterpriseLicense }}
+        "vault.hashicorp.com/agent-inject-secret-enterpriselicense.txt": "{{ .secretName }}"
+        "vault.hashicorp.com/agent-inject-template-enterpriselicense.txt": {{ template "consul.vaultSecretTemplate" . }}
         {{- end }}
+        {{- end }}        
         {{- end }}
     spec:
       {{- if .Values.client.tolerations }}
@@ -66,7 +74,7 @@ spec:
         - name: aclconfig
           emptyDir: {}
         {{- else }}
-        {{- if (and .Values.global.enterpriseLicense.secretName .Values.global.enterpriseLicense.secretKey .Values.global.enterpriseLicense.enableLicenseAutoload) }}
+        {{- if (and .Values.global.enterpriseLicense.secretName .Values.global.enterpriseLicense.secretKey .Values.global.enterpriseLicense.enableLicenseAutoload (not .Values.global.secretsBackend.vault.enabled)) }}
         - name: consul-license
           secret:
             secretName: {{ .Values.global.enterpriseLicense.secretName }}
@@ -118,7 +126,11 @@ spec:
             {{- else }}
             {{- if (and .Values.global.enterpriseLicense.secretName .Values.global.enterpriseLicense.secretKey .Values.global.enterpriseLicense.enableLicenseAutoload) }}
             - name: CONSUL_LICENSE_PATH
+              {{- if  .Values.global.secretsBackend.vault.enabled }}
+              value: /vault/secrets/enterpriselicense.txt
+              {{- else }}
               value: /consul/license/{{ .Values.global.enterpriseLicense.secretKey }}
+              {{- end }}
             {{- end }}
             {{- end}}
           command:
@@ -148,7 +160,7 @@ spec:
             - name: aclconfig
               mountPath: /consul/aclconfig
             {{- else }}
-            {{- if (and .Values.global.enterpriseLicense.secretName .Values.global.enterpriseLicense.secretKey .Values.global.enterpriseLicense.enableLicenseAutoload) }}
+            {{- if (and .Values.global.enterpriseLicense.secretName .Values.global.enterpriseLicense.secretKey .Values.global.enterpriseLicense.enableLicenseAutoload (not .Values.global.secretsBackend.vault.enabled)) }}
             - name: consul-license
               mountPath: /consul/license
               readOnly: true

charts/consul/templates/enterprise-license-job.yaml

@@ -54,10 +54,14 @@ spec:
           image: "{{ default .Values.global.image .Values.server.image }}"
           env:
             - name: ENTERPRISE_LICENSE
+              {{- if .Values.global.secretsBackend.vault.enabled }}
+              value: /vault/secrets/enterpriselicense.txt
+              {{- else }}
               valueFrom:
                 secretKeyRef:
                   name: {{ .Values.global.enterpriseLicense.secretName }}
                   key: {{ .Values.global.enterpriseLicense.secretKey }}
+              {{- end }}
             - name: CONSUL_HTTP_ADDR
               {{- if .Values.global.tls.enabled }}
               value: https://{{ template "consul.fullname" . }}-server:8501