Add Vault Helm ent support, service discovery (hashicorp#250)

* Add Vault Helm ent support, service discovery * Fix unit test * Update test/acceptance/server-ha-enterprise-dr.bats Co-Authored-By: Theron Voran <[email protected]> * Update test/acceptance/server-ha-enterprise-dr.bats Co-Authored-By: Theron Voran <[email protected]> * Update test/acceptance/server-ha-enterprise-perf.bats Co-Authored-By: Theron Voran <[email protected]> * Update test/acceptance/server-ha-enterprise-perf.bats Co-Authored-By: Theron Voran <[email protected]> * Update values.yaml Co-Authored-By: Theron Voran <[email protected]> Co-authored-by: Theron Voran <[email protected]>
radudd · Apr 9, 2020 · 0628838 · 0628838
1 parent 45126a8
commit 0628838
Show file tree

Hide file tree

Showing 17 changed files with 530 additions and 64 deletions.
diff --git a/templates/_helpers.tpl b/templates/_helpers.tpl
@@ -133,10 +133,6 @@ Set's additional environment variables based on the mode.
             - name: VAULT_DEV_ROOT_TOKEN_ID
               value: "root"
   {{ end }}
-  {{ if and (eq .mode "ha") (eq (.Values.server.ha.raft.enabled | toString) "true") }}
-            - name: VAULT_CLUSTER_ADDR
-              value: "https://$(HOSTNAME).{{ template "vault.fullname" . }}-internal:8201"
-  {{ end }}
 {{- end -}}
 
 {{/*

diff --git a/templates/server-discovery-role.yaml b/templates/server-discovery-role.yaml
@@ -0,0 +1,19 @@
+{{ template "vault.mode" . }}
+{{- if ne .mode "external" }}
+{{- if and (eq .mode "ha" ) (eq (.Values.global.enabled | toString) "true") }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  namespace: {{ .Release.Namespace }}
+  name: {{ template "vault.fullname" . }}-discovery-role
+  labels:
+    helm.sh/chart: {{ include "vault.chart" . }}
+    app.kubernetes.io/name: {{ include "vault.name" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+rules:
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["get", "watch", "list", "update", "patch"]
+{{ end }}
+{{ end }}
diff --git a/templates/server-discovery-rolebinding.yaml b/templates/server-discovery-rolebinding.yaml
@@ -0,0 +1,23 @@
+{{ template "vault.mode" . }}
+{{- if ne .mode "external" }}
+{{- if and (eq .mode "ha" ) (eq (.Values.global.enabled | toString) "true") }}
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: RoleBinding
+metadata:
+  name: {{ template "vault.fullname" . }}-discovery-rolebinding
+  namespace: {{ .Release.Namespace }}
+  labels:
+    helm.sh/chart: {{ include "vault.chart" . }}
+    app.kubernetes.io/name: {{ include "vault.name" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ template "vault.fullname" . }}-discovery-role
+subjects:
+- kind: ServiceAccount
+  name: {{ template "vault.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+{{ end }}
+{{ end }}
diff --git a/templates/server-ha-active-service.yaml b/templates/server-ha-active-service.yaml
@@ -0,0 +1,35 @@
+{{ template "vault.mode" . }}
+{{- if ne .mode "external" }}
+{{- if and (eq .mode "ha" ) (and (eq (.Values.server.service.enabled | toString) "true" ) (eq (.Values.global.enabled | toString) "true")) }}
+# Service for active Vault pod
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "vault.fullname" . }}-active
+  namespace: {{ .Release.Namespace }}
+  labels:
+    helm.sh/chart: {{ include "vault.chart" . }}
+    app.kubernetes.io/name: {{ include "vault.name" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+  annotations:
+{{- if .Values.server.service.annotations }}
+{{ toYaml .Values.server.service.annotations | indent 4 }}
+{{- end }}
+spec:
+  type: ClusterIP
+  publishNotReadyAddresses: true
+  ports:
+    - name: http
+      port: 8200
+      targetPort: 8200
+    - name: internal
+      port: 8201
+      targetPort: 8201
+  selector:
+    app.kubernetes.io/name: {{ include "vault.name" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    component: server
+    vault-active: "true"
+{{- end }}
+{{- end }}
diff --git a/templates/server-ha-standby-service.yaml b/templates/server-ha-standby-service.yaml
@@ -0,0 +1,35 @@
+{{ template "vault.mode" . }}
+{{- if ne .mode "external" }}
+{{- if and (eq .mode "ha" ) (and (eq (.Values.server.service.enabled | toString) "true" ) (eq (.Values.global.enabled | toString) "true")) }}
+# Service for active Vault pod
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "vault.fullname" . }}-standby
+  namespace: {{ .Release.Namespace }}
+  labels:
+    helm.sh/chart: {{ include "vault.chart" . }}
+    app.kubernetes.io/name: {{ include "vault.name" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+  annotations:
+{{- if .Values.server.service.annotations }}
+{{ toYaml .Values.server.service.annotations | indent 4 }}
+{{- end }}
+spec:
+  type: ClusterIP
+  publishNotReadyAddresses: true
+  ports:
+    - name: http
+      port: 8200
+      targetPort: 8200
+    - name: internal
+      port: 8201
+      targetPort: 8201
+  selector:
+    app.kubernetes.io/name: {{ include "vault.name" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    component: server
+    vault-active: "false"
+{{- end }}
+{{- end }}
diff --git a/templates/server-statefulset.yaml b/templates/server-statefulset.yaml
@@ -68,6 +68,14 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: status.podIP
+            - name: VAULT_K8S_POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: VAULT_K8S_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
             - name: VAULT_ADDR
               value: "{{ include "vault.scheme" . }}://127.0.0.1:8200"
             - name: VAULT_API_ADDR
@@ -80,6 +88,8 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.name
+            - name: VAULT_CLUSTER_ADDR
+              value: "https://$(HOSTNAME).{{ template "vault.fullname" . }}-internal:8201"
             {{ template "vault.envs" . }}
             {{- include "vault.extraEnvironmentVars" .Values.server | nindent 12 }}
             {{- include "vault.extraSecretEnvironmentVars" .Values.server | nindent 12 }}

diff --git a/test/acceptance/injector.bats b/test/acceptance/injector.bats
@@ -45,11 +45,14 @@ load _helpers
 
 # Clean up
 teardown() {
-  echo "helm/pvc teardown"
-  helm delete vault
-  kubectl delete --all pvc
-  kubectl delete secret test 
-  kubectl delete job pgdump
-  kubectl delete deployment postgres
-  kubectl delete namespace acceptance
+  if [[ ${CLEANUP:-true} == "true" ]]
+  then
+      echo "helm/pvc teardown"
+      helm delete vault
+      kubectl delete --all pvc
+      kubectl delete secret test 
+      kubectl delete job pgdump
+      kubectl delete deployment postgres
+      kubectl delete namespace acceptance
+  fi
 }
diff --git a/test/acceptance/server-dev.bats b/test/acceptance/server-dev.bats
@@ -54,8 +54,11 @@ load _helpers
 
 # Clean up
 teardown() {
-  echo "helm/pvc teardown"
-  helm delete vault
-  kubectl delete --all pvc
-  kubectl delete namespace acceptance --ignore-not-found=true
+  if [[ ${CLEANUP:-true} == "true" ]]
+  then
+      echo "helm/pvc teardown"
+      helm delete vault
+      kubectl delete --all pvc
+      kubectl delete namespace acceptance --ignore-not-found=true
+  fi
 }
diff --git a/test/acceptance/server-ha-enterprise-dr.bats b/test/acceptance/server-ha-enterprise-dr.bats
@@ -0,0 +1,167 @@
+#!/usr/bin/env bats
+
+load _helpers
+
+@test "server/ha-enterprise-raft: testing DR deployment" {
+  cd `chart_dir`
+
+  helm install "$(name_prefix)-east" \
+    --set='server.image.repository=hashicorp/vault-enterprise' \
+    --set='server.image.tag=1.4.0_ent' \
+    --set='injector.enabled=false' \
+    --set='server.ha.enabled=true' \
+    --set='server.ha.raft.enabled=true' .
+  wait_for_running "$(name_prefix)-east-0"
+
+  # Sealed, not initialized
+  local sealed_status=$(kubectl exec "$(name_prefix)-east-0" -- vault status -format=json |
+    jq -r '.sealed' )
+  [ "${sealed_status}" == "true" ]
+
+  local init_status=$(kubectl exec "$(name_prefix)-east-0" -- vault status -format=json |
+    jq -r '.initialized')
+  [ "${init_status}" == "false" ]
+
+  # Vault Init
+  local init=$(kubectl exec -ti "$(name_prefix)-east-0" -- \
+    vault operator init -format=json -n 1 -t 1)
+
+  local primary_token=$(echo ${init} | jq -r '.unseal_keys_b64[0]')
+  [ "${primary_token}" != "" ]
+
+  local primary_root=$(echo ${init} | jq -r '.root_token')
+  [ "${primary_root}" != "" ]
+
+  kubectl exec -ti "$(name_prefix)-east-0" -- vault operator unseal ${primary_token}
+  wait_for_ready "$(name_prefix)-east-0"
+
+  sleep 10
+
+  # Vault Unseal
+  local pods=($(kubectl get pods --selector='app.kubernetes.io/name=vault' -o json | jq -r '.items[].metadata.name'))
+  for pod in "${pods[@]}"
+  do
+      if [[ ${pod?} != "$(name_prefix)-east-0" ]]
+      then
+          kubectl exec -ti ${pod} -- vault operator raft join http://$(name_prefix)-east-0.$(name_prefix)-east-internal:8200
+          kubectl exec -ti ${pod} -- vault operator unseal ${primary_token}
+          wait_for_ready "${pod}"
+      fi
+  done
+
+  # Sealed, not initialized
+  local sealed_status=$(kubectl exec "$(name_prefix)-east-0" -- vault status -format=json |
+    jq -r '.sealed' )
+  [ "${sealed_status}" == "false" ]
+
+  local init_status=$(kubectl exec "$(name_prefix)-east-0" -- vault status -format=json |
+    jq -r '.initialized')
+  [ "${init_status}" == "true" ]
+
+  kubectl exec "$(name_prefix)-east-0" -- vault login ${primary_root}
+
+  local raft_status=$(kubectl exec "$(name_prefix)-east-0" -- vault operator raft list-peers -format=json | 
+    jq -r '.data.config.servers | length')
+  [ "${raft_status}" == "3" ]
+
+  kubectl exec -ti $(name_prefix)-east-0 -- vault write -f sys/replication/dr/primary/enable primary_cluster_addr=https://$(name_prefix)-east-active:8201
+
+  local secondary=$(kubectl exec -ti "$(name_prefix)-east-0" -- vault write sys/replication/dr/primary/secondary-token id=secondary -format=json)
+  [ "${secondary}" != "" ]
+
+  local secondary_replica_token=$(echo ${secondary} | jq -r '.wrap_info.token')
+  [ "${secondary_replica_token}" != "" ]
+
+  # Install vault-west
+  helm install "$(name_prefix)-west" \
+    --set='injector.enabled=false' \
+    --set='server.image.repository=hashicorp/vault-enterprise' \
+    --set='server.image.tag=1.4.0_ent' \
+    --set='server.ha.enabled=true' \
+    --set='server.ha.raft.enabled=true' .
+  wait_for_running "$(name_prefix)-west-0"
+
+  # Sealed, not initialized
+  local sealed_status=$(kubectl exec "$(name_prefix)-west-0" -- vault status -format=json |
+    jq -r '.sealed' )
+  [ "${sealed_status}" == "true" ]
+
+  local init_status=$(kubectl exec "$(name_prefix)-west-0" -- vault status -format=json |
+    jq -r '.initialized')
+  [ "${init_status}" == "false" ]
+
+  # Vault Init
+  local init=$(kubectl exec -ti "$(name_prefix)-west-0" -- \
+    vault operator init -format=json -n 1 -t 1)
+
+  local secondary_token=$(echo ${init} | jq -r '.unseal_keys_b64[0]')
+  [ "${secondary_token}" != "" ]
+
+  local secondary_root=$(echo ${init} | jq -r '.root_token')
+  [ "${secondary_root}" != "" ]
+
+  kubectl exec -ti "$(name_prefix)-west-0" -- vault operator unseal ${secondary_token}
+  wait_for_ready "$(name_prefix)-west-0"
+
+  sleep 10
+
+  # Vault Unseal
+  local pods=($(kubectl get pods --selector='app.kubernetes.io/instance=vault-west' -o json | jq -r '.items[].metadata.name'))
+  for pod in "${pods[@]}"
+  do
+      if [[ ${pod?} != "$(name_prefix)-west-0" ]]
+      then
+          kubectl exec -ti ${pod} -- vault operator raft join http://$(name_prefix)-west-0.$(name_prefix)-west-internal:8200
+          kubectl exec -ti ${pod} -- vault operator unseal ${secondary_token}
+          wait_for_ready "${pod}"
+      fi
+  done
+
+  # Sealed, not initialized
+  local sealed_status=$(kubectl exec "$(name_prefix)-west-0" -- vault status -format=json |
+    jq -r '.sealed' )
+  [ "${sealed_status}" == "false" ]
+
+  local init_status=$(kubectl exec "$(name_prefix)-west-0" -- vault status -format=json |
+    jq -r '.initialized')
+  [ "${init_status}" == "true" ]
+
+  kubectl exec "$(name_prefix)-west-0" -- vault login ${secondary_root}
+
+  local raft_status=$(kubectl exec "$(name_prefix)-west-0" -- vault operator raft list-peers -format=json |
+    jq -r '.data.config.servers | length')
+  [ "${raft_status}" == "3" ]
+
+  kubectl exec -ti "$(name_prefix)-west-0" -- vault write sys/replication/dr/secondary/enable token=${secondary_replica_token}
+
+  sleep 10
+
+  local pods=($(kubectl get pods --selector='app.kubernetes.io/instance=vault-west' -o json | jq -r '.items[].metadata.name'))
+  for pod in "${pods[@]}"
+  do
+      if [[ ${pod?} != "$(name_prefix)-west-0" ]]
+      then
+          kubectl delete pod "${pod?}"
+          wait_for_running "${pod?}"
+          kubectl exec -ti ${pod} -- vault operator unseal ${primary_token}
+          wait_for_ready "${pod}"
+      fi
+  done
+}
+
+setup() {
+  kubectl delete namespace acceptance --ignore-not-found=true
+  kubectl create namespace acceptance
+  kubectl config set-context --current --namespace=acceptance
+}
+
+#cleanup
+teardown() {
+  if [[ ${CLEANUP:-true} == "true" ]]
+  then
+	  helm delete vault-east
+	  helm delete vault-west
+	  kubectl delete --all pvc
+	  kubectl delete namespace acceptance --ignore-not-found=true
+  fi
+}