From 0628838e6c66256ef9fde33bce0ccdccfdf5190f Mon Sep 17 00:00:00 2001
From: Jason O'Donnell <2160810+jasonodonnell@users.noreply.github.com>
Date: Thu, 9 Apr 2020 09:26:58 -0400
Subject: [PATCH] Add Vault Helm ent support, service discovery (#250)

* Add Vault Helm ent support, service discovery

* Fix unit test

* Update test/acceptance/server-ha-enterprise-dr.bats

Co-Authored-By: Theron Voran <tvoran@users.noreply.github.com>

* Update test/acceptance/server-ha-enterprise-dr.bats

Co-Authored-By: Theron Voran <tvoran@users.noreply.github.com>

* Update test/acceptance/server-ha-enterprise-perf.bats

Co-Authored-By: Theron Voran <tvoran@users.noreply.github.com>

* Update test/acceptance/server-ha-enterprise-perf.bats

Co-Authored-By: Theron Voran <tvoran@users.noreply.github.com>

* Update values.yaml

Co-Authored-By: Theron Voran <tvoran@users.noreply.github.com>

Co-authored-by: Theron Voran <tvoran@users.noreply.github.com>
---
 templates/_helpers.tpl                        |   4 -
 templates/server-discovery-role.yaml          |  19 ++
 templates/server-discovery-rolebinding.yaml   |  23 +++
 templates/server-ha-active-service.yaml       |  35 ++++
 templates/server-ha-standby-service.yaml      |  35 ++++
 templates/server-statefulset.yaml             |  10 ++
 test/acceptance/injector.bats                 |  17 +-
 test/acceptance/server-dev.bats               |  11 +-
 test/acceptance/server-ha-enterprise-dr.bats  | 167 ++++++++++++++++++
 .../acceptance/server-ha-enterprise-perf.bats | 165 +++++++++++++++++
 test/acceptance/server-ha-raft.bats           |  11 +-
 test/acceptance/server-ha.bats                |  11 +-
 test/acceptance/server.bats                   |  11 +-
 test/unit/server-dev-statefulset.bats         |  20 +--
 test/unit/server-ha-statefulset.bats          |  32 ++--
 test/unit/server-statefulset.bats             |  16 +-
 values.yaml                                   |   7 +-
 17 files changed, 530 insertions(+), 64 deletions(-)
 create mode 100644 templates/server-discovery-role.yaml
 create mode 100644 templates/server-discovery-rolebinding.yaml
 create mode 100644 templates/server-ha-active-service.yaml
 create mode 100644 templates/server-ha-standby-service.yaml
 create mode 100644 test/acceptance/server-ha-enterprise-dr.bats
 create mode 100644 test/acceptance/server-ha-enterprise-perf.bats

diff --git a/templates/_helpers.tpl b/templates/_helpers.tpl
index 9a2203892..89d23d867 100644
--- a/templates/_helpers.tpl
+++ b/templates/_helpers.tpl
@@ -133,10 +133,6 @@ Set's additional environment variables based on the mode.
             - name: VAULT_DEV_ROOT_TOKEN_ID
               value: "root"
   {{ end }}
-  {{ if and (eq .mode "ha") (eq (.Values.server.ha.raft.enabled | toString) "true") }}
-            - name: VAULT_CLUSTER_ADDR
-              value: "https://$(HOSTNAME).{{ template "vault.fullname" . }}-internal:8201"
-  {{ end }}
 {{- end -}}
 
 {{/*
diff --git a/templates/server-discovery-role.yaml b/templates/server-discovery-role.yaml
new file mode 100644
index 000000000..4a39cec21
--- /dev/null
+++ b/templates/server-discovery-role.yaml
@@ -0,0 +1,19 @@
+{{ template "vault.mode" . }}
+{{- if ne .mode "external" }}
+{{- if and (eq .mode "ha" ) (eq (.Values.global.enabled | toString) "true") }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  namespace: {{ .Release.Namespace }}
+  name: {{ template "vault.fullname" . }}-discovery-role
+  labels:
+    helm.sh/chart: {{ include "vault.chart" . }}
+    app.kubernetes.io/name: {{ include "vault.name" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+rules:
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["get", "watch", "list", "update", "patch"]
+{{ end }}
+{{ end }}
diff --git a/templates/server-discovery-rolebinding.yaml b/templates/server-discovery-rolebinding.yaml
new file mode 100644
index 000000000..f9494b475
--- /dev/null
+++ b/templates/server-discovery-rolebinding.yaml
@@ -0,0 +1,23 @@
+{{ template "vault.mode" . }}
+{{- if ne .mode "external" }}
+{{- if and (eq .mode "ha" ) (eq (.Values.global.enabled | toString) "true") }}
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: RoleBinding
+metadata:
+  name: {{ template "vault.fullname" . }}-discovery-rolebinding
+  namespace: {{ .Release.Namespace }}
+  labels:
+    helm.sh/chart: {{ include "vault.chart" . }}
+    app.kubernetes.io/name: {{ include "vault.name" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ template "vault.fullname" . }}-discovery-role
+subjects:
+- kind: ServiceAccount
+  name: {{ template "vault.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+{{ end }}
+{{ end }}
diff --git a/templates/server-ha-active-service.yaml b/templates/server-ha-active-service.yaml
new file mode 100644
index 000000000..1af8520a6
--- /dev/null
+++ b/templates/server-ha-active-service.yaml
@@ -0,0 +1,35 @@
+{{ template "vault.mode" . }}
+{{- if ne .mode "external" }}
+{{- if and (eq .mode "ha" ) (and (eq (.Values.server.service.enabled | toString) "true" ) (eq (.Values.global.enabled | toString) "true")) }}
+# Service for active Vault pod
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "vault.fullname" . }}-active
+  namespace: {{ .Release.Namespace }}
+  labels:
+    helm.sh/chart: {{ include "vault.chart" . }}
+    app.kubernetes.io/name: {{ include "vault.name" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+  annotations:
+{{- if .Values.server.service.annotations }}
+{{ toYaml .Values.server.service.annotations | indent 4 }}
+{{- end }}
+spec:
+  type: ClusterIP
+  publishNotReadyAddresses: true
+  ports:
+    - name: http
+      port: 8200
+      targetPort: 8200
+    - name: internal
+      port: 8201
+      targetPort: 8201
+  selector:
+    app.kubernetes.io/name: {{ include "vault.name" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    component: server
+    vault-active: "true"
+{{- end }}
+{{- end }}
diff --git a/templates/server-ha-standby-service.yaml b/templates/server-ha-standby-service.yaml
new file mode 100644
index 000000000..2dd75228f
--- /dev/null
+++ b/templates/server-ha-standby-service.yaml
@@ -0,0 +1,35 @@
+{{ template "vault.mode" . }}
+{{- if ne .mode "external" }}
+{{- if and (eq .mode "ha" ) (and (eq (.Values.server.service.enabled | toString) "true" ) (eq (.Values.global.enabled | toString) "true")) }}
+# Service for active Vault pod
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "vault.fullname" . }}-standby
+  namespace: {{ .Release.Namespace }}
+  labels:
+    helm.sh/chart: {{ include "vault.chart" . }}
+    app.kubernetes.io/name: {{ include "vault.name" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+  annotations:
+{{- if .Values.server.service.annotations }}
+{{ toYaml .Values.server.service.annotations | indent 4 }}
+{{- end }}
+spec:
+  type: ClusterIP
+  publishNotReadyAddresses: true
+  ports:
+    - name: http
+      port: 8200
+      targetPort: 8200
+    - name: internal
+      port: 8201
+      targetPort: 8201
+  selector:
+    app.kubernetes.io/name: {{ include "vault.name" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    component: server
+    vault-active: "false"
+{{- end }}
+{{- end }}
diff --git a/templates/server-statefulset.yaml b/templates/server-statefulset.yaml
index d2b2ac154..255a84429 100644
--- a/templates/server-statefulset.yaml
+++ b/templates/server-statefulset.yaml
@@ -68,6 +68,14 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: status.podIP
+            - name: VAULT_K8S_POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: VAULT_K8S_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
             - name: VAULT_ADDR
               value: "{{ include "vault.scheme" . }}://127.0.0.1:8200"
             - name: VAULT_API_ADDR
@@ -80,6 +88,8 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.name
+            - name: VAULT_CLUSTER_ADDR
+              value: "https://$(HOSTNAME).{{ template "vault.fullname" . }}-internal:8201"
             {{ template "vault.envs" . }}
             {{- include "vault.extraEnvironmentVars" .Values.server | nindent 12 }}
             {{- include "vault.extraSecretEnvironmentVars" .Values.server | nindent 12 }}
diff --git a/test/acceptance/injector.bats b/test/acceptance/injector.bats
index 2fdb7a576..e7fb393ac 100644
--- a/test/acceptance/injector.bats
+++ b/test/acceptance/injector.bats
@@ -45,11 +45,14 @@ load _helpers
 
 # Clean up
 teardown() {
-  echo "helm/pvc teardown"
-  helm delete vault
-  kubectl delete --all pvc
-  kubectl delete secret test 
-  kubectl delete job pgdump
-  kubectl delete deployment postgres
-  kubectl delete namespace acceptance
+  if [[ ${CLEANUP:-true} == "true" ]]
+  then
+      echo "helm/pvc teardown"
+      helm delete vault
+      kubectl delete --all pvc
+      kubectl delete secret test 
+      kubectl delete job pgdump
+      kubectl delete deployment postgres
+      kubectl delete namespace acceptance
+  fi
 }
diff --git a/test/acceptance/server-dev.bats b/test/acceptance/server-dev.bats
index 05f36618f..ffda94640 100644
--- a/test/acceptance/server-dev.bats
+++ b/test/acceptance/server-dev.bats
@@ -54,8 +54,11 @@ load _helpers
 
 # Clean up
 teardown() {
-  echo "helm/pvc teardown"
-  helm delete vault
-  kubectl delete --all pvc
-  kubectl delete namespace acceptance --ignore-not-found=true
+  if [[ ${CLEANUP:-true} == "true" ]]
+  then
+      echo "helm/pvc teardown"
+      helm delete vault
+      kubectl delete --all pvc
+      kubectl delete namespace acceptance --ignore-not-found=true
+  fi
 }
diff --git a/test/acceptance/server-ha-enterprise-dr.bats b/test/acceptance/server-ha-enterprise-dr.bats
new file mode 100644
index 000000000..35348e3c5
--- /dev/null
+++ b/test/acceptance/server-ha-enterprise-dr.bats
@@ -0,0 +1,167 @@
+#!/usr/bin/env bats
+
+load _helpers
+
+@test "server/ha-enterprise-raft: testing DR deployment" {
+  cd `chart_dir`
+
+  helm install "$(name_prefix)-east" \
+    --set='server.image.repository=hashicorp/vault-enterprise' \
+    --set='server.image.tag=1.4.0_ent' \
+    --set='injector.enabled=false' \
+    --set='server.ha.enabled=true' \
+    --set='server.ha.raft.enabled=true' .
+  wait_for_running "$(name_prefix)-east-0"
+
+  # Sealed, not initialized
+  local sealed_status=$(kubectl exec "$(name_prefix)-east-0" -- vault status -format=json |
+    jq -r '.sealed' )
+  [ "${sealed_status}" == "true" ]
+
+  local init_status=$(kubectl exec "$(name_prefix)-east-0" -- vault status -format=json |
+    jq -r '.initialized')
+  [ "${init_status}" == "false" ]
+
+  # Vault Init
+  local init=$(kubectl exec -ti "$(name_prefix)-east-0" -- \
+    vault operator init -format=json -n 1 -t 1)
+
+  local primary_token=$(echo ${init} | jq -r '.unseal_keys_b64[0]')
+  [ "${primary_token}" != "" ]
+  
+  local primary_root=$(echo ${init} | jq -r '.root_token')
+  [ "${primary_root}" != "" ]
+
+  kubectl exec -ti "$(name_prefix)-east-0" -- vault operator unseal ${primary_token}
+  wait_for_ready "$(name_prefix)-east-0"
+
+  sleep 10
+
+  # Vault Unseal
+  local pods=($(kubectl get pods --selector='app.kubernetes.io/name=vault' -o json | jq -r '.items[].metadata.name'))
+  for pod in "${pods[@]}"
+  do
+      if [[ ${pod?} != "$(name_prefix)-east-0" ]]
+      then
+          kubectl exec -ti ${pod} -- vault operator raft join http://$(name_prefix)-east-0.$(name_prefix)-east-internal:8200
+          kubectl exec -ti ${pod} -- vault operator unseal ${primary_token}
+          wait_for_ready "${pod}"
+      fi
+  done
+
+  # Sealed, not initialized
+  local sealed_status=$(kubectl exec "$(name_prefix)-east-0" -- vault status -format=json |
+    jq -r '.sealed' )
+  [ "${sealed_status}" == "false" ]
+
+  local init_status=$(kubectl exec "$(name_prefix)-east-0" -- vault status -format=json |
+    jq -r '.initialized')
+  [ "${init_status}" == "true" ]
+
+  kubectl exec "$(name_prefix)-east-0" -- vault login ${primary_root}
+
+  local raft_status=$(kubectl exec "$(name_prefix)-east-0" -- vault operator raft list-peers -format=json | 
+    jq -r '.data.config.servers | length')
+  [ "${raft_status}" == "3" ]
+
+  kubectl exec -ti $(name_prefix)-east-0 -- vault write -f sys/replication/dr/primary/enable primary_cluster_addr=https://$(name_prefix)-east-active:8201
+
+  local secondary=$(kubectl exec -ti "$(name_prefix)-east-0" -- vault write sys/replication/dr/primary/secondary-token id=secondary -format=json)
+  [ "${secondary}" != "" ]
+
+  local secondary_replica_token=$(echo ${secondary} | jq -r '.wrap_info.token')
+  [ "${secondary_replica_token}" != "" ]
+
+  # Install vault-west
+  helm install "$(name_prefix)-west" \
+    --set='injector.enabled=false' \
+    --set='server.image.repository=hashicorp/vault-enterprise' \
+    --set='server.image.tag=1.4.0_ent' \
+    --set='server.ha.enabled=true' \
+    --set='server.ha.raft.enabled=true' .
+  wait_for_running "$(name_prefix)-west-0"
+
+  # Sealed, not initialized
+  local sealed_status=$(kubectl exec "$(name_prefix)-west-0" -- vault status -format=json |
+    jq -r '.sealed' )
+  [ "${sealed_status}" == "true" ]
+
+  local init_status=$(kubectl exec "$(name_prefix)-west-0" -- vault status -format=json |
+    jq -r '.initialized')
+  [ "${init_status}" == "false" ]
+
+  # Vault Init
+  local init=$(kubectl exec -ti "$(name_prefix)-west-0" -- \
+    vault operator init -format=json -n 1 -t 1)
+
+  local secondary_token=$(echo ${init} | jq -r '.unseal_keys_b64[0]')
+  [ "${secondary_token}" != "" ]
+
+  local secondary_root=$(echo ${init} | jq -r '.root_token')
+  [ "${secondary_root}" != "" ]
+
+  kubectl exec -ti "$(name_prefix)-west-0" -- vault operator unseal ${secondary_token}
+  wait_for_ready "$(name_prefix)-west-0"
+
+  sleep 10
+
+  # Vault Unseal
+  local pods=($(kubectl get pods --selector='app.kubernetes.io/instance=vault-west' -o json | jq -r '.items[].metadata.name'))
+  for pod in "${pods[@]}"
+  do
+      if [[ ${pod?} != "$(name_prefix)-west-0" ]]
+      then
+          kubectl exec -ti ${pod} -- vault operator raft join http://$(name_prefix)-west-0.$(name_prefix)-west-internal:8200
+          kubectl exec -ti ${pod} -- vault operator unseal ${secondary_token}
+          wait_for_ready "${pod}"
+      fi
+  done
+
+  # Sealed, not initialized
+  local sealed_status=$(kubectl exec "$(name_prefix)-west-0" -- vault status -format=json |
+    jq -r '.sealed' )
+  [ "${sealed_status}" == "false" ]
+
+  local init_status=$(kubectl exec "$(name_prefix)-west-0" -- vault status -format=json |
+    jq -r '.initialized')
+  [ "${init_status}" == "true" ]
+
+  kubectl exec "$(name_prefix)-west-0" -- vault login ${secondary_root}
+
+  local raft_status=$(kubectl exec "$(name_prefix)-west-0" -- vault operator raft list-peers -format=json |
+    jq -r '.data.config.servers | length')
+  [ "${raft_status}" == "3" ]
+
+  kubectl exec -ti "$(name_prefix)-west-0" -- vault write sys/replication/dr/secondary/enable token=${secondary_replica_token}
+
+  sleep 10
+
+  local pods=($(kubectl get pods --selector='app.kubernetes.io/instance=vault-west' -o json | jq -r '.items[].metadata.name'))
+  for pod in "${pods[@]}"
+  do
+      if [[ ${pod?} != "$(name_prefix)-west-0" ]]
+      then
+          kubectl delete pod "${pod?}"
+          wait_for_running "${pod?}"
+          kubectl exec -ti ${pod} -- vault operator unseal ${primary_token}
+          wait_for_ready "${pod}"
+      fi
+  done
+}
+
+setup() {
+  kubectl delete namespace acceptance --ignore-not-found=true
+  kubectl create namespace acceptance
+  kubectl config set-context --current --namespace=acceptance
+}
+
+#cleanup
+teardown() {
+  if [[ ${CLEANUP:-true} == "true" ]]
+  then
+	  helm delete vault-east
+	  helm delete vault-west
+	  kubectl delete --all pvc
+	  kubectl delete namespace acceptance --ignore-not-found=true
+  fi
+}
diff --git a/test/acceptance/server-ha-enterprise-perf.bats b/test/acceptance/server-ha-enterprise-perf.bats
new file mode 100644
index 000000000..6543663ec
--- /dev/null
+++ b/test/acceptance/server-ha-enterprise-perf.bats
@@ -0,0 +1,165 @@
+#!/usr/bin/env bats
+
+load _helpers
+
+@test "server/ha-enterprise-raft: testing performance replica deployment" {
+  cd `chart_dir`
+
+  helm install "$(name_prefix)-east" \
+    --set='injector.enabled=false' \
+    --set='server.image.repository=hashicorp/vault-enterprise' \
+    --set='server.image.tag=1.4.0_ent' \
+    --set='server.ha.enabled=true' \
+    --set='server.ha.raft.enabled=true' .
+  wait_for_running "$(name_prefix)-east-0"
+
+  # Sealed, not initialized
+  local sealed_status=$(kubectl exec "$(name_prefix)-east-0" -- vault status -format=json |
+    jq -r '.sealed' )
+  [ "${sealed_status}" == "true" ]
+
+  local init_status=$(kubectl exec "$(name_prefix)-east-0" -- vault status -format=json |
+    jq -r '.initialized')
+  [ "${init_status}" == "false" ]
+
+  # Vault Init
+  local init=$(kubectl exec -ti "$(name_prefix)-east-0" -- \
+    vault operator init -format=json -n 1 -t 1)
+
+  local primary_token=$(echo ${init} | jq -r '.unseal_keys_b64[0]')
+  [ "${primary_token}" != "" ]
+  
+  local primary_root=$(echo ${init} | jq -r '.root_token')
+  [ "${primary_root}" != "" ]
+
+  kubectl exec -ti "$(name_prefix)-east-0" -- vault operator unseal ${primary_token}
+  wait_for_ready "$(name_prefix)-east-0"
+
+  sleep 10
+
+  # Vault Unseal
+  local pods=($(kubectl get pods --selector='app.kubernetes.io/name=vault' -o json | jq -r '.items[].metadata.name'))
+  for pod in "${pods[@]}"
+  do
+      if [[ ${pod?} != "$(name_prefix)-east-0" ]]
+      then
+          kubectl exec -ti ${pod} -- vault operator raft join http://$(name_prefix)-east-0.$(name_prefix)-east-internal:8200
+          kubectl exec -ti ${pod} -- vault operator unseal ${primary_token}
+          wait_for_ready "${pod}"
+      fi
+  done
+
+  # Sealed, not initialized
+  local sealed_status=$(kubectl exec "$(name_prefix)-east-0" -- vault status -format=json |
+    jq -r '.sealed' )
+  [ "${sealed_status}" == "false" ]
+
+  local init_status=$(kubectl exec "$(name_prefix)-east-0" -- vault status -format=json |
+    jq -r '.initialized')
+  [ "${init_status}" == "true" ]
+
+  kubectl exec "$(name_prefix)-east-0" -- vault login ${primary_root}
+
+  local raft_status=$(kubectl exec "$(name_prefix)-east-0" -- vault operator raft list-peers -format=json | 
+    jq -r '.data.config.servers | length')
+  [ "${raft_status}" == "3" ]
+
+  kubectl exec -ti $(name_prefix)-east-0 -- vault write -f sys/replication/performance/primary/enable primary_cluster_addr=https://$(name_prefix)-east-active:8201
+
+  local secondary=$(kubectl exec -ti "$(name_prefix)-east-0" -- vault write sys/replication/performance/primary/secondary-token id=secondary -format=json)
+  [ "${secondary}" != "" ]
+
+  local secondary_replica_token=$(echo ${secondary} | jq -r '.wrap_info.token')
+  [ "${secondary_replica_token}" != "" ]
+
+  # Install vault-west
+  helm install "$(name_prefix)-west" \
+    --set='injector.enabled=false' \
+    --set='server.image.repository=hashicorp/vault-enterprise' \
+    --set='server.image.tag=1.4.0_ent' \
+    --set='server.ha.enabled=true' \
+    --set='server.ha.raft.enabled=true' .
+  wait_for_running "$(name_prefix)-west-0"
+
+  # Sealed, not initialized
+  local sealed_status=$(kubectl exec "$(name_prefix)-west-0" -- vault status -format=json |
+    jq -r '.sealed' )
+  [ "${sealed_status}" == "true" ]
+
+  local init_status=$(kubectl exec "$(name_prefix)-west-0" -- vault status -format=json |
+    jq -r '.initialized')
+  [ "${init_status}" == "false" ]
+
+  # Vault Init
+  local init=$(kubectl exec -ti "$(name_prefix)-west-0" -- \
+    vault operator init -format=json -n 1 -t 1)
+
+  local secondary_token=$(echo ${init} | jq -r '.unseal_keys_b64[0]')
+  [ "${secondary_token}" != "" ]
+
+  local secondary_root=$(echo ${init} | jq -r '.root_token')
+  [ "${secondary_root}" != "" ]
+
+  kubectl exec -ti "$(name_prefix)-west-0" -- vault operator unseal ${secondary_token}
+  wait_for_ready "$(name_prefix)-west-0"
+
+  sleep 10
+
+  # Vault Unseal
+  local pods=($(kubectl get pods --selector='app.kubernetes.io/instance=vault-west' -o json | jq -r '.items[].metadata.name'))
+  for pod in "${pods[@]}"
+  do
+      if [[ ${pod?} != "$(name_prefix)-west-0" ]]
+      then
+          kubectl exec -ti ${pod} -- vault operator raft join http://$(name_prefix)-west-0.$(name_prefix)-west-internal:8200
+          kubectl exec -ti ${pod} -- vault operator unseal ${secondary_token}
+          wait_for_ready "${pod}"
+      fi
+  done
+
+  # Sealed, not initialized
+  local sealed_status=$(kubectl exec "$(name_prefix)-west-0" -- vault status -format=json |
+    jq -r '.sealed' )
+  [ "${sealed_status}" == "false" ]
+
+  local init_status=$(kubectl exec "$(name_prefix)-west-0" -- vault status -format=json |
+    jq -r '.initialized')
+  [ "${init_status}" == "true" ]
+
+  kubectl exec "$(name_prefix)-west-0" -- vault login ${secondary_root}
+
+  local raft_status=$(kubectl exec "$(name_prefix)-west-0" -- vault operator raft list-peers -format=json |
+    jq -r '.data.config.servers | length')
+  [ "${raft_status}" == "3" ]
+
+  kubectl exec -ti "$(name_prefix)-west-0" -- vault write sys/replication/performance/secondary/enable token=${secondary_replica_token}
+
+  sleep 10
+
+  local pods=($(kubectl get pods --selector='app.kubernetes.io/instance=vault-west' -o json | jq -r '.items[].metadata.name'))
+  for pod in "${pods[@]}"
+  do
+      if [[ ${pod?} != "$(name_prefix)-west-0" ]]
+      then
+          kubectl exec -ti ${pod} -- vault operator unseal ${primary_token}
+          wait_for_ready "${pod}"
+      fi
+  done
+}
+
+setup() {
+  kubectl delete namespace acceptance --ignore-not-found=true
+  kubectl create namespace acceptance
+  kubectl config set-context --current --namespace=acceptance
+}
+
+#cleanup
+teardown() {
+  if [[ ${CLEANUP:-true} == "true" ]]
+  then
+      helm delete vault-east
+      helm delete vault-west
+      kubectl delete --all pvc
+      kubectl delete namespace acceptance --ignore-not-found=true
+  fi
+}
diff --git a/test/acceptance/server-ha-raft.bats b/test/acceptance/server-ha-raft.bats
index 17951b8ad..a411f3c93 100644
--- a/test/acceptance/server-ha-raft.bats
+++ b/test/acceptance/server-ha-raft.bats
@@ -102,7 +102,7 @@ load _helpers
 
   kubectl exec "$(name_prefix)-0" -- vault login ${root}
 
-  local raft_status=$(kubectl exec "$(name_prefix)-0" -- vault operator raft configuration -format=json | 
+  local raft_status=$(kubectl exec "$(name_prefix)-0" -- vault operator raft list-peers -format=json | 
     jq -r '.data.config.servers | length')
   [ "${raft_status}" == "3" ]
 }
@@ -115,7 +115,10 @@ setup() {
 
 #cleanup
 teardown() {
-  helm delete vault
-  kubectl delete --all pvc
-  kubectl delete namespace acceptance --ignore-not-found=true
+  if [[ ${CLEANUP:-true} == "true" ]]
+  then
+      helm delete vault
+      kubectl delete --all pvc
+      kubectl delete namespace acceptance --ignore-not-found=true
+  fi
 }
diff --git a/test/acceptance/server-ha.bats b/test/acceptance/server-ha.bats
index f29e31f9d..74a3c117c 100644
--- a/test/acceptance/server-ha.bats
+++ b/test/acceptance/server-ha.bats
@@ -103,8 +103,11 @@ setup() {
 
 #cleanup
 teardown() {
-  helm delete vault
-  helm delete consul
-  kubectl delete --all pvc
-  kubectl delete namespace acceptance --ignore-not-found=true
+  if [[ ${CLEANUP:-true} == "true" ]]
+  then
+      helm delete vault
+      helm delete consul
+      kubectl delete --all pvc
+      kubectl delete namespace acceptance --ignore-not-found=true
+  fi
 }
diff --git a/test/acceptance/server.bats b/test/acceptance/server.bats
index d8edbd51c..beb2fa2a2 100644
--- a/test/acceptance/server.bats
+++ b/test/acceptance/server.bats
@@ -111,8 +111,11 @@ load _helpers
 
 # Clean up
 teardown() {
-  echo "helm/pvc teardown"
-  helm delete vault
-  kubectl delete --all pvc
-  kubectl delete namespace acceptance --ignore-not-found=true
+  if [[ ${CLEANUP:-true} == "true" ]]
+  then
+      echo "helm/pvc teardown"
+      helm delete vault
+      kubectl delete --all pvc
+      kubectl delete namespace acceptance --ignore-not-found=true
+  fi
 }
diff --git a/test/unit/server-dev-statefulset.bats b/test/unit/server-dev-statefulset.bats
index 5ce3405b2..3b38eab41 100755
--- a/test/unit/server-dev-statefulset.bats
+++ b/test/unit/server-dev-statefulset.bats
@@ -249,19 +249,19 @@ load _helpers
       yq -r '.spec.template.spec.containers[0].env' | tee /dev/stderr)
 
   local actual=$(echo $object |
-     yq -r '.[8].name' | tee /dev/stderr)
+     yq -r '.[11].name' | tee /dev/stderr)
   [ "${actual}" = "FOO" ]
 
   local actual=$(echo $object |
-      yq -r '.[8].value' | tee /dev/stderr)
+      yq -r '.[11].value' | tee /dev/stderr)
   [ "${actual}" = "bar" ]
 
   local actual=$(echo $object |
-      yq -r '.[9].name' | tee /dev/stderr)
+      yq -r '.[12].name' | tee /dev/stderr)
   [ "${actual}" = "FOOBAR" ]
 
   local actual=$(echo $object |
-      yq -r '.[9].value' | tee /dev/stderr)
+      yq -r '.[12].value' | tee /dev/stderr)
   [ "${actual}" = "foobar" ]
 }
 
@@ -282,23 +282,23 @@ load _helpers
       yq -r '.spec.template.spec.containers[0].env' | tee /dev/stderr)
 
   local actual=$(echo $object |
-      yq -r '.[7].name' | tee /dev/stderr)
+      yq -r '.[10].name' | tee /dev/stderr)
   [ "${actual}" = "ENV_FOO_0" ]
   local actual=$(echo $object |
-      yq -r '.[7].valueFrom.secretKeyRef.name' | tee /dev/stderr)
+      yq -r '.[10].valueFrom.secretKeyRef.name' | tee /dev/stderr)
   [ "${actual}" = "secret_name_0" ]
   local actual=$(echo $object |
-      yq -r '.[7].valueFrom.secretKeyRef.key' | tee /dev/stderr)
+      yq -r '.[10].valueFrom.secretKeyRef.key' | tee /dev/stderr)
   [ "${actual}" = "secret_key_0" ]
 
   local actual=$(echo $object |
-      yq -r '.[8].name' | tee /dev/stderr)
+      yq -r '.[11].name' | tee /dev/stderr)
   [ "${actual}" = "ENV_FOO_1" ]
   local actual=$(echo $object |
-      yq -r '.[8].valueFrom.secretKeyRef.name' | tee /dev/stderr)
+      yq -r '.[11].valueFrom.secretKeyRef.name' | tee /dev/stderr)
   [ "${actual}" = "secret_name_1" ]
   local actual=$(echo $object |
-      yq -r '.[8].valueFrom.secretKeyRef.key' | tee /dev/stderr)
+      yq -r '.[11].valueFrom.secretKeyRef.key' | tee /dev/stderr)
   [ "${actual}" = "secret_key_1" ]
 }
 
diff --git a/test/unit/server-ha-statefulset.bats b/test/unit/server-ha-statefulset.bats
index 8e19ae0a0..e93bf3104 100755
--- a/test/unit/server-ha-statefulset.bats
+++ b/test/unit/server-ha-statefulset.bats
@@ -71,11 +71,11 @@ load _helpers
       yq -r '.spec.template.spec.containers[0].env' | tee /dev/stderr)
 
   local actual=$(echo $object |
-     yq -r '.[2].name' | tee /dev/stderr)
+     yq -r '.[4].name' | tee /dev/stderr)
   [ "${actual}" = "VAULT_ADDR" ]
 
   local actual=$(echo $object |
-     yq -r '.[2].value' | tee /dev/stderr)
+     yq -r '.[4].value' | tee /dev/stderr)
   [ "${actual}" = "http://127.0.0.1:8200" ]
 }
 @test "server/ha-StatefulSet: tls enabled" {
@@ -87,11 +87,11 @@ load _helpers
       yq -r '.spec.template.spec.containers[0].env' | tee /dev/stderr)
 
   local actual=$(echo $object |
-     yq -r '.[2].name' | tee /dev/stderr)
+     yq -r '.[4].name' | tee /dev/stderr)
   [ "${actual}" = "VAULT_ADDR" ]
 
   local actual=$(echo $object |
-     yq -r '.[2].value' | tee /dev/stderr)
+     yq -r '.[4].value' | tee /dev/stderr)
   [ "${actual}" = "https://127.0.0.1:8200" ]
 }
 
@@ -349,19 +349,19 @@ load _helpers
       yq -r '.spec.template.spec.containers[0].env' | tee /dev/stderr)
 
   local actual=$(echo $object |
-     yq -r '.[7].name' | tee /dev/stderr)
+     yq -r '.[10].name' | tee /dev/stderr)
   [ "${actual}" = "FOO" ]
 
   local actual=$(echo $object |
-      yq -r '.[7].value' | tee /dev/stderr)
+      yq -r '.[10].value' | tee /dev/stderr)
   [ "${actual}" = "bar" ]
 
   local actual=$(echo $object |
-      yq -r '.[8].name' | tee /dev/stderr)
+      yq -r '.[11].name' | tee /dev/stderr)
   [ "${actual}" = "FOOBAR" ]
 
   local actual=$(echo $object |
-      yq -r '.[8].value' | tee /dev/stderr)
+      yq -r '.[11].value' | tee /dev/stderr)
   [ "${actual}" = "foobar" ]
 }
 
@@ -383,23 +383,23 @@ load _helpers
       yq -r '.spec.template.spec.containers[0].env' | tee /dev/stderr)
 
   local actual=$(echo $object |
-      yq -r '.[7].name' | tee /dev/stderr)
+      yq -r '.[10].name' | tee /dev/stderr)
   [ "${actual}" = "ENV_FOO_0" ]
   local actual=$(echo $object |
-      yq -r '.[7].valueFrom.secretKeyRef.name' | tee /dev/stderr)
+      yq -r '.[10].valueFrom.secretKeyRef.name' | tee /dev/stderr)
   [ "${actual}" = "secret_name_0" ]
   local actual=$(echo $object |
-      yq -r '.[7].valueFrom.secretKeyRef.key' | tee /dev/stderr)
+      yq -r '.[10].valueFrom.secretKeyRef.key' | tee /dev/stderr)
   [ "${actual}" = "secret_key_0" ]
 
   local actual=$(echo $object |
-      yq -r '.[8].name' | tee /dev/stderr)
+      yq -r '.[11].name' | tee /dev/stderr)
   [ "${actual}" = "ENV_FOO_1" ]
   local actual=$(echo $object |
-      yq -r '.[8].valueFrom.secretKeyRef.name' | tee /dev/stderr)
+      yq -r '.[11].valueFrom.secretKeyRef.name' | tee /dev/stderr)
   [ "${actual}" = "secret_name_1" ]
   local actual=$(echo $object |
-      yq -r '.[8].valueFrom.secretKeyRef.key' | tee /dev/stderr)
+      yq -r '.[11].valueFrom.secretKeyRef.key' | tee /dev/stderr)
   [ "${actual}" = "secret_key_1" ]
 }
 
@@ -417,11 +417,11 @@ load _helpers
       yq -r '.spec.template.spec.containers[0].env' | tee /dev/stderr)
   
   local actual=$(echo $object |
-     yq -r '.[7].name' | tee /dev/stderr)
+     yq -r '.[9].name' | tee /dev/stderr)
   [ "${actual}" = "VAULT_CLUSTER_ADDR" ]
 
   local actual=$(echo $object |
-     yq -r '.[7].value' | tee /dev/stderr)
+     yq -r '.[9].value' | tee /dev/stderr)
   [ "${actual}" = 'https://$(HOSTNAME).RELEASE-NAME-vault-internal:8201' ]
 }
 
diff --git a/test/unit/server-statefulset.bats b/test/unit/server-statefulset.bats
index 35ebf214f..b0dc6fbd1 100755
--- a/test/unit/server-statefulset.bats
+++ b/test/unit/server-statefulset.bats
@@ -384,19 +384,19 @@ load _helpers
       yq -r '.spec.template.spec.containers[0].env' | tee /dev/stderr)
 
   local actual=$(echo $object |
-     yq -r '.[7].name' | tee /dev/stderr)
+     yq -r '.[10].name' | tee /dev/stderr)
   [ "${actual}" = "FOO" ]
 
   local actual=$(echo $object |
-      yq -r '.[7].value' | tee /dev/stderr)
+      yq -r '.[10].value' | tee /dev/stderr)
   [ "${actual}" = "bar" ]
 
   local actual=$(echo $object |
-      yq -r '.[8].name' | tee /dev/stderr)
+      yq -r '.[11].name' | tee /dev/stderr)
   [ "${actual}" = "FOOBAR" ]
 
   local actual=$(echo $object |
-      yq -r '.[8].value' | tee /dev/stderr)
+      yq -r '.[11].value' | tee /dev/stderr)
   [ "${actual}" = "foobar" ]
 
   local object=$(helm template \
@@ -407,19 +407,19 @@ load _helpers
       yq -r '.spec.template.spec.containers[0].env' | tee /dev/stderr)
 
   local actual=$(echo $object |
-     yq -r '.[7].name' | tee /dev/stderr)
+     yq -r '.[10].name' | tee /dev/stderr)
   [ "${actual}" = "FOO" ]
 
   local actual=$(echo $object |
-      yq -r '.[7].value' | tee /dev/stderr)
+      yq -r '.[10].value' | tee /dev/stderr)
   [ "${actual}" = "bar" ]
 
   local actual=$(echo $object |
-      yq -r '.[8].name' | tee /dev/stderr)
+      yq -r '.[11].name' | tee /dev/stderr)
   [ "${actual}" = "FOOBAR" ]
 
   local actual=$(echo $object |
-      yq -r '.[8].value' | tee /dev/stderr)
+      yq -r '.[11].value' | tee /dev/stderr)
   [ "${actual}" = "foobar" ]
 }
 
diff --git a/values.yaml b/values.yaml
index 9e0326a61..a7d7b92aa 100644
--- a/values.yaml
+++ b/values.yaml
@@ -110,7 +110,7 @@ server:
 
   image:
     repository: "vault"
-    tag: "1.3.3"
+    tag: "1.4.0"
     # Overrides the default Image Pull Policy
     pullPolicy: IfNotPresent
 
@@ -349,7 +349,6 @@ server:
       enabled: false
       config: |
         ui = true
-        cluster_addr = "https://POD_IP:8201"
 
         listener "tcp" {
           tls_disable = 1
@@ -361,12 +360,12 @@ server:
           path = "/vault/data"
         }
 
+        service_registration "kubernetes" {}
     # config is a raw string of default configuration when using a Stateful
     # deployment. Default is to use a Consul for its HA storage backend.
     # This should be HCL.
     config: |
       ui = true
-      cluster_addr = "https://POD_IP:8201"
 
       listener "tcp" {
         tls_disable = 1
@@ -378,6 +377,8 @@ server:
         address = "HOST_IP:8500"
       }
 
+      service_registration "kubernetes" {}
+
       # Example configuration for using auto-unseal, using Google Cloud KMS. The
       # GKMS keys must already exist, and the cluster must have a service account
       # that is authorized to access GCP KMS.