From 4ccf54d801735374b97e8a0161847028b73b00a0 Mon Sep 17 00:00:00 2001 From: erzetpe <2865776+erzetpe@users.noreply.github.com> Date: Fri, 14 Jan 2022 15:01:11 +0100 Subject: [PATCH] Vault removal (#2872) * Update vault documentation * Remove role and defaults * Add changelog for 2.0 --- CHANGELOG.md | 1 + README.md | 1 - .../roles/preflight/defaults/main.yml | 2 - .../centos-7/requirements.aarch64.txt | 4 - .../centos-7/requirements.x86_64.txt | 4 - .../redhat-7/requirements.x86_64.txt | 4 - .../ubuntu-20.04/requirements.x86_64.txt | 4 - .../playbooks/roles/vault/defaults/main.yml | 5 - .../files/kubernetes/vault-namespace.yml | 6 - .../roles/vault/files/tls-certs/readme.md | 3 - .../vault/files/vault/configure-vault.sh | 378 ------------------ .../roles/vault/files/vault/unseal-vault.sh | 95 ----- ansible/playbooks/roles/vault/meta/main.yml | 3 - .../vault/tasks/generate-certificate.yml | 59 --- ansible/playbooks/roles/vault/tasks/main.yml | 314 --------------- .../templates/kubernetes/app-namespace.yml.j2 | 6 - .../kubernetes/app-service-account.yml.j2 | 7 - .../kubernetes/vault-default-policy.yml.j2 | 18 - .../kubernetes/vault-service-account.yml.j2 | 27 -- .../roles/vault/templates/vault/config.hcl.j2 | 13 - .../vault/templates/vault/policy-admin.hcl.j2 | 50 --- .../templates/vault/policy-application.hcl.j2 | 7 - .../templates/vault/policy-provisioner.hcl.j2 | 33 -- .../vault/templates/vault/script.config.j2 | 21 - .../vault/templates/vault/selfsigned.cfr.j2 | 27 -- .../roles/vault/templates/vault/users.csv.j2 | 4 - .../vault/templates/vault/vault.service.j2 | 44 -- ansible/playbooks/vault.yml | 15 - ...hany_Component_Diagram_execution_place.png | 4 +- ...y_Component_Diagram_human_interactions.png | 4 +- ...Component_Diagram_runtime_dependencies.png | 4 +- ...any_Component_Diagram_metrics_and_logs.png | 4 +- .../current-c4/c2-arch-1-execution.puml | 2 - .../c2-arch-2-human-interactions.puml | 3 - .../c2-arch-3-runtime-dependencies.puml | 3 - .../c2-arch-4-metrics-and-logs.puml | 3 - docs/changelogs/CHANGELOG-2.0.md | 19 + .../k8s-vault-integration.md | 2 +- docs/home/ARM.md | 14 - docs/home/COMPONENTS.md | 2 - docs/home/DEPRECATION-NOTE.md | 16 +- docs/home/HOWTO.md | 2 - docs/home/SECURITY.md | 7 +- docs/home/howto/CLUSTER.md | 1 - docs/home/howto/SECURITY.md | 304 -------------- docs/home/howto/UPGRADE.md | 1 - .../configuration/feature-mapping.yml | 4 - .../defaults/configuration/image-registry.yml | 8 - .../common/defaults/configuration/vault.yml | 48 --- .../common/validation/configuration/vault.yml | 100 ----- 50 files changed, 39 insertions(+), 1671 deletions(-) delete mode 100644 ansible/playbooks/roles/vault/defaults/main.yml delete mode 100644 ansible/playbooks/roles/vault/files/kubernetes/vault-namespace.yml delete mode 100644 ansible/playbooks/roles/vault/files/tls-certs/readme.md delete mode 100644 ansible/playbooks/roles/vault/files/vault/configure-vault.sh delete mode 100644 ansible/playbooks/roles/vault/files/vault/unseal-vault.sh delete mode 100644 ansible/playbooks/roles/vault/meta/main.yml delete mode 100644 ansible/playbooks/roles/vault/tasks/generate-certificate.yml delete mode 100644 ansible/playbooks/roles/vault/tasks/main.yml delete mode 100644 ansible/playbooks/roles/vault/templates/kubernetes/app-namespace.yml.j2 delete mode 100644 ansible/playbooks/roles/vault/templates/kubernetes/app-service-account.yml.j2 delete mode 100644 ansible/playbooks/roles/vault/templates/kubernetes/vault-default-policy.yml.j2 delete mode 100644 ansible/playbooks/roles/vault/templates/kubernetes/vault-service-account.yml.j2 delete mode 100644 ansible/playbooks/roles/vault/templates/vault/config.hcl.j2 delete mode 100644 ansible/playbooks/roles/vault/templates/vault/policy-admin.hcl.j2 delete mode 100644 ansible/playbooks/roles/vault/templates/vault/policy-application.hcl.j2 delete mode 100644 ansible/playbooks/roles/vault/templates/vault/policy-provisioner.hcl.j2 delete mode 100644 ansible/playbooks/roles/vault/templates/vault/script.config.j2 delete mode 100644 ansible/playbooks/roles/vault/templates/vault/selfsigned.cfr.j2 delete mode 100644 ansible/playbooks/roles/vault/templates/vault/users.csv.j2 delete mode 100644 ansible/playbooks/roles/vault/templates/vault/vault.service.j2 delete mode 100644 ansible/playbooks/vault.yml create mode 100644 docs/changelogs/CHANGELOG-2.0.md delete mode 100644 schema/common/defaults/configuration/vault.yml delete mode 100644 schema/common/validation/configuration/vault.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 3fe0961fa0..06e5215615 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. Reference for actual cluster component versions can be found [here](docs/home/COMPONENTS.md) +- [CHANGELOG-2.0.x](./docs/changelogs/CHANGELOG-2.0.md) - [CHANGELOG-1.3.x](./docs/changelogs/CHANGELOG-1.3.md) - [CHANGELOG-1.2.x](./docs/changelogs/CHANGELOG-1.2.md) - [CHANGELOG-1.1.x](./docs/changelogs/CHANGELOG-1.1.md) diff --git a/README.md b/README.md index b04b6fe43c..b996c4b75d 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,6 @@ Epiphany at its core is a full automation of Kubernetes and Docker plus addition - HAProxy for loadbalancing - Postgres and Elasticsearch for data storage - KeyCloak for authentication -- Vault (MVP) for protecting secrets and other sensitive data - Helm as package manager for Kubernetes The following target platforms are available: AWS, Azure and on-prem installation. diff --git a/ansible/playbooks/roles/preflight/defaults/main.yml b/ansible/playbooks/roles/preflight/defaults/main.yml index d29247746b..a4a151c3a3 100644 --- a/ansible/playbooks/roles/preflight/defaults/main.yml +++ b/ansible/playbooks/roles/preflight/defaults/main.yml @@ -37,7 +37,6 @@ unsupported_roles: - helm - zookeeper - haproxy - - vault - logging - elasticsearch_curator - opendistro_for_elasticsearch @@ -77,7 +76,6 @@ unsupported_roles: - helm - zookeeper - haproxy - - vault - logging - elasticsearch_curator - opendistro_for_elasticsearch diff --git a/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.aarch64.txt b/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.aarch64.txt index 3fa1e46563..ff0c03ebfa 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.aarch64.txt +++ b/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.aarch64.txt @@ -164,9 +164,7 @@ https://github.com/prometheus/prometheus/releases/download/v2.31.1/prometheus-2. https://github.com/prometheus/alertmanager/releases/download/v0.23.0/alertmanager-0.23.0.linux-arm64.tar.gz https://archive.apache.org/dist/zookeeper/zookeeper-3.5.8/apache-zookeeper-3.5.8-bin.tar.gz https://archive.apache.org/dist/ignite/2.9.1/apache-ignite-2.9.1-bin.zip -https://releases.hashicorp.com/vault/1.7.0/vault_1.7.0_linux_arm64.zip https://get.helm.sh/helm-v3.2.0-linux-arm64.tar.gz -https://github.com/hashicorp/vault-helm/archive/v0.11.0.tar.gz https://archive.apache.org/dist/logging/log4j/2.17.1/apache-log4j-2.17.1-bin.tar.gz # --- Helm charts --- https://charts.bitnami.com/bitnami/node-exporter-2.3.17.tgz @@ -204,8 +202,6 @@ haproxy:2.2.2-alpine kubernetesui/dashboard:v2.3.1 kubernetesui/metrics-scraper:v1.0.7 registry:2 -hashicorp/vault-k8s:0.10.0 -vault:1.7.0 # applications epiphanyplatform/ignite:2.9.1 epiphanyplatform/keycloak:14.0.0 diff --git a/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.x86_64.txt b/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.x86_64.txt index 8dc6f611fb..8e1cd804da 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.x86_64.txt +++ b/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.x86_64.txt @@ -165,9 +165,7 @@ https://github.com/prometheus/prometheus/releases/download/v2.31.1/prometheus-2. https://github.com/prometheus/alertmanager/releases/download/v0.23.0/alertmanager-0.23.0.linux-amd64.tar.gz https://archive.apache.org/dist/zookeeper/zookeeper-3.5.8/apache-zookeeper-3.5.8-bin.tar.gz https://archive.apache.org/dist/ignite/2.9.1/apache-ignite-2.9.1-bin.zip -https://releases.hashicorp.com/vault/1.7.0/vault_1.7.0_linux_amd64.zip https://get.helm.sh/helm-v3.2.0-linux-amd64.tar.gz -https://github.com/hashicorp/vault-helm/archive/v0.11.0.tar.gz https://archive.apache.org/dist/logging/log4j/2.17.1/apache-log4j-2.17.1-bin.tar.gz # --- Helm charts --- https://helm.elastic.co/helm/filebeat/filebeat-7.9.2.tgz @@ -205,8 +203,6 @@ haproxy:2.2.2-alpine kubernetesui/dashboard:v2.3.1 kubernetesui/metrics-scraper:v1.0.7 registry:2 -hashicorp/vault-k8s:0.10.0 -vault:1.7.0 # applications epiphanyplatform/ignite:2.9.1 bitnami/pgpool:4.2.4 diff --git a/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.x86_64.txt b/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.x86_64.txt index 79ab22372d..d3e5b5fae1 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.x86_64.txt +++ b/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.x86_64.txt @@ -163,9 +163,7 @@ https://github.com/prometheus/prometheus/releases/download/v2.31.1/prometheus-2. https://github.com/prometheus/alertmanager/releases/download/v0.23.0/alertmanager-0.23.0.linux-amd64.tar.gz https://archive.apache.org/dist/zookeeper/zookeeper-3.5.8/apache-zookeeper-3.5.8-bin.tar.gz https://archive.apache.org/dist/ignite/2.9.1/apache-ignite-2.9.1-bin.zip -https://releases.hashicorp.com/vault/1.7.0/vault_1.7.0_linux_amd64.zip https://get.helm.sh/helm-v3.2.0-linux-amd64.tar.gz -https://github.com/hashicorp/vault-helm/archive/v0.11.0.tar.gz https://archive.apache.org/dist/logging/log4j/2.17.1/apache-log4j-2.17.1-bin.tar.gz # --- Helm charts --- https://helm.elastic.co/helm/filebeat/filebeat-7.9.2.tgz @@ -203,8 +201,6 @@ haproxy:2.2.2-alpine kubernetesui/dashboard:v2.3.1 kubernetesui/metrics-scraper:v1.0.7 registry:2 -hashicorp/vault-k8s:0.10.0 -vault:1.7.0 # applications epiphanyplatform/ignite:2.9.1 bitnami/pgpool:4.2.4 diff --git a/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-20.04/requirements.x86_64.txt b/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-20.04/requirements.x86_64.txt index 37a928db2f..574d59a7d8 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-20.04/requirements.x86_64.txt +++ b/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-20.04/requirements.x86_64.txt @@ -200,9 +200,7 @@ https://archive.apache.org/dist/zookeeper/zookeeper-3.5.8/apache-zookeeper-3.5.8 https://github.com/prometheus/alertmanager/releases/download/v0.23.0/alertmanager-0.23.0.linux-amd64.tar.gz https://github.com/prometheus/prometheus/releases/download/v2.31.1/prometheus-2.31.1.linux-amd64.tar.gz https://archive.apache.org/dist/ignite/2.9.1/apache-ignite-2.9.1-bin.zip -https://releases.hashicorp.com/vault/1.7.0/vault_1.7.0_linux_amd64.zip https://get.helm.sh/helm-v3.2.0-linux-amd64.tar.gz -https://github.com/hashicorp/vault-helm/archive/v0.11.0.tar.gz https://archive.apache.org/dist/logging/log4j/2.17.1/apache-log4j-2.17.1-bin.tar.gz # --- Helm charts --- https://charts.bitnami.com/bitnami/node-exporter-2.3.17.tgz @@ -240,8 +238,6 @@ haproxy:2.2.2-alpine kubernetesui/dashboard:v2.3.1 kubernetesui/metrics-scraper:v1.0.7 registry:2 -hashicorp/vault-k8s:0.10.0 -vault:1.7.0 # applications epiphanyplatform/ignite:2.9.1 bitnami/pgpool:4.2.4 diff --git a/ansible/playbooks/roles/vault/defaults/main.yml b/ansible/playbooks/roles/vault/defaults/main.yml deleted file mode 100644 index b84ca6f837..0000000000 --- a/ansible/playbooks/roles/vault/defaults/main.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -vault: - file_name: - x86_64: vault_1.7.0_linux_amd64.zip - aarch64: vault_1.7.0_linux_arm64.zip diff --git a/ansible/playbooks/roles/vault/files/kubernetes/vault-namespace.yml b/ansible/playbooks/roles/vault/files/kubernetes/vault-namespace.yml deleted file mode 100644 index 69ee157663..0000000000 --- a/ansible/playbooks/roles/vault/files/kubernetes/vault-namespace.yml +++ /dev/null @@ -1,6 +0,0 @@ -# {{ ansible_managed }} ---- -apiVersion: v1 -kind: Namespace -metadata: - name: vault diff --git a/ansible/playbooks/roles/vault/files/tls-certs/readme.md b/ansible/playbooks/roles/vault/files/tls-certs/readme.md deleted file mode 100644 index 316b77b334..0000000000 --- a/ansible/playbooks/roles/vault/files/tls-certs/readme.md +++ /dev/null @@ -1,3 +0,0 @@ -# Note - -If this directory contains a file with .pem extension, it will be used for Vault. Otherwise self-singed certificate is generated. diff --git a/ansible/playbooks/roles/vault/files/vault/configure-vault.sh b/ansible/playbooks/roles/vault/files/vault/configure-vault.sh deleted file mode 100644 index 080badbdfb..0000000000 --- a/ansible/playbooks/roles/vault/files/vault/configure-vault.sh +++ /dev/null @@ -1,378 +0,0 @@ -#!/usr/bin/env bash -# Description: This script configures Hashicorp Vault to be used with Epiphany -# You can find more information in Epiphany documentation in HOWTO.md -# TODO: Revoke root token -# TODO: Add configurable log paths -# TODO: Make devweb-app policy and role configurable (function integrate_with_kubernetes) -# TODO: Make Helm chart location configurable (function configure_kubernetes) - -HELP_MESSAGE="Usage: configure-vault.sh -c SCRIPT_CONFIGURATION_FILE_PATH -a VAULT_IP_ADDRESS -p {http|https} -v {true|false}" - -function print_help { echo "$HELP_MESSAGE"; } - -function log_and_print { - local string_to_log="$1"; - echo "$(date +"%Y-%m-%d-%H:%M:%S"): $string_to_log" | tee -a /opt/vault/logs/configure_vault.log; -} - -function exit_with_error { - local string_to_log="$1"; - log_and_print "ERROR: $string_to_log"; - exit 1; -} - -function check_status { - local exit_code="$1"; - local success_message="$2"; - local failure_message="$3"; - if [ "$exit_code" = "0" ] ; then - log_and_print "$success_message"; - else - exit_with_error "$failure_message Exit status: $exit_code"; - fi -} - -function initialize_vault { - local init_file_path="$1"; - log_and_print "Checking if Vault is already initialized..."; - vault status -format json | grep -q '"initialized": true'; - local command_result=( "${PIPESTATUS[@]}" ); - if [ "${command_result[0]}" = "1" ] ; then - exit_with_error "There was an error during checking status of Vault."; - fi - if [ "${command_result[1]}" = "0" ] ; then - log_and_print "Vault is already initialized."; - elif [ "${command_result[1]}" = "1" ] ; then - touch "$init_file_path"; - chmod 0640 "$init_file_path"; - log_and_print "Initializing Vault..."; - vault operator init > "$init_file_path"; - check_status $? "Vault initialized." "There was an error during initialization of Vault."; - fi -} - -function unseal_vault { - local init_file_path="$1"; - log_and_print "Checking if vault is already unsealed..."; - vault status -format json | grep -q '"sealed": false'; - local command_result=( "${PIPESTATUS[@]}" ); - if [ "${command_result[0]}" = "1" ] ; then - exit_with_error "There was an error during checking status of Vault."; - fi - if [ "${command_result[1]}" = "0" ] ; then - log_and_print "Vault is already unsealed."; - elif [ "${command_result[1]}" = "1" ] ; then - log_and_print "Unsealing Vault..."; - grep --max-count=3 Unseal "$init_file_path" | awk '{print $4}' | while read -r line ; do - vault operator unseal "$line"; - check_status $? "Unseal performed." "There was an error during unsealing of Vault."; - done - fi -} - -function check_if_vault_is_unsealed { - log_and_print "Checking if vault is already unsealed..."; - vault status; - local command_result=$?; - if [ "$command_result" = "1" ] ; then - exit_with_error "There was an error during checking status of Vault."; - elif [ "$command_result" = "2" ] ; then - exit_with_error "Vault hasn't been successfully unsealed. Please configure script for auto-unsealing or unseal Vault manually."; - fi -} - -function enable_vault_audit_logs { - log_and_print "Checking if audit logging is enabled..."; - vault audit list | grep "file"; - local command_result=( "${PIPESTATUS[@]}" ); - if [ "${command_result[0]}" = "1" ] ; then #IMPORTANT exit code = 2 if audit list is empty so it is ignored - exit_with_error "There was an error during listing audit devices. Exit status: ${command_result[0]}"; - fi - if [ "${command_result[1]}" = "0" ] ; then - log_and_print "Audit logging is already enabled."; - elif [ "${command_result[1]}" = "1" ] ; then - log_and_print "Enabling audit logging..."; - vault audit enable file file_path="/opt/vault/logs/vault_audit.log"; - check_status $? "Audit logging enabled." "There was an error during enabling audit logging."; - fi -} - -function mount_secret_path { - local secret_path="$1"; - log_and_print "Checking if secret engine is already initialized..."; - vault secrets list | grep "$secret_path/"; - local command_result=( "${PIPESTATUS[@]}" ); - if [ "${command_result[0]}" != "0" ] ; then - exit_with_error "There was an error during listing secret engines. Exit status: ${command_result[0]}"; - fi - if [ "${command_result[1]}" = "0" ] ; then - log_and_print "Secret engine is already mounted under path: $secret_path."; - elif [ "${command_result[1]}" = "1" ] ; then - log_and_print "Mounting secret engine..."; - vault secrets enable -path="$secret_path" -version=2 kv; - check_status $? "Secret engine enabled under path: $secret_path." "There was an error during enabling secret engine under path: $secret_path."; - fi -} - -function enable_vault_kubernetes_authentication { - log_and_print "Checking if Kubernetes authentication is enabled..."; - vault auth list | grep kubernetes; - local command_result=( "${PIPESTATUS[@]}" ); - if [ "${command_result[0]}" != "0" ] ; then - exit_with_error "There was an error during listing authentication methods. Exit status: ${command_result[0]}"; - fi - if [ "${command_result[1]}" = "0" ] ; then - log_and_print "Kubernetes authentication is already enabled."; - elif [ "${command_result[1]}" = "1" ] ; then - log_and_print "Turning on Kubernetes authentication..."; - vault auth enable kubernetes; - check_status $? "Kubernetes authentication enabled." "There was an error during enabling Kubernetes authentication."; - fi -} - -function kubectl_with_retries { - local kubectl_args="$1"; - local number_of_retries="${2:-69}"; # use default value of 69 - local number_of_seconds="${3:-2}"; # use default value of 2 - local retry; - - for (( retry = 0; retry < number_of_retries; retry++ )); do - sleep $number_of_seconds; - if RESULT="$($SHELL -c "kubectl $kubectl_args")"; then # please note, RESULT is a global variable! - return 0; - fi; - done; - - # Return original command string for debugging purposes. - RESULT="Command >>>> kubectl $kubectl_args <<<< failed after $retry retries."; - return 1; -} - -function integrate_with_kubernetes { - local vault_config_data_path="$1"; - local kubernetes_namespace="$2"; - local policy_name="devweb-app"; - local role_name="devweb-app"; - - log_and_print "Turning on Kubernetes integration..."; - - local token_reviewer_jwt; - if kubectl_with_retries "--kubeconfig=/etc/kubernetes/admin.conf get secret vault-auth -o go-template='{{ .data.token }}'"; then - if ! token_reviewer_jwt="$(base64 --decode <<< "$RESULT")"; then - exit_with_error "Unable to base64/decode vault-auth secret."; - fi; - else - exit_with_error "$RESULT"; - fi; - - local kube_ca_cert; - if kubectl_with_retries "--kubeconfig=/etc/kubernetes/admin.conf config view --raw --minify --flatten -o jsonpath='{.clusters[].cluster.certificate-authority-data}'"; then - if ! kube_ca_cert="$(base64 --decode <<< "$RESULT")"; then - exit_with_error "Unable to base64/decode kubernetes certificate authority data." - fi; - else - exit_with_error "$RESULT"; - fi; - - local kube_host; - if kubectl_with_retries "--kubeconfig=/etc/kubernetes/admin.conf config view --raw --minify --flatten -o jsonpath='{.clusters[].cluster.server}'"; then - kube_host="$RESULT"; - else - exit_with_error "$RESULT"; - fi; - - vault write auth/kubernetes/config token_reviewer_jwt="$token_reviewer_jwt" kubernetes_host="$kube_host" kubernetes_ca_cert="$kube_ca_cert"; - check_status $? "Kubernetes parameters written to auth/kubernetes/config." "There was an error during writing Kubernetes parameters to auth/kubernetes/config."; - - vault policy write "$policy_name" "$vault_config_data_path/policies/policy-application.hcl"; - check_status $? "Application policy applied." "There was an error during applying application policy."; - - vault write "auth/kubernetes/role/$role_name" bound_service_account_names=internal-app bound_service_account_namespaces="$kubernetes_namespace" policies="$policy_name" ttl=24h; - check_status $? "Application role applied." "There was an error during applying application role."; -} - -function configure_kubernetes { - local vault_install_path="$1"; - local kubernetes_namespace="$2"; - local vault_protocol="$3"; - local helm_custom_values_set_bool="$4"; - log_and_print "Configuring Kubernetes..."; - local files_to_apply=( app-namespace.yml vault-namespace.yml vault-default-policy.yml vault-service-account.yml app-service-account.yml ) - for file in "${files_to_apply[@]}" ; do - if [ "$file" = "app-namespace.yml" ] && [ "$kubernetes_namespace" = "default" ]; then - continue - fi - log_and_print "Applying $file..."; - kubectl apply -f "$vault_install_path/kubernetes/$file"; - check_status $? "$file: Success." "$file: Failure."; - done - log_and_print "Checking if Vault Agent Helm Chart is already installed..."; - helm list | grep vault; - local command_result=( "${PIPESTATUS[@]}" ); - if [ "${command_result[0]}" != "0" ] ; then - exit_with_error "There was an error during checking if Vault Agent Helm Chart is already installed. Exit status: ${command_result[0]}"; - fi - if [ "${command_result[1]}" = "0" ] ; then - log_and_print "Vault Agent Helm Chart is already installed."; - elif [ "${command_result[1]}" = "1" ] ; then - log_and_print "Installing Vault Agent Helm Chart..."; - if [ "$helm_custom_values_set_bool" = "true" ] ; then - helm upgrade --install --wait -f /tmp/vault_helm_chart_values.yaml vault /tmp/v0.11.0.tar.gz --namespace vault - else - helm upgrade --install --wait vault /tmp/v0.11.0.tar.gz --namespace vault - fi - check_status $? "Vault Agent Helm Chart installed." "There was an error during installation of Vault Agent Helm Chart."; - fi -} - -function apply_epiphany_vault_policies { - log_and_print "Applying Epiphany default Vault policies..."; - local vault_config_data_path="$1"; - vault policy write admin "$vault_config_data_path/policies/policy-admin.hcl"; - check_status $? "Admin policy applied." "There was an error during applying admin policy."; - vault policy write provisioner "$vault_config_data_path/policies/policy-provisioner.hcl"; - check_status $? "Provisioner policy applied." "There was an error during applying provisioner policy."; -} - -function enable_vault_userpass_authentication { - log_and_print "Checking if userpass authentication is enabled..."; - vault auth list | grep userpass; - local command_result=( "${PIPESTATUS[@]}" ); - if [ "${command_result[0]}" != "0" ] ; then - exit_with_error "There was an error during listing authentication methods. Exit status: ${command_result[0]}"; - fi - if [ "${command_result[1]}" = "0" ] ; then - log_and_print "Userpass authentication is already enabled."; - elif [ "${command_result[1]}" = "1" ] ; then - log_and_print "Turning on userpass authentication..."; - vault auth enable userpass; - check_status $? "Userpass authentication enabled." "There was an error during enabling userpass authentication."; - fi -} - -function create_vault_user { - local username="$1"; - local policy="$2"; - local token_path="$3"; - local token="$4"; - local vault_addr="$5"; - local override_existing_vault_users="$6"; - - if [ ! -f "$token_path" ]; then - touch "$token_path"; - chmod 0640 "$token_path"; - fi - local users_path_response - users_path_response=$(curl -o -I -L -s -w "%{http_code}" --header "X-Vault-Token: $token" --request LIST "$vault_addr/v1/auth/userpass/users"); - if (( users_path_response == 200 )) ; then - curl --header "X-Vault-Token: $token" --request LIST "$vault_addr/v1/auth/userpass/users" | jq -e ".data.keys[] | select(.== \"$username\")"; - local command_result=$?; - fi - if [ "${override_existing_vault_users,,}" = "true" ] || (( users_path_response == 404 )) || (( command_result == 4 )); then - log_and_print "Creating user: $username..."; - local password; - password="$( < /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c32 )"; - vault write "auth/userpass/users/$username" password="$password" policies="$policy"; - check_status $? "User: $username created." "There was an error during creation of user: $username."; - echo "$username;$policy;$password;" >> "$token_path"; - elif [ "$command_result" = "0" ]; then - log_and_print "$username already exists. Not adding or modyfing."; - echo "$username;$policy;ALREADY_EXISTS;" >> "$token_path"; - else - exit_with_error "There was a critical error during adding user: $username."; - fi -} - -function create_vault_users_from_file { - local vault_install_path="$1"; - local token="$2"; - local vault_addr="$3"; - local override_existing_vault_users="$4"; - local users_csv_file_path="$vault_install_path/users.csv"; - local users_token_path="$vault_install_path/tokens-$(date +"%Y-%m-%d-%H%M%S").csv"; - local user_name - local policy - grep -v '#' "$users_csv_file_path" | while read -r line ; do - user_name="$( echo "$line" | cut -d ';' -f 1 )"; - policy="$( echo "$line" | cut -d ';' -f 2 )"; - create_vault_user "$user_name" "$policy" "$users_token_path" "$token" "$vault_addr" "$override_existing_vault_users"; - done -} - -function cleanup { - rm -f "$HOME/.vault-token"; -} - -# --- Start --- - -if [ "$#" -lt 6 ]; then - print_help; - exit_with_error "Mandatory argument is missing. Aborting."; -fi - -while getopts ":a:c:p:v:h" opt; do - case "$opt" in - a) VAULT_IP=$OPTARG;; - c) CONFIG_FILE=$OPTARG;; - p) VAULT_PROTOCOL=$OPTARG;; - v) HELM_CUSTOM_VALUES_SET_BOOL=$OPTARG;; - \?) print_help; exit_with_error "Invalid parameter: -$OPTARG. Aborting.";; - :) print_help; exit_with_error "Parameter -$OPTARG requires an argument. Aborting.";; - h) print_help; exit 0;; - esac -done -shift $((OPTIND-1)) - -test -f "$CONFIG_FILE" || exit_with_error "Config file not found. Aborting."; - -# shellcheck source=/dev/null -source "$CONFIG_FILE"; - -INIT_FILE_PATH="$VAULT_INSTALL_PATH/init.txt" -VAULT_CONFIG_DATA_PATH="$VAULT_INSTALL_PATH/config" -PATH=$VAULT_INSTALL_PATH/bin:/usr/local/bin/:$PATH - -export VAULT_ADDR="$VAULT_PROTOCOL://$VAULT_IP:8200" -export KUBECONFIG=/etc/kubernetes/admin.conf - -if [ "${VAULT_TOKEN_CLEANUP,,}" = "true" ] ; then - trap cleanup EXIT INT TERM; -fi - -initialize_vault "$INIT_FILE_PATH"; - -if [ "${UNSEAL_VAULT,,}" = "true" ] ; then - unseal_vault "$INIT_FILE_PATH"; -fi - -check_if_vault_is_unsealed; - -log_and_print "Logging into Vault."; -LOGIN_TOKEN="$(grep "Initial Root Token:" "$INIT_FILE_PATH" | awk -F'[ ]' '{print $4}')"; -vault login -no-print "$LOGIN_TOKEN"; -check_status $? "Login successful." "There was an error while logging into Vault."; - -if [ "${ENABLE_VAULT_AUDIT_LOGS,,}" = "true" ] ; then - enable_vault_audit_logs; -fi - -mount_secret_path "$SECRETS_ENGINE_PATH"; - -if [ "${KUBERNETES_INTEGRATION,,}" = "true" ] || [ "${ENABLE_VAULT_KUBERNETES_AUTHENTICATION,,}" = "true" ] ; then - enable_vault_kubernetes_authentication; -fi - -apply_epiphany_vault_policies "$VAULT_CONFIG_DATA_PATH"; -enable_vault_userpass_authentication; - -if [ "${CREATE_VAULT_USERS,,}" = "true" ] ; then - create_vault_users_from_file "$VAULT_INSTALL_PATH" "$LOGIN_TOKEN" "$VAULT_ADDR" "$OVERRIDE_EXISTING_VAULT_USERS"; -fi - -if [ "${KUBERNETES_CONFIGURATION,,}" = "true" ] ; then - configure_kubernetes "$VAULT_INSTALL_PATH" "$KUBERNETES_NAMESPACE" "$VAULT_PROTOCOL" "$HELM_CUSTOM_VALUES_SET_BOOL"; -fi - -if [ "${KUBERNETES_INTEGRATION,,}" = "true" ] ; then - integrate_with_kubernetes "$VAULT_CONFIG_DATA_PATH" "$KUBERNETES_NAMESPACE"; -fi diff --git a/ansible/playbooks/roles/vault/files/vault/unseal-vault.sh b/ansible/playbooks/roles/vault/files/vault/unseal-vault.sh deleted file mode 100644 index 1926309742..0000000000 --- a/ansible/playbooks/roles/vault/files/vault/unseal-vault.sh +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/env bash - -function log_and_print { - local string_to_log="$1"; - echo "$(date +"%Y-%m-%d-%H:%M:%S"): $string_to_log" | tee -a /opt/vault/logs/unseal_vault.log; -} - -function exit_with_error { - local string_to_log="$1"; - log_and_print "ERROR: $string_to_log"; - exit 1; -} - -function check_vault_error { - local exit_code="$1"; - local success_message="$2"; - local failure_message="$3"; - if [ "$exit_code" == "0" ] ; then - log_and_print "$success_message"; - else - exit_with_error "$failure_message Exit status: $exit_code"; - fi -} - -function check_if_vault_is_initialized { - vault status -format json | grep -q '"initialized": true'; - local command_result=( "${PIPESTATUS[@]}" ); - if [ "${command_result[0]}" = "1" ] ; then - exit_with_error "There was an error during checking status of Vault."; - fi - if [ "${command_result[1]}" = "0" ] ; then - log_and_print "Vault is initialized."; - elif [ "${command_result[1]}" = "1" ] ; then - log_and_print "Vault is not initialized, nothing to unseal."; - exit 0; - fi -} - -function unseal_vault { - local init_file_path="$1"; - log_and_print "Checking if vault is already unsealed..."; - vault status -format json | grep -q '"sealed": false'; - local command_result=( "${PIPESTATUS[@]}" ); - if [ "${command_result[0]}" = "1" ] ; then - exit_with_error "There was an error during checking status of Vault."; - fi - if [ "${command_result[1]}" = "0" ] ; then - log_and_print "Vault is already unsealed. Nothing to do."; - elif [ "${command_result[1]}" = "1" ] ; then - log_and_print "Unsealing Vault..."; - grep --max-count=3 Unseal "$init_file_path" | awk '{print $4}' | while read -r line ; do - vault operator unseal "$line"; - check_vault_error "$?" "Unseal performed." "There was an error during unsealing of Vault."; - done - fi -} - -function check_if_vault_is_running { - local vault_address="${1:?missing value}" - local timeout="${2:?missing value}" - local is_running="false" - local delay=2 - while (( timeout >= 0 )) && [ "$is_running" = "false" ] ; do - log_and_print "Checking if Vault is running..." - response_code=$(curl -o -I -L -s -w "%{http_code}" "$vault_address/v1/sys/seal-status") - if (( response_code == 200 )) ; then - is_running="true" - fi - sleep $delay - timeout=$((timeout - delay)) - done - if [ "$is_running" = "false" ] ; then - exit_with_error "Vault is not running. Please solve the problem and run the script again."; - else - log_and_print "Vault is running." - fi -} - -# --- Start --- - -INIT_FILE_PATH="${1:?missing argument}"; -VAULT_IP="${2:?missing argument}"; -VAULT_PROTOCOL="${3:?missing argument}"; - -PATH=$VAULT_INSTALL_PATH/bin:/usr/local/bin:$PATH; -export VAULT_ADDR="$VAULT_PROTOCOL://$VAULT_IP:8200" - -check_if_vault_is_running "$VAULT_ADDR" 10; -check_if_vault_is_initialized; - -if [ -f "$INIT_FILE_PATH" ]; then - unseal_vault "$INIT_FILE_PATH"; -else - exit_with_error "Init file doesn't exist. Cannot unseal."; -fi diff --git a/ansible/playbooks/roles/vault/meta/main.yml b/ansible/playbooks/roles/vault/meta/main.yml deleted file mode 100644 index 745ba4d956..0000000000 --- a/ansible/playbooks/roles/vault/meta/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -dependencies: - - role: preflight_facts diff --git a/ansible/playbooks/roles/vault/tasks/generate-certificate.yml b/ansible/playbooks/roles/vault/tasks/generate-certificate.yml deleted file mode 100644 index d7d37487ed..0000000000 --- a/ansible/playbooks/roles/vault/tasks/generate-certificate.yml +++ /dev/null @@ -1,59 +0,0 @@ -- name: Create selfsigned.cfr file for TLS certificate generation - template: - dest: "{{ specification.vault_install_dir }}/tls/selfsigned.cfr" - owner: "{{ specification.vault_system_user }}" - group: "{{ specification.vault_system_group }}" - mode: u=rw,g=r,o= - src: vault/selfsigned.cfr.j2 - -- name: Check if self-signed certificate exists - stat: - path: "{{ specification.vault_install_dir }}/tls/{{ specification.certificate_name }}" - register: certificate_file_stat - -- name: Check if private key for self-signed certificate exists - stat: - path: "{{ specification.vault_install_dir }}/tls/{{ specification.private_key_name }}" - register: priv_key_file_stat - -- name: Generate self-signed cert and private key if files were not found in directory - shell: >- - openssl req -x509 -batch -nodes -newkey rsa:4096 - -keyout {{ specification.vault_install_dir }}/tls/{{ specification.private_key_name }} - -out {{ specification.vault_install_dir }}/tls/{{ specification.certificate_name }} - -config {{ specification.vault_install_dir }}/tls/selfsigned.cfr - -days {{ specification.vault_tls_valid_days }} - when: - - not certificate_file_stat.stat.exists - - not priv_key_file_stat.stat.exists - -- name: Set file permissions for self-signed certificate - file: - path: "{{ specification.vault_install_dir }}/tls/{{ specification.certificate_name }}" - owner: "{{ specification.vault_system_user }}" - group: "{{ specification.vault_system_group }}" - mode: u=rw,g=r,o=r - -- name: Set file permissions for private key - file: - path: "{{ specification.vault_install_dir }}/tls/{{ specification.private_key_name }}" - owner: "{{ specification.vault_system_user }}" - group: "{{ specification.vault_system_group }}" - mode: u=rw,g=r,o= - -- name: Copy generated cert file to CA certificates location - copy: - src: "{{ specification.vault_install_dir }}/tls/{{ specification.certificate_name }}" - dest: "{{ ca_certificates_dir[ansible_os_family] }}/{{ specification.certificate_name }}.crt" - remote_src: yes - vars: - ca_certificates_dir: - Debian: /usr/local/share/ca-certificates - RedHat: /etc/pki/ca-trust/source/anchors - -- name: Update list of trusted CA certificates - shell: "{{ update_command[ansible_os_family] }}" - vars: - update_command: - Debian: update-ca-certificates - RedHat: update-ca-trust extract diff --git a/ansible/playbooks/roles/vault/tasks/main.yml b/ansible/playbooks/roles/vault/tasks/main.yml deleted file mode 100644 index 296e29f66c..0000000000 --- a/ansible/playbooks/roles/vault/tasks/main.yml +++ /dev/null @@ -1,314 +0,0 @@ ---- -# This role is responsible for installing and configuring Hashicorp Vault -# TODO: Add swap disable -# TODO: Limit to set only on 1 master until vault will be clustered - -- name: Determine if helm chart custom values are set - set_fact: - # Handling "undefined", "null" and "empty" values all at once. - vault_helm_chart_values_bool: "{{ specification.vault_helm_chart_values | default(false) | ternary(true, false) }}" - -- name: Clean up temporary Vault Helm chart values file - file: - state: absent - path: /tmp/vault_helm_chart_values.yaml - -- name: Set full Vault address - set_fact: - full_vault_address: "{{ specification.tls_disable | ternary('http', 'https') }}://{{ hostvars[groups['vault'][0]]['ansible_default_ipv4']['address'] }}:8200" - when: specification.vault_helm_chart_values.injector.externalVaultAddr is not defined - -- name: Set final Vault Helm chart's custom configuration file content - set_fact: - final_vault_helm_chart_values: "{{ specification.vault_helm_chart_values | combine({'injector': {'externalVaultAddr': full_vault_address}},recursive=True) }}" - when: specification.vault_helm_chart_values.injector.externalVaultAddr is not defined - -- name: Set final Vault Helm chart's custom configuration file content - manual externalVaultAddr config - set_fact: - final_vault_helm_chart_values: "{{ specification.vault_helm_chart_values }}" - when: specification.vault_helm_chart_values.injector.externalVaultAddr is defined - -- name: Copy Vault Helm chart's custom configuration to file - copy: - content: "{{ final_vault_helm_chart_values }}" - dest: /tmp/vault_helm_chart_values.yaml - when: vault_helm_chart_values_bool - -- name: Install and configure Hashicorp Vault - block: - - name: Create Vault system group - group: - name: "{{ specification.vault_system_group }}" - system: yes - - - name: Create Vault system user - user: - name: "{{ specification.vault_system_user }}" - group: "{{ specification.vault_system_group }}" - system: yes - shell: "/usr/sbin/nologin" - - - name: Create Vault directories - file: - path: "{{ specification.vault_install_dir }}/{{ item }}" - state: directory - owner: "{{ specification.vault_system_user }}" - group: "{{ specification.vault_system_group }}" - mode: u=rwx,g=rx,o= - loop: - - "bin" - - "config" - - "data" - - "kubernetes" - - "logs" - - "config/policies" - - "tls" - - - name: Set Vault file name to install - set_fact: - vault_file_name: "{{ vault.file_name[ansible_architecture] }}" - - - name: Download Vault binary package - include_role: - name: download - tasks_from: download_file - vars: - file_name: "{{ vault_file_name }}" - - - name: Download Vault Agent Helm chart package - include_role: - name: download - tasks_from: download_file - vars: - file_name: "{{ specification.files.vault_helm_chart_file_name }}" - - - name: Check for Vault binary - stat: - path: "{{ specification.vault_install_dir }}/bin/vault" - register: vault_binary - - - name: Uncompress the Vault zip - when: not vault_binary.stat.exists - unarchive: - remote_src: yes - owner: "{{ specification.vault_system_user }}" - group: "{{ specification.vault_system_group }}" - src: "{{ download_directory }}/{{ vault_file_name }}" - creates: "{{ specification.vault_install_dir }}/bin/vault" - dest: "{{ specification.vault_install_dir }}/bin/" - mode: u=rwx,g=rx,o= - - - name: Create a symbolic link to Vault - file: - src: "{{ specification.vault_install_dir }}/bin/vault" - dest: /usr/local/bin/vault - owner: "{{ specification.vault_system_user }}" - group: "{{ specification.vault_system_group }}" - state: link - - - name: Create Vault configuration file - template: - dest: "{{ specification.vault_install_dir }}/config/config.hcl" - owner: "{{ specification.vault_system_user }}" - group: "{{ specification.vault_system_group }}" - mode: u=rw,g=r,o= - src: vault/config.hcl.j2 - - - name: Set capabilities required to lock memory - capabilities: - path: "{{ specification.vault_install_dir }}/bin/vault" - capability: cap_ipc_lock=+ep - state: present - - - name: Set fs.suid_dumpable to 0 - sysctl: - name: fs.suid_dumpable - value: '0' - sysctl_set: yes - state: present - reload: yes - - - name: Set security limits for Vault - pam_limits: - domain: '*' - limit_type: hard - limit_item: core - value: '0' - - - name: Copy Vault unsealing script - copy: - src: vault/unseal-vault.sh - dest: "{{ specification.vault_install_dir }}/bin/" - owner: "{{ specification.vault_system_user }}" - group: "{{ specification.vault_system_group }}" - mode: u=rwx,g=rx,o= - - - name: Create systemd service - template: - dest: /etc/systemd/system/vault.service - owner: root - group: root - mode: u=rw,g=r,o= - src: vault/vault.service.j2 - register: service_conf - - - name: Configure Hashicorp Vault TLS support if enabled - when: - - not specification.tls_disable - block: - - name: Look for certificate in role directory - local_action: find paths="{{ role_path }}/files/tls-certs" patterns="*.pem" - become: no - register: provided_certificates - - - name: Check if any certificate exists in role directory - set_fact: - certificate_exists: "{{ provided_certificates.matched > 0 }}" - - - name: Display result - debug: - msg: Certificate file {{ (provided_certificates.matched > 0) | ternary('found','not found') }} - - - name: Copy certificate to remote host - copy: - src: "{{ item.path }}" - dest: "{{ specification.vault_install_dir }}/tls/" - loop: "{{ provided_certificates.files }}" - when: - - certificate_exists - - - name: Generate TLS certificate - include_tasks: generate-certificate.yml - when: - - not certificate_exists - - - name: Restart Vault service and reload config - systemd: - name: vault - state: restarted - daemon_reload: yes - when: - - service_conf.changed - - - name: Start Vault service - systemd: - name: vault - state: started - enabled: yes - - - name: Copy Vault configuration script - copy: - src: vault/configure-vault.sh - dest: "{{ specification.vault_install_dir }}/bin/" - owner: "{{ specification.vault_system_user }}" - group: "{{ specification.vault_system_group }}" - mode: u=rwx,g=rx,o= - - - name: Generate admin policy - template: - dest: "{{ specification.vault_install_dir }}/config/policies/policy-admin.hcl" - owner: "{{ specification.vault_system_user }}" - group: "{{ specification.vault_system_group }}" - mode: u=rw,g=r,o= - src: vault/policy-admin.hcl.j2 - - - name: Generate provisioner policy - template: - dest: "{{ specification.vault_install_dir }}/config/policies/policy-provisioner.hcl" - owner: "{{ specification.vault_system_user }}" - group: "{{ specification.vault_system_group }}" - mode: u=rw,g=r,o= - src: vault/policy-provisioner.hcl.j2 - - - name: Generate application policy - template: - dest: "{{ specification.vault_install_dir }}/config/policies/policy-application.hcl" - owner: "{{ specification.vault_system_user }}" - group: "{{ specification.vault_system_group }}" - mode: u=rw,g=r,o= - src: vault/policy-application.hcl.j2 - - - name: Generate config file for Vault configuration script - template: - dest: "{{ specification.vault_install_dir }}/script.config" - owner: "{{ specification.vault_system_user }}" - group: "{{ specification.vault_system_group }}" - mode: u=rw,g=r,o= - src: vault/script.config.j2 - - - name: Generate file with users to be created by the script - template: - dest: "{{ specification.vault_install_dir }}/users.csv" - owner: "{{ specification.vault_system_user }}" - group: "{{ specification.vault_system_group }}" - mode: u=rw,g=r,o= - src: vault/users.csv.j2 - - - name: Copy Vault namespace yaml - copy: - src: kubernetes/vault-namespace.yml - dest: "{{ specification.vault_install_dir }}/kubernetes/" - owner: "{{ specification.vault_system_user }}" - group: "{{ specification.vault_system_group }}" - mode: u=rw,g=r,o= - - - name: Create policy yaml for default namespace - template: - dest: "{{ specification.vault_install_dir }}/kubernetes/vault-default-policy.yml" - owner: "{{ specification.vault_system_user }}" - group: "{{ specification.vault_system_group }}" - mode: u=rw,g=r,o= - src: kubernetes/vault-default-policy.yml.j2 - - - name: Create application namespace yaml - template: - dest: "{{ specification.vault_install_dir }}/kubernetes/app-namespace.yml" - owner: "{{ specification.vault_system_user }}" - group: "{{ specification.vault_system_group }}" - mode: u=rw,g=r,o= - src: kubernetes/app-namespace.yml.j2 - when: not specification.kubernetes_namespace == "default" - - - name: Create Vault service account yaml - template: - dest: "{{ specification.vault_install_dir }}/kubernetes/vault-service-account.yml" - owner: "{{ specification.vault_system_user }}" - group: "{{ specification.vault_system_group }}" - mode: u=rw,g=r,o= - src: kubernetes/vault-service-account.yml.j2 - - - name: Create application service account yaml - template: - dest: "{{ specification.vault_install_dir }}/kubernetes/app-service-account.yml" - owner: "{{ specification.vault_system_user }}" - group: "{{ specification.vault_system_group }}" - mode: u=rw,g=r,o= - src: kubernetes/app-service-account.yml.j2 - - - name: Check if Vault protocol is set to http - set_fact: - vault_protocol: http - when: specification.tls_disable - - - name: Check if Vault protocol is set to https - set_fact: - vault_protocol: https - when: not specification.tls_disable - - - name: Run configuration script - when: - - specification.vault_script_autoconfiguration - shell: >- - {{ specification.vault_install_dir }}/bin/configure-vault.sh - -c {{ specification.vault_install_dir }}/script.config - -a {{ ansible_default_ipv4.address }} - -p {{ vault_protocol }} - -v {{ vault_helm_chart_values_bool|lower }} - - - name: Display information about running configuration script - when: - - not specification.vault_script_autoconfiguration - debug: - msg: "Init file doesn't exist or you have chosen manual unsealing, so to finish configuration please run script manually with command: - {{ specification.vault_install_dir }}/bin/configure-vault.sh -c {{ specification.vault_install_dir }}/script.config -a {{ ansible_default_ipv4.address }} -p {{ vault_protocol }} -v {{ vault_helm_chart_values_bool|lower }}. - Also please put 'init.txt' file, containing output from 'vault operator init' command, in ''{{ specification.vault_install_dir }}' directory." diff --git a/ansible/playbooks/roles/vault/templates/kubernetes/app-namespace.yml.j2 b/ansible/playbooks/roles/vault/templates/kubernetes/app-namespace.yml.j2 deleted file mode 100644 index be24c156f4..0000000000 --- a/ansible/playbooks/roles/vault/templates/kubernetes/app-namespace.yml.j2 +++ /dev/null @@ -1,6 +0,0 @@ -# {{ ansible_managed }} ---- -apiVersion: v1 -kind: Namespace -metadata: - name: {{ specification.kubernetes_namespace }} diff --git a/ansible/playbooks/roles/vault/templates/kubernetes/app-service-account.yml.j2 b/ansible/playbooks/roles/vault/templates/kubernetes/app-service-account.yml.j2 deleted file mode 100644 index 559a237d0c..0000000000 --- a/ansible/playbooks/roles/vault/templates/kubernetes/app-service-account.yml.j2 +++ /dev/null @@ -1,7 +0,0 @@ -# {{ ansible_managed }} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - namespace: {{ specification.kubernetes_namespace }} - name: internal-app diff --git a/ansible/playbooks/roles/vault/templates/kubernetes/vault-default-policy.yml.j2 b/ansible/playbooks/roles/vault/templates/kubernetes/vault-default-policy.yml.j2 deleted file mode 100644 index 0242cb6eaf..0000000000 --- a/ansible/playbooks/roles/vault/templates/kubernetes/vault-default-policy.yml.j2 +++ /dev/null @@ -1,18 +0,0 @@ -# {{ ansible_managed }} ---- -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: vault-policy - namespace: default -spec: - podSelector: {} - policyTypes: - - Egress - egress: - - to: - {% for server in ansible_play_hosts_all %}- ipBlock: - cidr: {{ hostvars[server]['ansible_default_ipv4']['address'] }}/32 - {% endfor %}ports: - - protocol: TCP - port: 8200 diff --git a/ansible/playbooks/roles/vault/templates/kubernetes/vault-service-account.yml.j2 b/ansible/playbooks/roles/vault/templates/kubernetes/vault-service-account.yml.j2 deleted file mode 100644 index 947f409b9c..0000000000 --- a/ansible/playbooks/roles/vault/templates/kubernetes/vault-service-account.yml.j2 +++ /dev/null @@ -1,27 +0,0 @@ -# {{ ansible_managed }} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: vault-auth ---- -apiVersion: v1 -kind: Secret -metadata: - name: vault-auth - annotations: - kubernetes.io/service-account.name: vault-auth -type: kubernetes.io/service-account-token ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: role-tokenreview-binding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:auth-delegator -subjects: - - kind: ServiceAccount - name: vault-auth - namespace: default diff --git a/ansible/playbooks/roles/vault/templates/vault/config.hcl.j2 b/ansible/playbooks/roles/vault/templates/vault/config.hcl.j2 deleted file mode 100644 index 74203c3e32..0000000000 --- a/ansible/playbooks/roles/vault/templates/vault/config.hcl.j2 +++ /dev/null @@ -1,13 +0,0 @@ -# {{ ansible_managed }} -ui = {{ specification.enable_vault_ui | lower }} -storage "file" { - path = "{{ specification.vault_install_dir }}/data" -} -listener "tcp" { - address = "{{ ansible_default_ipv4.address }}:8200" - tls_disable = {{ specification.tls_disable | lower }} - {% if not specification.tls_disable -%} - tls_cert_file = "{{ specification.vault_install_dir }}/tls/{{ specification.certificate_name }}" - tls_key_file = "{{ specification.vault_install_dir }}/tls/{{ specification.private_key_name }}" - {% endif %} -} diff --git a/ansible/playbooks/roles/vault/templates/vault/policy-admin.hcl.j2 b/ansible/playbooks/roles/vault/templates/vault/policy-admin.hcl.j2 deleted file mode 100644 index 162fe9f020..0000000000 --- a/ansible/playbooks/roles/vault/templates/vault/policy-admin.hcl.j2 +++ /dev/null @@ -1,50 +0,0 @@ -# {{ ansible_managed }} -# Auth methods -path "auth/*" -{ - capabilities = ["create", "read", "update", "delete", "list", "sudo"] -} - -path "sys/auth/*" -{ - capabilities = ["create", "update", "delete", "sudo"] -} - -path "sys/auth" -{ - capabilities = ["read"] -} - -# Policies -path "sys/policies/acl" -{ - capabilities = ["list"] -} - -path "sys/policies/acl/*" -{ - capabilities = ["create", "read", "update", "delete", "list", "sudo"] -} - -# Secrets engines -path "sys/mounts/*" -{ - capabilities = ["create", "read", "update", "delete", "list", "sudo"] -} - -path "sys/mounts" -{ - capabilities = ["read"] -} - -# Health checks -path "sys/health" -{ - capabilities = ["read", "sudo"] -} - -# KV secrets from path {{ specification.secret_mount_path }} -path "{{ specification.secret_mount_path }}/*" -{ - capabilities = ["create", "read", "update", "delete", "list", "sudo"] -} diff --git a/ansible/playbooks/roles/vault/templates/vault/policy-application.hcl.j2 b/ansible/playbooks/roles/vault/templates/vault/policy-application.hcl.j2 deleted file mode 100644 index 6d7b7dc3b7..0000000000 --- a/ansible/playbooks/roles/vault/templates/vault/policy-application.hcl.j2 +++ /dev/null @@ -1,7 +0,0 @@ -# {{ ansible_managed }} -path "{{ specification.secret_mount_path }}/data/{{ specification.app_secret_path }}/*" { - capabilities = ["read"] -} -path "{{ specification.secret_mount_path }}/data/{{ specification.app_secret_path }}/" { - capabilities = ["list"] -} diff --git a/ansible/playbooks/roles/vault/templates/vault/policy-provisioner.hcl.j2 b/ansible/playbooks/roles/vault/templates/vault/policy-provisioner.hcl.j2 deleted file mode 100644 index 88ea5b5bed..0000000000 --- a/ansible/playbooks/roles/vault/templates/vault/policy-provisioner.hcl.j2 +++ /dev/null @@ -1,33 +0,0 @@ -# {{ ansible_managed }} -# Auth methods -path "auth/*" -{ - capabilities = ["create", "read", "update", "delete", "list", "sudo"] -} - -path "sys/auth/*" -{ - capabilities = ["create", "update", "delete", "sudo"] -} - -path "sys/auth" -{ - capabilities = ["read"] -} - -# Policies -path "sys/policies/acl" -{ - capabilities = ["list"] -} - -path "sys/policies/acl/*" -{ - capabilities = ["create", "read", "update", "delete", "list", "sudo"] -} - -# KV secrets from path {{ specification.secret_mount_path }} -path "{{ specification.secret_mount_path }}/*" -{ - capabilities = ["create", "read", "update", "delete", "list"] -} diff --git a/ansible/playbooks/roles/vault/templates/vault/script.config.j2 b/ansible/playbooks/roles/vault/templates/vault/script.config.j2 deleted file mode 100644 index 0fbf263500..0000000000 --- a/ansible/playbooks/roles/vault/templates/vault/script.config.j2 +++ /dev/null @@ -1,21 +0,0 @@ -# {{ ansible_managed }} -# Features -ENABLE_VAULT_AUDIT_LOGS={{ specification.enable_vault_audit_logs }} - -# Paths -VAULT_INSTALL_PATH="{{ specification.vault_install_dir }}" -SECRETS_ENGINE_PATH={{ specification.secret_mount_path }} - -# Actions -UNSEAL_VAULT={{ specification.vault_script_autoconfiguration }} -VAULT_TOKEN_CLEANUP={{ specification.vault_token_cleanup }} - -# Kubernetes integration -ENABLE_VAULT_KUBERNETES_AUTHENTICATION={{ specification.enable_vault_kubernetes_authentication }} -KUBERNETES_CONFIGURATION={{ specification.kubernetes_configuration }} -KUBERNETES_INTEGRATION={{ specification.kubernetes_integration }} -KUBERNETES_NAMESPACE={{ specification.kubernetes_namespace }} - -# Users -CREATE_VAULT_USERS={{ specification.vault_users is defined and specification.vault_users | list | length > 0 }} -OVERRIDE_EXISTING_VAULT_USERS={{ specification.override_existing_vault_users }} diff --git a/ansible/playbooks/roles/vault/templates/vault/selfsigned.cfr.j2 b/ansible/playbooks/roles/vault/templates/vault/selfsigned.cfr.j2 deleted file mode 100644 index 7b89887b77..0000000000 --- a/ansible/playbooks/roles/vault/templates/vault/selfsigned.cfr.j2 +++ /dev/null @@ -1,27 +0,0 @@ -[req] -distinguished_name = req_distinguished_name -x509_extensions = v3_req -prompt = no - -[req_distinguished_name] -C = {{ specification.selfsigned_certificate.country }} -ST = {{ specification.selfsigned_certificate.state }} -L = {{ specification.selfsigned_certificate.city }} -O = {{ specification.selfsigned_certificate.company }} -CN = {{ specification.selfsigned_certificate.common_name }} - -[v3_req] -subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid,issuer -basicConstraints = CA:TRUE -subjectAltName = @alt_names - -[alt_names] -DNS.1 = * -DNS.2 = *.* -DNS.3 = *.*.* -DNS.4 = *.*.*.* -DNS.5 = *.*.*.*.* -DNS.6 = *.*.*.*.*.* -DNS.7 = *.*.*.*.*.*.* -IP.1 = {{ ansible_default_ipv4.address }} diff --git a/ansible/playbooks/roles/vault/templates/vault/users.csv.j2 b/ansible/playbooks/roles/vault/templates/vault/users.csv.j2 deleted file mode 100644 index eb830a5560..0000000000 --- a/ansible/playbooks/roles/vault/templates/vault/users.csv.j2 +++ /dev/null @@ -1,4 +0,0 @@ -# {{ ansible_managed }} -{% for vault_user in specification.vault_users %} -{{ vault_user.name }};{{ vault_user.policy }}; -{% endfor %} diff --git a/ansible/playbooks/roles/vault/templates/vault/vault.service.j2 b/ansible/playbooks/roles/vault/templates/vault/vault.service.j2 deleted file mode 100644 index e196e735e7..0000000000 --- a/ansible/playbooks/roles/vault/templates/vault/vault.service.j2 +++ /dev/null @@ -1,44 +0,0 @@ -# {{ ansible_managed }} -[Unit] -Description=Hashicorp Vault server -Requires=network-online.target -After=network-online.target -ConditionFileNotEmpty={{ specification.vault_install_dir }}/config/config.hcl -StartLimitIntervalSec=60 -StartLimitBurst=3 - -[Service] -User={{ specification.vault_system_user }} -Group={{ specification.vault_system_group }} -Restart=on-failure -ProtectSystem=full -ProtectHome=read-only -PrivateTmp=yes -PrivateDevices=yes -SecureBits=keep-caps -AmbientCapabilities=CAP_IPC_LOCK -Capabilities=CAP_IPC_LOCK+ep -CapabilityBoundingSet=CAP_SYSLOG CAP_IPC_LOCK -NoNewPrivileges=yes -ExecStart={{ specification.vault_install_dir }}/bin/vault server -config={{ specification.vault_install_dir }}/config/config.hcl -log-level={{ specification.vault_log_level|lower }} -ExecReload=/bin/kill --signal HUP $MAINPID -KillMode=process -KillSignal=SIGINT -Restart=on-failure -RestartSec=5 -TimeoutStopSec=30 -StartLimitInterval=60 -StartLimitIntervalSec=60 -StartLimitBurst=3 -LimitNOFILE=65536 -LimitMEMLOCK=infinity -{% if specification.vault_script_autounseal is defined and specification.vault_script_autounseal == True %} -{% if specification.tls_disable is defined and specification.tls_disable == True %} -ExecStartPost=/opt/vault/bin/unseal-vault.sh {{ specification.vault_install_dir }}/init.txt {{ ansible_default_ipv4.address }} http -{% else %} -ExecStartPost=/opt/vault/bin/unseal-vault.sh {{ specification.vault_install_dir }}/init.txt {{ ansible_default_ipv4.address }} https -{% endif %} -{% endif %} - -[Install] -WantedBy=multi-user.target diff --git a/ansible/playbooks/vault.yml b/ansible/playbooks/vault.yml deleted file mode 100644 index c9ebd12fae..0000000000 --- a/ansible/playbooks/vault.yml +++ /dev/null @@ -1,15 +0,0 @@ ---- -# Ansible playbook that installs Hashicorp Vault -# Installing Vault on multi-master K8s is not supported so install only on the first host -- hosts: >- - {{ 'vault[0]' if (groups.kubernetes_master is defined and - groups.vault is defined and - groups.vault | intersect(groups.kubernetes_master)) else - 'vault' }} - become: true - become_method: sudo - roles: - - role: vault - when: - - specification.vault_enabled - - k8s_as_cloud_service is undefined or not k8s_as_cloud_service diff --git a/docs/architecture/current-c4/1_Epiphany_Component_Diagram_execution_place.png b/docs/architecture/current-c4/1_Epiphany_Component_Diagram_execution_place.png index 27c7f75750..50af13f52c 100644 --- a/docs/architecture/current-c4/1_Epiphany_Component_Diagram_execution_place.png +++ b/docs/architecture/current-c4/1_Epiphany_Component_Diagram_execution_place.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:139d3dc61a422df10033371f13b2c66fd6153708bfaecc01af2d131fd912c93f -size 120608 +oid sha256:29f7e7f900f000e16bede8674e88b93ee4b310ee29373ea346073d4ecc9e3b96 +size 59016 diff --git a/docs/architecture/current-c4/2_Epiphany_Component_Diagram_human_interactions.png b/docs/architecture/current-c4/2_Epiphany_Component_Diagram_human_interactions.png index 84ce2ebe88..6a64858f73 100644 --- a/docs/architecture/current-c4/2_Epiphany_Component_Diagram_human_interactions.png +++ b/docs/architecture/current-c4/2_Epiphany_Component_Diagram_human_interactions.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:146303da3eaf7cba8783fe81406fe3bcdb4f0e0f215d9c75ecab9fec6204f68f -size 180886 +oid sha256:3ef6d7ac518fb77466a6ea9d919a6a133df64dc42ddc86ad7425e5a25d1b5d5c +size 166969 diff --git a/docs/architecture/current-c4/3_Epiphany_Component_Diagram_runtime_dependencies.png b/docs/architecture/current-c4/3_Epiphany_Component_Diagram_runtime_dependencies.png index 0f74a26819..ef55716128 100644 --- a/docs/architecture/current-c4/3_Epiphany_Component_Diagram_runtime_dependencies.png +++ b/docs/architecture/current-c4/3_Epiphany_Component_Diagram_runtime_dependencies.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3de4317c730ed9f5e4843a553f8108fb93a703e84aff222b1c0eadb709cb557d -size 121048 +oid sha256:b7faa4e735f5bdaae48202c685360cc94b709f034d68da95a6e0c59581a53294 +size 105211 diff --git a/docs/architecture/current-c4/4_Epiphany_Component_Diagram_metrics_and_logs.png b/docs/architecture/current-c4/4_Epiphany_Component_Diagram_metrics_and_logs.png index 812e556ae3..fd69873eeb 100644 --- a/docs/architecture/current-c4/4_Epiphany_Component_Diagram_metrics_and_logs.png +++ b/docs/architecture/current-c4/4_Epiphany_Component_Diagram_metrics_and_logs.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:02114f89fbb95603965ec90c8d2bcbb3d3c73b7f2fc7e0885fc5c28b5b28c8de -size 124972 +oid sha256:fa7c558959f4f1baeadfea50802aa32425a825bd53e4cd2bca4baf30ff1573ef +size 112939 diff --git a/docs/architecture/current-c4/c2-arch-1-execution.puml b/docs/architecture/current-c4/c2-arch-1-execution.puml index c484161f04..de742a943e 100644 --- a/docs/architecture/current-c4/c2-arch-1-execution.puml +++ b/docs/architecture/current-c4/c2-arch-1-execution.puml @@ -15,7 +15,6 @@ Container_Boundary(epiphany, "Epiphany managed environment"){ Container(lb, "Load Balancer", "HAProxy") Container(messages, "Message Broker", "RabbitMQ") Container(memdb, "In-Memory Database", "Apache Ignite") - Container(vault, "Vault", "Hashicorp Vault") Container(repository, "Repository", "Apache") Container(auth, "Authentication and Authorization", "Keycloak") @@ -36,7 +35,6 @@ Rel(monitoring, provider, "executes on") Rel(streams, provider, "executes on") Rel(rdb, provider, "executes on") Rel(lb, provider, "executes on") -Rel(vault, provider, "executes on") Rel(repository, provider, "executes on") Rel(messages, provider, "can execute on") diff --git a/docs/architecture/current-c4/c2-arch-2-human-interactions.puml b/docs/architecture/current-c4/c2-arch-2-human-interactions.puml index b6916c6a37..b687ebb422 100644 --- a/docs/architecture/current-c4/c2-arch-2-human-interactions.puml +++ b/docs/architecture/current-c4/c2-arch-2-human-interactions.puml @@ -20,7 +20,6 @@ Container_Boundary(epiphany, "Epiphany managed environment"){ Container(lb, "Load Balancer", "HAProxy") Container(messages, "Message Broker", "RabbitMQ") Container(memdb, "In-Memory Database", "Apache Ignite") - Container(vault, "Vault", "Hashicorp Vault") Container(repository, "Repository", "Apache") Container(auth, "Authentication and Authorization", "Keycloak") @@ -40,7 +39,6 @@ Rel(developer, memdb, "manages") Rel(developer, messages, "manages") Rel(developer, auth, "manages") Rel(developer, lb, "manages") -Rel(developer, vault, "manages") Rel(developer, workload, "provides") Rel(user, lb, "accesses", "TCP") @@ -53,7 +51,6 @@ Rel(epicli, monitoring, "operates") Rel(epicli, streams, "operates") Rel(epicli, rdb, "operates") Rel(epicli, lb, "operates") -Rel(epicli, vault, "operates") Rel(epicli, repository, "operates") Rel(epicli, memdb, "operates") Rel(epicli, messages, "operates") diff --git a/docs/architecture/current-c4/c2-arch-3-runtime-dependencies.puml b/docs/architecture/current-c4/c2-arch-3-runtime-dependencies.puml index a67d75d191..d5991a6800 100644 --- a/docs/architecture/current-c4/c2-arch-3-runtime-dependencies.puml +++ b/docs/architecture/current-c4/c2-arch-3-runtime-dependencies.puml @@ -17,7 +17,6 @@ Container_Boundary(epiphany, "Epiphany managed environment"){ Container(lb, "Load Balancer", "HAProxy") Container(messages, "Message Broker", "RabbitMQ") Container(memdb, "In-Memory Database", "Apache Ignite") - Container(vault, "Vault", "Hashicorp Vault") Container(repository, "Repository", "Apache") Container(auth, "Authentication and Authorization", "Keycloak") @@ -25,7 +24,6 @@ Container_Boundary(epiphany, "Epiphany managed environment"){ Rel(lb, orchestrator, "sends traffic to") - Rel(vault, orchestrator, "injects secrets to") } System(workload, "Application", "Provides business value.") @@ -39,7 +37,6 @@ Rel(workload, rdb, "uses") Rel(workload, memdb, "uses") Rel(workload, messages, "uses") Rel(workload, auth, "uses") -Rel(workload, vault, "uses") Rel(logging, workload, "transfers logs from") Rel(monitoring, workload, "scraps metrics from") diff --git a/docs/architecture/current-c4/c2-arch-4-metrics-and-logs.puml b/docs/architecture/current-c4/c2-arch-4-metrics-and-logs.puml index 854b6da920..15b2d1de9e 100644 --- a/docs/architecture/current-c4/c2-arch-4-metrics-and-logs.puml +++ b/docs/architecture/current-c4/c2-arch-4-metrics-and-logs.puml @@ -15,7 +15,6 @@ Container_Boundary(epiphany, "Epiphany managed environment"){ Container(lb, "Load Balancer", "HAProxy") Container(messages, "Message Broker", "RabbitMQ") Container(memdb, "In-Memory Database", "Apache Ignite") - Container(vault, "Vault", "Hashicorp Vault") Container(repository, "Repository", "Apache") Container(auth, "Authentication and Authorization", "Keycloak") @@ -28,7 +27,6 @@ Container_Boundary(epiphany, "Epiphany managed environment"){ Rel(orchestrator, logging, "sends logs") Rel(repository, logging, "sends logs") Rel(memdb, logging, "sends logs") - Rel(vault, logging, "sends logs") Rel(auth, logging, "sends logs") Rel(monitoring, streams, "scraps metrics") @@ -40,7 +38,6 @@ Container_Boundary(epiphany, "Epiphany managed environment"){ Rel(monitoring, orchestrator, "scraps metrics") Rel(monitoring, repository, "scraps metrics") Rel(monitoring, memdb, "scraps metrics") - Rel(monitoring, vault, "scraps metrics") Rel(monitoring, auth, "scraps metrics") } diff --git a/docs/changelogs/CHANGELOG-2.0.md b/docs/changelogs/CHANGELOG-2.0.md new file mode 100644 index 0000000000..16ffdc8e11 --- /dev/null +++ b/docs/changelogs/CHANGELOG-2.0.md @@ -0,0 +1,19 @@ +# Changelog 1.3 + +## [1.3.0] YYYY-MM-DD + +### Added + +### Fixed + +### Updated + +### Removed + +- [#2834](https://github.com/epiphany-platform/epiphany/issues/2834) - Removal of Hashicorp Vault component + +### Deprecated + +### Breaking changes + +### Known issues diff --git a/docs/design-docs/k8s-vault-integration/k8s-vault-integration.md b/docs/design-docs/k8s-vault-integration/k8s-vault-integration.md index 610f5d9eb2..b3b2eef36a 100644 --- a/docs/design-docs/k8s-vault-integration/k8s-vault-integration.md +++ b/docs/design-docs/k8s-vault-integration/k8s-vault-integration.md @@ -1,6 +1,6 @@ # Epiphany Platform Kubernetes with Hashicorp Vault integration -Affected version: 0.7.x +Affected version: Left for future reference. Hashicorp Vault has been removed in Epiphany 2.0. ## 1. Introduction diff --git a/docs/home/ARM.md b/docs/home/ARM.md index 2510fdd533..59fda6a113 100644 --- a/docs/home/ARM.md +++ b/docs/home/ARM.md @@ -276,13 +276,6 @@ specification: enabled_plugins: - ignite-kubernetes # required to work on K8s - ignite-rest-http ---- -kind: configuration/vault -title: Vault Config -name: default -provider: any -specification: - vault_enabled: true ``` ### ```AWS``` provider @@ -551,13 +544,6 @@ specification: enabled_plugins: - ignite-kubernetes # required to work on K8s - ignite-rest-http ---- -kind: configuration/vault -title: Vault Config -name: default -provider: aws -specification: - vault_enabled: true ``` ### ```Azure``` provider diff --git a/docs/home/COMPONENTS.md b/docs/home/COMPONENTS.md index b415c60cac..fd1cb811a4 100644 --- a/docs/home/COMPONENTS.md +++ b/docs/home/COMPONENTS.md @@ -40,8 +40,6 @@ Note that versions are default versions and can be changed in certain cases thro | Alertmanager | 0.23.0 | https://github.com/prometheus/alertmanager | [Apache License 2.0](https://github.com/prometheus/alertmanager/blob/master/LICENSE) | | Apache Ignite | 2.9.1 | https://github.com/apache/ignite | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | | Apache2 | 2.4.29 | https://httpd.apache.org/ | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | -| Hashicorp Vault | 1.7.0 | https://github.com/hashicorp/vault | [Mozilla Public License 2.0](https://github.com/hashicorp/vault/blob/master/LICENSE) | -| Hashicorp Vault Helm Chart | 0.11.0 | https://github.com/hashicorp/vault-helm | [Mozilla Public License 2.0](https://github.com/hashicorp/vault-helm/blob/master/LICENSE.md) | | Istio | 1.8.1 | https://github.com/istio/istio | [Apache License 2.0](https://github.com/istio/istio/blob/master/LICENSE) | ## Epicli binary dependencies diff --git a/docs/home/DEPRECATION-NOTE.md b/docs/home/DEPRECATION-NOTE.md index bc8f8f3173..8986b3e27e 100644 --- a/docs/home/DEPRECATION-NOTE.md +++ b/docs/home/DEPRECATION-NOTE.md @@ -3,6 +3,15 @@ This page is related to our components deprecation plan. Components listed below are marked as deprecated with plan of removal in Epiphany version 2.0. +The list of components removed: + +- Hashicorp Vault + +The role related to Hashicorp Vault has never been developed beyond MVP/PoC stage and cannot be used for production usage. It also causes troubles with upgrade to new Kubernetes versions. + +Removed: Epiphany 2.0 +Issue: [2834](https://github.com/epiphany-platform/epiphany/issues/2834) + The list of components to be deprecated: - Logstash @@ -12,13 +21,6 @@ The functionality that the Logstash is serving in Epiphany - exporting csv can r Planed removal: Epiphany 2.0 Issue: [2833](https://github.com/epiphany-platform/epiphany/issues/2833) -- Hashicorp Vault - -The role related to Hashicorp Vault has never been developed beyond MVP/PoC stage and cannot be used for production usage. It also causes troubles with upgrade to new Kubernetes versions. - -Planed removal: Epiphany 2.0 -Issue: [2834](https://github.com/epiphany-platform/epiphany/issues/2834) - - Istio We are considering replacement of Istio component with different solution. Additionally, Istio causes problems during upgrade of Kubernetes. diff --git a/docs/home/HOWTO.md b/docs/home/HOWTO.md index 8d89348fc1..66bac44a5a 100644 --- a/docs/home/HOWTO.md +++ b/docs/home/HOWTO.md @@ -80,8 +80,6 @@ - [How to authenticate to Azure AD app](./howto/SECURITY.md#how-to-authenticate-to-azure-ad-app) - [How to run epicli with password](./howto/SECURITY.md#how-to-run-epicli-with-password) - [How to make kubectl work for non-root user on master node](./howto/SECURITY.md#how-to-make-kubectl-work-for-non-root-user-on-master-node) - - [How to turn on Hashicorp Vault functionality](./howto/SECURITY.md#how-to-turn-on-hashicorp-vault-functionality) - - [How to turn on Hashicorp Vault integration with k8s](./howto/SECURITY.md#how-to-turn-on-hashicorp-vault-integration-with-k8s) - [Databases](./howto/DATABASES.md) - [How to configure PostgreSQL](./howto/DATABASES.md#how-to-configure-postgresql) diff --git a/docs/home/SECURITY.md b/docs/home/SECURITY.md index 2b466a0514..50978aece9 100644 --- a/docs/home/SECURITY.md +++ b/docs/home/SECURITY.md @@ -28,7 +28,6 @@ Additional to users created by each component Epiphany creates also users and gr - rabbitmq/rabbitmq - zookeeper/zookeeper - kafka/kafka -- vault/vault Other accounts created by each component you can find in the documentation of these components. @@ -146,14 +145,10 @@ different values. The list does not include ports that are bound to the loopback - 80 - deb/rpm package repository (httpd is stopped at the end of installation) - 5000 - Docker image registry -17. Hashicorp Vault: - - - 8200 - REST API - ### Connection protocols and ciphers used by components in Epiphany Below you can find list of cipersuites and protocols used for communication set in Epiphany on per component basis. This -doesn't contain information about potentially deprecated components: Apache Ignite, Istio, Hashicorp Vault. +doesn't contain information about potentially deprecated components: Apache Ignite, Istio. 1. OS services: diff --git a/docs/home/howto/CLUSTER.md b/docs/home/howto/CLUSTER.md index b5e3c9fecb..1e77191d3a 100644 --- a/docs/home/howto/CLUSTER.md +++ b/docs/home/howto/CLUSTER.md @@ -351,7 +351,6 @@ specification: - node-exporter - filebeat - firewall - - vault --- kind: configuration/shared-config title: Shared configuration that will be visible to all roles diff --git a/docs/home/howto/SECURITY.md b/docs/home/howto/SECURITY.md index 9d3c0f3fdd..9949a77fde 100644 --- a/docs/home/howto/SECURITY.md +++ b/docs/home/howto/SECURITY.md @@ -445,307 +445,3 @@ mkdir -p $HOME/.kube sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config sudo chown $(id -u):$(id -g) $HOME/.kube/config ``` - -## How to turn on Hashicorp Vault functionality - -In Epiphany beside storing secrets in Kubernetes secrets there is also a possibility of using secrets stored in Vault -from Hashicorp. This can provide much more sophisticated solution for using secrets and also higher level of security -than standard Kubernetes secrets implementation. Also Epiphany provides transparent method to access Hashicorp Vault -secrets with applications running on Kubernetes. You can read in the more about it in [How to turn on Hashicorp Vault integration with k8s](./SECURITY.md#how-to-turn-on-hashicorp-vault-integration-with-k8s) section. In the future we want also to provide additional features -that right now can be configured manually according to Hashicorp Vault [documentation](https://www.vaultproject.io/docs). - -At the moment only installation on Kubernetes Master is supported, but we are also planning separate installation with no -other components. Also at this moment we are not providing clustered option for Vault deployment, but this will be part -of the future releases. For multi-master (HA) Kubernetes, Vault is installed only on the first master defined in Ansible inventory. - -Below you can find sample configuration for Vault with description of all options. - -```yaml -kind: configuration/vault -title: Vault Config -name: default -specification: - vault_enabled: true # enable Vault install - vault_system_user: vault # user name under which Vault service will be running - vault_system_group: vault # group name under which Vault service will be running - enable_vault_audit_logs: false # turn on audit logs that can be found at /opt/vault/logs/vault_audit.log - enable_vault_ui: false # enable Vault UI, shouldn't be used at production - vault_script_autounseal: true # enable automatic unseal vault at the start of the service, shouldn't be used at production - vault_script_autoconfiguration: true # enable automatic configuration of Hashicorp Vault. It sets the UNSEAL_VAULT variable in script.config - ... - app_secret_path: devwebapp # application specific path where application secrets will be mounted - revoke_root_token: false # not implemented yet (more about in section Root token revocation) - secret_mount_path: secret # start of the path that where secrets will be mounted - vault_token_cleanup: true # should configuration script clean token - vault_install_dir: /opt/vault # directory where vault will be installed - vault_log_level: info # logging level that will be set for Vault service - override_existing_vault_users: false # should user from vault_users ovverride existing user and generate new password - vault_users: # users that will be created with vault - - name: admin # name of the user that will be created in Vault - policy: admin # name of the policy that will be assigned to user (descrption bellow) - - name: provisioner - policy: provisioner - vault_helm_chart_values: # helm chart values overwriting the default package (to be able to use internal registry for offline purposes) - injector: - externalVaultAddr: https://your-external-address:8200 # external vault address (only if you want to setup address to provide full name to use with signed certificate) [IMPORTANT: switch https->http if tls_disable parameter is set to true] - image: - repository: "{{ image_registry_address }}/hashicorp/vault-k8s" # docker image used by vault injector in kubernetes - agentImage: - repository: "{{ image_registry_address }}/vault" # docker image used by vault injector in kubernetes - server: - image: - repository: "{{ image_registry_address }}/vault" # docker image used by vault injector in kubernetes - # TLS part - tls_disable: false # enable TLS support, should be used always in production - certificate_name: fullchain.pem # certificate file name - private_key_name: privkey.pem # private key file name for certificate - vault_tls_valid_days: 365 # certificate valid time in days - selfsigned_certificate: # selfsigned certificate information - country: US # selfexplanatory - state: state # selfexplanatory - city: city # selfexplanatory - company: company # selfexplanatory - common_name: "*" # selfexplanatory - -``` - -More information about configuration of Vault in Epiphany and some guidance how to start working with Vault with Epiphany you can find below. - -To get more familiarity with Vault usage you can reffer to [official getting started](https://learn.hashicorp.com/vault) guide. - -### Creation of user using Epiphany in Vault - -To create user by Epiphany please provide list of users with name of policy that should be assigned to user. You can -use predefined policy delivered by Epiphany, default Vault policies or your own policy. Remember that if you have -written your own policy it must exist before user creation. - -Password for user will be generated automatically and can be found in directory /opt/vault in files matching -tokens-*.csv pattern. If user password will be generated or changed you will see corresponding line in csv file with -username, policy and password. If password won't be updated you will see `ALREADY_EXISTS` in password place. - -### Predefined Vault policies - -Vault policies are used to define Role-Based Access Control that can be assigned to clients, applications and other -components that are using Vault. You can find more information about policies [here](https://www.hashicorp.com/resources/policies-vault/). - -Epiphany besides two already included in vault policies (root and default) provides two additional predefined policies: - -- admin - policy granting administration privileges, have sudo permission on Vault system endpoints -- provisioner - policy granting permissions to create user secrets, adding secrets, enable authentication methods, but - without access to Vault system endpoints - -### Manual unsealing of the Vault - -By design Hashicorp Vault starts in sealed mode. It means that Vault data is encrypted and operator needs to provide unsealing key to be able to access data. - -Vault can be unsealed manually using command: - -```bash -vault operator unseal -``` - -and passing three unseal keys from /opt/vault/init.txt file. -Number of keys will be defined from the level of Epiphany configuration in the future releases. -Right now we are using default Hashicorp Vault settings. - -For development purposes you can also use `vault_script_autounseal` option in Epiphany configuration. - -More information about unseal you can find in documentation for [CLI](https://www.vaultproject.io/docs/commands/operator/unseal) -and about concepts [here](https://www.vaultproject.io/docs/concepts/seal). - -### Configuration with manual unsealing - -If you are using option with manual unseal or want to perform manual configuration you can run script later on manually -from the command line: - -```bash -/opt/vault/bin/configure-vault.sh - -c /opt/vault/script.config - -a ip_address_of_vault - -p http | https - -v helm_chart_values_be_override -``` - -Values for script configuration in script.config are automatically generated by Epiphany and can be later on used to -perform configuration. - -### Log into Vault with token - -To log into Vault with token you just need to pass token. You can do this using command: - -```bash -vault login -``` - -Only root token has no expiration date, so be aware that all other tokens can expire. To avoid such situations you need -to renew the token. You can assign policy to token to define access. - -More information about login with tokens you can find [here](https://www.vaultproject.io/docs/commands/login) and about -tokens [here](https://www.vaultproject.io/docs/concepts/tokens). - -### Log into Vault with user and password - -Other option to log into Vault is to use user/password pair. This method doesn't have disadvantage of login each time -with different token after expire. To login with user/password pair you need to have userpass method and login with command: - -```bash -vault login -method=userpass username=your-username -``` - -More information about login with tokens you can find [here](https://www.vaultproject.io/docs/commands/login) and about -userpass authentication [here](https://www.vaultproject.io/docs/auth/userpass). - -### Token Helpers - -Vault provide option to use token helper. By default Vault is creating a file .vault-token in home directory of user -running command vault login, which let to user perform automatically commands without providing a token. This token -will be removed by default after Epiphany configuration, but this can be changed using `vault_token_cleanup flag`. - -More information about token helper you can find [here](https://www.vaultproject.io/docs/commands/token-helper). - -### Creating your own policy - -In order to create your own policy using CLI please refer to [CLI documentation](https://www.vaultproject.io/docs/commands) -and [documentation](https://www.vaultproject.io/docs/concepts/policies). - -### Creating your own user - -In order to create your own user with user and password login please refer to [documentation](https://www.vaultproject.io/docs/auth/userpass). -If you have configured any user using Epiphany authentication userpass will be enabled, if not needs to be enabled manually. - -### Root token revocation - -In production is a good practice to [revoke root token](https://www.vaultproject.io/docs/commands/token/revoke). This option is not implemented yet, -by Epiphany, but will be implemented in the future releases. - -Be aware that after revoking root token you won't be able to use configuration script without generating new token -and replace old token with the new one in /opt/vault/init.txt (field `Initial Root Token`). For new root token generation -please refer to documentation accessible [here](https://learn.hashicorp.com/vault/operations/ops-generate-root). - -### TLS support - -By default tls_disable is set to false which means that certificates are used by vault. There are 2 ways of certificate configuration: - -1. selfsigned - -Vault selfsigned certificates are generated automatically during vault setup if no custom certificates are present in dedicated location. - -2. certificate provided by user - -In dedicated location user can add certificate (and private key). File names are important and have to be the same as provided in configuration and ```.pem``` file extensions are required. - -Dedicated location of custom certificates: -```core/src/epicli/data/common/ansible/playbooks/roles/vault/files/tls-certs``` - -Certificate files names configuration: - -```yaml -kind: configuration/vault -title: Vault Config -name: default -specification: -... - certificate_name: fullchain.pem # certificate file name - private_key_name: privkey.pem # private key file name for certificate -... -``` - -### Production hardening for Vault - -In Epiphany we have performed a lot of things to improve Vault security, e.g.: - -- End-to-End TLS -- Disable Swap (when running on Kubernetes machine) -- Don't Run as Root -- Turn Off Core -- Enable Auditing -- Restrict Storage Access -- Tweak ulimits - -However if you want to provide more security please refer to this -[guide](https://learn.hashicorp.com/vault/operations/production-hardening). - -### Troubleshooting - -To perform troubleshooting of vault and find the root cause of the problem please enable audit logs and set vault_log_level -to debug. Please be aware that audit logs can contain sensitive data. - -## How to turn on Hashicorp Vault integration with k8s - -In Epiphany there is also an option to configure automatically integration with Kubernetes. This is achieved -with applying additional settings to Vault configuration. Sample config with description you can find below. - -```yaml -kind: configuration/vault -title: Vault Config -name: default -specification: - vault_enabled: true - ... - vault_script_autounseal: true - vault_script_autoconfiguration: true - ... - kubernetes_integration: true # enable setup kubernetes integration on vault side - kubernetes_configuration: true # enable setup kubernetes integration on vault side - enable_vault_kubernetes_authentication: true # enable kubernetes authentication on vault side - kubernetes_namespace: default # namespace where your application will be deployed - ... -``` - -Vault and Kubernetes integration in Epiphany relies on vault-k8s tool. -Thit tool enables sidecar injection of secret into pod with usage of Kubernetes Mutating Admission Webhook. This is transparent -for your application and you do not need to perform any binding to Hashicorp libaries to use secret stored in Vault. - -You can also configure Vault manually on your own enabling by Epiphany only options that are necessary for you. - -More about Kubernetes sidecar integration you can find at the [link](https://www.hashicorp.com/blog/injecting-vault-secrets-into-kubernetes-pods-via-a-sidecar/). - -### Vault Kubernetes authentication - -To work with sidecar integration with Vault you need to enable Kubernetes authentication. Without that sidecar won't be able -to access secret stored in Vault. - -If you don't want to use sidecar integration, but you want to access automatically Vault secrets you can use Kubernetes -authentication. To find more information about capabilities of Kubernetes authentication please refer to [documentation](https://www.vaultproject.io/docs/auth/kubernetes). - -### Create your secret in Vault - -In Epiphany you can use integration of key value secrets to inject them into container. To do this you need to create them -using vault CLI. - -You can do this running command similar to sample below: - -```shell -vault kv put secret/yourpath/to/secret username='some_user' password='some_password' -``` - -Epiphany as backend for Vault secrets is using kv secrets engine. More information about kv secrets engine you can find -[here](https://www.vaultproject.io/docs/secrets/kv). - -### Kubernetes namespace - -In Epiphany we are creating additional Kubernetes objects to inject secrets automatically using sidecar. Those objects -to have access to your application pods needs to be deployed in the same namespace. - -### Annotations - -Below you can find sample of deployment configuration excerpt with annotations. For this moment `vault.hashicorp.com/role` -cannot be changed, but this will change in future release. - -```yaml - template: - metadata: - labels: - app: yourapp - annotations: - vault.hashicorp.com/agent-inject: "true" - vault.hashicorp.com/role: "devweb-app" - vault.hashicorp.com/agent-inject-secret-credentials.txt: "secret/data/yourpath/to/secret" - vault.hashicorp.com/tls-skip-verify: "true" -``` - -```vault.hashicorp.com/tls-skip-verify``` -If true, configures the Vault Agent to skip verification of Vault's TLS certificate. -It's mandatory for selfsigned certificates and not recommended to set this value to true in a production environment. - -More information about annotations you can find [here](https://www.vaultproject.io/docs/platform/k8s/injector/annotations). diff --git a/docs/home/howto/UPGRADE.md b/docs/home/howto/UPGRADE.md index 171c88ee7c..4a63212ef4 100644 --- a/docs/home/howto/UPGRADE.md +++ b/docs/home/howto/UPGRADE.md @@ -215,7 +215,6 @@ specification: - node-exporter - filebeat - firewall - - vault - repository # add repository here - image-registry # add image-registry here ... diff --git a/schema/common/defaults/configuration/feature-mapping.yml b/schema/common/defaults/configuration/feature-mapping.yml index cdea25b7c7..6d7400eaa9 100644 --- a/schema/common/defaults/configuration/feature-mapping.yml +++ b/schema/common/defaults/configuration/feature-mapping.yml @@ -49,8 +49,6 @@ specification: enabled: true - name: haproxy enabled: true - - name: vault - enabled: true - name: applications enabled: true - name: ignite @@ -115,7 +113,6 @@ specification: - rabbitmq - postgresql - firewall - - vault kubernetes_master: - kubernetes-master - helm @@ -123,7 +120,6 @@ specification: - node-exporter - filebeat - firewall - - vault kubernetes_node: - kubernetes-node - node-exporter diff --git a/schema/common/defaults/configuration/image-registry.yml b/schema/common/defaults/configuration/image-registry.yml index 549a9fd518..eacdcaa4d4 100644 --- a/schema/common/defaults/configuration/image-registry.yml +++ b/schema/common/defaults/configuration/image-registry.yml @@ -19,10 +19,6 @@ specification: file_name: dashboard-v2.3.1.tar - name: "kubernetesui/metrics-scraper:v1.0.7" file_name: metrics-scraper-v1.0.7.tar - - name: "vault:1.7.0" - file_name: vault-1.7.0.tar - - name: "hashicorp/vault-k8s:0.10.0" - file_name: vault-k8s-0.10.0.tar - name: "istio/proxyv2:1.8.1" file_name: proxyv2-1.8.1.tar - name: "istio/pilot:1.8.1" @@ -144,10 +140,6 @@ specification: file_name: dashboard-v2.3.1.tar - name: "kubernetesui/metrics-scraper:v1.0.7" file_name: metrics-scraper-v1.0.7.tar - - name: "vault:1.7.0" - file_name: vault-1.7.0.tar - - name: "hashicorp/vault-k8s:0.10.0" - file_name: vault-k8s-0.10.0.tar current: - name: "haproxy:2.2.2-alpine" file_name: haproxy-2.2.2-alpine.tar diff --git a/schema/common/defaults/configuration/vault.yml b/schema/common/defaults/configuration/vault.yml deleted file mode 100644 index e416a25f84..0000000000 --- a/schema/common/defaults/configuration/vault.yml +++ /dev/null @@ -1,48 +0,0 @@ -kind: configuration/vault -title: Vault Config -name: default -specification: - vault_enabled: false - vault_system_user: vault - vault_system_group: vault - enable_vault_audit_logs: false - enable_vault_ui: false - vault_script_autounseal: true - vault_script_autoconfiguration: true - tls_disable: false - kubernetes_integration: true - kubernetes_configuration: true - kubernetes_namespace: default - enable_vault_kubernetes_authentication: true - app_secret_path: devwebapp - revoke_root_token: false - secret_mount_path: secret - vault_token_cleanup: true - vault_install_dir: /opt/vault - vault_log_level: info - override_existing_vault_users: false - certificate_name: fullchain.pem - private_key_name: privkey.pem - selfsigned_certificate: - country: US - state: state - city: city - company: company - common_name: "*" - vault_tls_valid_days: 365 - vault_users: - - name: admin - policy: admin - - name: provisioner - policy: provisioner - files: - vault_helm_chart_file_name: v0.11.0.tar.gz - vault_helm_chart_values: - injector: - image: - repository: "{{ image_registry_address }}/hashicorp/vault-k8s" - agentImage: - repository: "{{ image_registry_address }}/vault" - server: - image: - repository: "{{ image_registry_address }}/vault" diff --git a/schema/common/validation/configuration/vault.yml b/schema/common/validation/configuration/vault.yml deleted file mode 100644 index 93d5c1b606..0000000000 --- a/schema/common/validation/configuration/vault.yml +++ /dev/null @@ -1,100 +0,0 @@ -"$id": "#/specification" -title: "Vault specification schema" -description: "Vault specification schema" -type: object -properties: - vault_enabled: - type: boolean - vault_system_user: - type: string - vault_system_group: - type: string - enable_vault_audit_logs: - type: boolean - enable_vault_ui: - type: boolean - vault_script_autounseal: - type: boolean - vault_script_autoconfiguration: - type: boolean - tls_disable: - type: boolean - kubernetes_integration: - type: boolean - kubernetes_configuration: - type: boolean - kubernetes_namespace: - type: string - enable_vault_kubernetes_authentication: - type: boolean - app_secret_path: - type: string - revoke_root_token: - type: boolean - secret_mount_path: - type: string - vault_token_cleanup: - type: boolean - vault_install_dir: - type: string - vault_log_level: - type: string - override_existing_vault_users: - type: boolean - certificate_name: - type: string - private_key_name: - type: string - selfsigned_certificate: - type: object - properties: - country: - type: string - state: - type: string - city: - type: string - company: - type: string - common_name: - type: string - vault_tls_valid_days: - type: integer - vault_users: - type: array - items: - type: object - properties: - name: - type: string - policy: - type: string - files: - type: object - properties: - vault_helm_chart_file_name: - type: string - vault_helm_chart_values: - type: object - properties: - injector: - type: object - properties: - image: - type: object - properties: - repository: - type: string - agentImage: - type: object - properties: - repository: - type: string - server: - type: object - properties: - image: - type: object - properties: - repository: - type: string