Backport of test: fix drift between enos directories into release/1.13.x

.github/workflows/enos-fmt.yml → .github/workflows/enos-lint.yml

@@ -1,27 +1,32 @@
 ---
-name: enos_fmt
+name: lint-enos
 
 on:
   pull_request:
     paths:
       - enos/**
 
 jobs:
-  fmt_check:
+  lint:
     # Only run this workflow on pull requests from hashicorp/vault branches
     # as we need secrets to install enos.
     if: "! github.event.pull_request.head.repo.fork"
     runs-on: ubuntu-latest
     env:
       GITHUB_TOKEN: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
+      ENOS_VAR_tfc_api_token: ${{ secrets.TF_API_TOKEN }}
     steps:
       - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+      - id: get-version
+        run: echo "version=$(make ci-get-version)" >> "$GITHUB_OUTPUT"
       - uses: hashicorp/setup-terraform@v2
         with:
           terraform_wrapper: false
       - uses: hashicorp/action-setup-enos@v1
         with:
           github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
-      - name: check formatting
+      - name: lint
         working-directory: ./enos
-        run: make check-fmt
+        env:
+          ENOS_VAR_vault_product_version: ${{ steps.get-version.outputs.version }}
+        run: make lint

.github/workflows/test-run-enos-scenario-matrix.yml

@@ -99,10 +99,10 @@ jobs:
       ENOS_VAR_artifactory_username: ${{ secrets.ARTIFACTORY_USER }}
       ENOS_VAR_artifactory_token: ${{ secrets.ARTIFACTORY_TOKEN }}
       ENOS_VAR_terraform_plugin_cache_dir: ./support/terraform-plugin-cache
+      ENOS_VAR_vault_artifact_path: ./support/downloads/${{ inputs.build-artifact-name }}
       ENOS_VAR_vault_build_date: ${{ needs.metadata.outputs.build-date }}
       ENOS_VAR_vault_product_version: ${{ needs.metadata.outputs.version }}
       ENOS_VAR_vault_revision: ${{ inputs.vault-revision }}
-      ENOS_VAR_vault_bundle_path: ./support/downloads/${{ inputs.build-artifact-name }}
       ENOS_VAR_vault_license_path: ./support/vault.hclic
       ENOS_DEBUG_DATA_ROOT_DIR: /tmp/enos-debug-data
     steps:

enos/Makefile

@@ -22,3 +22,10 @@ check-fmt-modules:
 .PHONY: fmt-modules
 fmt-modules:
 	terraform fmt -diff -recursive ./modules
+
+.PHONY: validate-enos
+validate-enos:
+	enos scenario validate
+
+.PHONY: lint
+lint: check-fmt validate-enos

enos/enos-modules.hcl

@@ -101,6 +101,13 @@ module "vault_agent" {
   vault_instance_count = var.vault_instance_count
 }
 
+module "vault_proxy" {
+  source = "./modules/vault_proxy"
+
+  vault_install_dir    = var.vault_install_dir
+  vault_instance_count = var.vault_instance_count
+}
+
 module "vault_verify_agent_output" {
   source = "./modules/vault_verify_agent_output"
 

enos/enos-scenario-agent.hcl

@@ -22,7 +22,7 @@ scenario "agent" {
       "ent.hsm"          = ["ui", "enterprise", "cgo", "hsm", "venthsm"]
       "ent.hsm.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.hsm.fips1402"]
     }
-    bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_bundle_path) : null
+    bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_artifact_path) : null
     distro_version = {
       "rhel"   = var.rhel_distro_version
       "ubuntu" = var.ubuntu_distro_version
@@ -121,14 +121,14 @@ scenario "agent" {
       artifactory_release      = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
       awskms_unseal_key_arn    = step.create_vpc.kms_key_arn
       cluster_name             = step.create_vault_cluster_targets.cluster_name
+      enable_file_audit_device = var.vault_enable_file_audit_device
       install_dir              = var.vault_install_dir
       license                  = matrix.edition != "oss" ? step.read_license.license : null
       local_artifact_path      = local.bundle_path
       packages                 = local.packages
       storage_backend          = "raft"
       target_hosts             = step.create_vault_cluster_targets.hosts
       unseal_method            = "shamir"
-      enable_file_audit_device = var.vault_enable_file_audit_device
     }
   }
 

enos/enos-scenario-autopilot.hcl

@@ -35,7 +35,7 @@ scenario "autopilot" {
       "ent.hsm"          = ["ui", "enterprise", "cgo", "hsm", "venthsm"]
       "ent.hsm.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.hsm.fips1402"]
     }
-    bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_bundle_path) : null
+    bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_artifact_path) : null
     distro_version = {
       "rhel"   = var.rhel_distro_version
       "ubuntu" = var.ubuntu_distro_version

enos/enos-scenario-proxy.hcl

@@ -0,0 +1,210 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+scenario "proxy" {
+  matrix {
+    arch            = ["amd64", "arm64"]
+    artifact_source = ["local", "crt", "artifactory"]
+    distro          = ["ubuntu", "rhel"]
+    edition         = ["oss", "ent", "ent.fips1402", "ent.hsm", "ent.hsm.fips1402"]
+  }
+
+  terraform_cli = terraform_cli.default
+  terraform     = terraform.default
+  providers = [
+    provider.aws.default,
+    provider.enos.ubuntu,
+    provider.enos.rhel
+  ]
+
+  locals {
+    backend_tag_key = "VaultStorage"
+    build_tags = {
+      "oss"              = ["ui"]
+      "ent"              = ["ui", "enterprise", "ent"]
+      "ent.fips1402"     = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.fips1402"]
+      "ent.hsm"          = ["ui", "enterprise", "cgo", "hsm", "venthsm"]
+      "ent.hsm.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.hsm.fips1402"]
+    }
+    bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_artifact_path) : null
+    distro_version = {
+      "rhel"   = var.rhel_distro_version
+      "ubuntu" = var.ubuntu_distro_version
+    }
+    enos_provider = {
+      rhel   = provider.enos.rhel
+      ubuntu = provider.enos.ubuntu
+    }
+    install_artifactory_artifact = local.bundle_path == null
+    packages                     = ["jq"]
+    tags = merge({
+      "Project Name" : var.project_name
+      "Project" : "Enos",
+      "Environment" : "ci"
+    }, var.tags)
+    vault_license_path = abspath(var.vault_license_path != null ? var.vault_license_path : joinpath(path.root, "./support/vault.hclic"))
+    vault_tag_key      = "Type" // enos_vault_start expects Type as the tag key
+  }
+
+  step "get_local_metadata" {
+    skip_step = matrix.artifact_source != "local"
+    module    = module.get_local_metadata
+  }
+
+  step "build_vault" {
+    module = "build_${matrix.artifact_source}"
+
+    variables {
+      build_tags           = var.vault_local_build_tags != null ? var.vault_local_build_tags : local.build_tags[matrix.edition]
+      bundle_path          = local.bundle_path
+      goarch               = matrix.arch
+      goos                 = "linux"
+      artifactory_host     = matrix.artifact_source == "artifactory" ? var.artifactory_host : null
+      artifactory_repo     = matrix.artifact_source == "artifactory" ? var.artifactory_repo : null
+      artifactory_username = matrix.artifact_source == "artifactory" ? var.artifactory_username : null
+      artifactory_token    = matrix.artifact_source == "artifactory" ? var.artifactory_token : null
+      arch                 = matrix.artifact_source == "artifactory" ? matrix.arch : null
+      product_version      = var.vault_product_version
+      artifact_type        = matrix.artifact_source == "artifactory" ? var.vault_artifact_type : null
+      distro               = matrix.artifact_source == "artifactory" ? matrix.distro : null
+      edition              = matrix.artifact_source == "artifactory" ? matrix.edition : null
+      revision             = var.vault_revision
+    }
+  }
+
+  step "ec2_info" {
+    module = module.ec2_info
+  }
+
+  step "create_vpc" {
+    module = module.create_vpc
+
+    variables {
+      common_tags = local.tags
+    }
+  }
+
+  step "read_license" {
+    skip_step = matrix.edition == "oss"
+    module    = module.read_license
+
+    variables {
+      file_name = local.vault_license_path
+    }
+  }
+
+  step "create_vault_cluster_targets" {
+    module     = module.target_ec2_instances
+    depends_on = [step.create_vpc]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      ami_id                = step.ec2_info.ami_ids[matrix.arch][matrix.distro][local.distro_version[matrix.distro]]
+      awskms_unseal_key_arn = step.create_vpc.kms_key_arn
+      cluster_tag_key       = local.vault_tag_key
+      common_tags           = local.tags
+      vpc_id                = step.create_vpc.vpc_id
+    }
+  }
+
+  step "create_vault_cluster" {
+    module = module.vault_cluster
+    depends_on = [
+      step.build_vault,
+      step.create_vault_cluster_targets
+    ]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      artifactory_release      = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
+      awskms_unseal_key_arn    = step.create_vpc.kms_key_arn
+      cluster_name             = step.create_vault_cluster_targets.cluster_name
+      enable_file_audit_device = var.vault_enable_file_audit_device
+      install_dir              = var.vault_install_dir
+      license                  = matrix.edition != "oss" ? step.read_license.license : null
+      local_artifact_path      = local.bundle_path
+      packages                 = local.packages
+      storage_backend          = "raft"
+      target_hosts             = step.create_vault_cluster_targets.hosts
+      unseal_method            = "shamir"
+    }
+  }
+
+  step "start_vault_proxy" {
+    module = "vault_proxy"
+    depends_on = [
+      step.build_vault,
+      step.create_vault_cluster,
+    ]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      vault_instances  = step.create_vault_cluster_targets.hosts
+      vault_root_token = step.create_vault_cluster.root_token
+    }
+  }
+
+  output "awkms_unseal_key_arn" {
+    description = "The Vault cluster KMS key arn"
+    value       = step.create_vpc.kms_key_arn
+  }
+
+  output "cluster_name" {
+    description = "The Vault cluster name"
+    value       = step.create_vault_cluster.cluster_name
+  }
+
+  output "hosts" {
+    description = "The Vault cluster target hosts"
+    value       = step.create_vault_cluster.target_hosts
+  }
+
+  output "private_ips" {
+    description = "The Vault cluster private IPs"
+    value       = step.create_vault_cluster.private_ips
+  }
+
+  output "public_ips" {
+    description = "The Vault cluster public IPs"
+    value       = step.create_vault_cluster.public_ips
+  }
+
+  output "root_token" {
+    description = "The Vault cluster root token"
+    value       = step.create_vault_cluster.root_token
+  }
+
+  output "recovery_key_shares" {
+    description = "The Vault cluster recovery key shares"
+    value       = step.create_vault_cluster.recovery_key_shares
+  }
+
+  output "recovery_keys_b64" {
+    description = "The Vault cluster recovery keys b64"
+    value       = step.create_vault_cluster.recovery_keys_b64
+  }
+
+  output "recovery_keys_hex" {
+    description = "The Vault cluster recovery keys hex"
+    value       = step.create_vault_cluster.recovery_keys_hex
+  }
+
+  output "unseal_keys_b64" {
+    description = "The Vault cluster unseal keys"
+    value       = step.create_vault_cluster.unseal_keys_b64
+  }
+
+  output "unseal_keys_hex" {
+    description = "The Vault cluster unseal keys hex"
+    value       = step.create_vault_cluster.unseal_keys_hex
+  }
+}

enos/enos-scenario-replication.hcl

@@ -47,7 +47,7 @@ scenario "replication" {
       "rhel"   = var.rhel_distro_version
       "ubuntu" = var.ubuntu_distro_version
     }
-    bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_bundle_path) : null
+    bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_artifact_path) : null
     enos_provider = {
       rhel   = provider.enos.rhel
       ubuntu = provider.enos.ubuntu

enos/enos-scenario-smoke.hcl

@@ -39,7 +39,7 @@ scenario "smoke" {
       "ent.hsm"          = ["ui", "enterprise", "cgo", "hsm", "venthsm"]
       "ent.hsm.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.hsm.fips1402"]
     }
-    bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_bundle_path) : null
+    bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_artifact_path) : null
     distro_version = {
       "rhel"   = var.rhel_distro_version
       "ubuntu" = var.ubuntu_distro_version

enos/enos-scenario-ui.hcl

@@ -19,7 +19,7 @@ scenario "ui" {
       "oss" = ["ui"]
       "ent" = ["ui", "enterprise", "ent"]
     }
-    bundle_path    = abspath(var.vault_bundle_path)
+    bundle_path    = abspath(var.vault_artifact_path)
     distro         = "ubuntu"
     consul_version = "1.14.2"
     seal           = "awskms"

enos/enos-scenario-upgrade.hcl

@@ -33,7 +33,7 @@ scenario "upgrade" {
       "ent.hsm"          = ["ui", "enterprise", "cgo", "hsm", "venthsm"]
       "ent.hsm.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.hsm.fips1402"]
     }
-    bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_bundle_path) : null
+    bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_artifact_path) : null
     distro_version = {
       "rhel"   = var.rhel_distro_version
       "ubuntu" = var.ubuntu_distro_version