diff --git a/Makefile b/Makefile index 8f85a38aa..1d7b08049 100644 --- a/Makefile +++ b/Makefile @@ -52,6 +52,7 @@ SKIP_CLEANUP ?= SKIP_AWS_TESTS ?= true SKIP_AWS_STATIC_CREDS_TEST ?= true SKIP_GCP_TESTS ?= true +SKIP_HCPVSAPPS_TESTS ?= false # filter bats unit tests to run. BATS_TESTS_FILTER ?= .\* diff --git a/scale-testing.mk b/scale-testing.mk index 140ef3928..6b89b9a72 100644 --- a/scale-testing.mk +++ b/scale-testing.mk @@ -5,6 +5,11 @@ AWS_REGION ?= us-east-2 EKS_K8S_VERSION ?= 1.30 +# testing dev instances is currently not supported +# TODO: create the docker registry (e.g. ECR) to enable dev builds +VERSION ?= 0.8.1 +INTEGRATION_TESTS_PARALLEL ?= true + # directories for cloud hosted k8s infrastructure for running tests # root directory for all integration tests TF_EKS_SRC_DIR ?= $(INTEGRATION_TEST_ROOT)/infra/scale-testing/eks-cluster @@ -12,21 +17,12 @@ TF_EKS_STATE_DIR ?= $(TF_EKS_SRC_DIR)/state TF_DEPLOY_SRC_DIR ?= $(INTEGRATION_TEST_ROOT)/infra/scale-testing/deployments TF_DEPLOY_STATE_DIR ?= $(TF_DEPLOY_SRC_DIR)/state -include ./Makefile +SCALE_TESTS ?= 1 -.PHONY: create-eks -create-eks: ## Create a new EKS cluster - @mkdir -p $(TF_EKS_STATE_DIR) - rm -f $(TF_EKS_STATE_DIR)/*.tf - cp -v $(TF_EKS_SRC_DIR)/*.tf $(TF_EKS_STATE_DIR)/. - $(TERRAFORM) -chdir=$(TF_EKS_STATE_DIR) init -upgrade - $(TERRAFORM) -chdir=$(TF_EKS_STATE_DIR) apply -auto-approve \ - -var region=$(AWS_REGION) \ - -var kubernetes_version=$(EKS_K8S_VERSION) || exit 1 - rm -f $(TF_EKS_STATE_DIR)/*.tfvars +include ./aws.mk .PHONY: deploy-workload -deploy-workload: set-vault-license ## Deploy the workload to the EKS cluster +deploy-workload: set-vault-license import-aws-vars ## Deploy the workload to the EKS cluster @mkdir -p $(TF_DEPLOY_STATE_DIR) ifeq ($(VAULT_ENTERPRISE), true) ## ensure that the license is *not* emitted to the console @@ -35,11 +31,33 @@ endif rm -f $(TF_DEPLOY_STATE_DIR)/*.tf cp -v $(TF_DEPLOY_SRC_DIR)/*.tf $(TF_DEPLOY_STATE_DIR)/. $(TERRAFORM) -chdir=$(TF_DEPLOY_STATE_DIR) init -upgrade - $(TERRAFORM) -chdir=$(TF_DEPLOY_STATE_DIR) apply -auto-approve || exit 1 + $(TERRAFORM) -chdir=$(TF_DEPLOY_STATE_DIR) apply -auto-approve \ + -var cluster_name=$(EKS_CLUSTER_NAME) || exit 1 rm -f $(TF_DEPLOY_STATE_DIR)/*.tfvars -.PHONY: destroy-eks -destroy-eks: ## Destroy the EKS cluster - $(TERRAFORM) -chdir=$(TF_EKS_STATE_DIR) destroy -auto-approve \ - -var region=$(AWS_REGION) \ - -var kubernetes_version=$(EKS_K8S_VERSION) || exit 1 +.PHONY: update-kubeconfig +update-kubeconfig: import-aws-vars + aws eks --region $(AWS_REGION) update-kubeconfig --name $(EKS_CLUSTER_NAME) + +.PHONY: cleanup-port-forward +cleanup-port-forward: ## Kill orphan port-forward processes + @echo "Cleaning up orphan port-forward processes..." + @pgrep -f 'kubectl port-forward -n $(K8S_VAULT_NAMESPACE) statefulset/vault' | xargs -r kill -9 && \ + echo "Port-forward processes terminated successfully." || \ + echo "No port-forward processes found or an error occurred." + +.PHONY: set image scale-tests +scale-tests: cleanup-port-forward set-image update-kubeconfig import-aws-vars + $(MAKE) port-forward & + SCALE_TESTS=true VAULT_ENTERPRISE=true ENT_TESTS=$(VAULT_ENTERPRISE) \ + SUPPRESS_TF_OUTPUT=$(SUPPRESS_TF_OUTPUT) SKIP_CLEANUP=$(SKIP_CLEANUP) \ + OPERATOR_IMAGE_REPO=$(IMAGE_TAG_BASE) OPERATOR_IMAGE_TAG=$(VERSION) \ + OPERATOR_NAMESPACE=$(OPERATOR_NAMESPACE) \ + VAULT_OIDC_DISC_URL=$(EKS_OIDC_URL) VAULT_OIDC_CA=false \ + INTEGRATION_TESTS=true EKS_CLUSTER_NAME=$(EKS_CLUSTER_NAME) \ + K8S_CLUSTER_CONTEXT=$(K8S_CLUSTER_CONTEXT) CGO_ENABLED=0 \ + K8S_VAULT_NAMESPACE=$(K8S_VAULT_NAMESPACE) \ + SKIP_AWS_TESTS=$(SKIP_AWS_TESTS) SKIP_AWS_STATIC_CREDS_TEST=$(SKIP_AWS_STATIC_CREDS_TEST) \ + SKIP_GCP_TESTS=$(SKIP_GCP_TESTS) SKIP_HCPVSAPPS_TESTS=$(SKIP_HCPVSAPPS_TESTS) \ + PARALLEL_INT_TESTS=$(INTEGRATION_TESTS_PARALLEL) \ + go test github.com/hashicorp/vault-secrets-operator/test/integration/... $(TESTARGS) -timeout=30m diff --git a/test/integration/hcpvaultsecretsapp_integration_test.go b/test/integration/hcpvaultsecretsapp_integration_test.go index 19680779b..f076fd429 100644 --- a/test/integration/hcpvaultsecretsapp_integration_test.go +++ b/test/integration/hcpvaultsecretsapp_integration_test.go @@ -49,7 +49,7 @@ func TestHCPVaultSecretsApp(t *testing.T) { } testID := "hvs" - clusterName := os.Getenv("KIND_CLUSTER_NAME") + clusterName := kindClusterName assert.NotEmpty(t, clusterName, "KIND_CLUSTER_NAME is not set") operatorNS := os.Getenv("OPERATOR_NAMESPACE") diff --git a/test/integration/infra/scale-testing/deployments/main.tf b/test/integration/infra/scale-testing/deployments/main.tf index 541742446..06adb6af3 100644 --- a/test/integration/infra/scale-testing/deployments/main.tf +++ b/test/integration/infra/scale-testing/deployments/main.tf @@ -1,22 +1,6 @@ # Copyright (c) HashiCorp, Inc. # SPDX-License-Identifier: BUSL-1.1 -module "vso-helm" { - source = "../../../../modules/vso-helm" - operator_namespace = var.operator_namespace - operator_image_repo = var.operator_image_repo - operator_image_tag = var.operator_image_tag - enable_default_connection = false - enable_default_auth_method = false - operator_helm_chart_path = var.operator_helm_chart_path - k8s_vault_connection_address = var.k8s_vault_connection_address - - manager_extra_args = [ - "-min-refresh-after-hvsa=3s", - "-zap-log-level=6" - ] -} - module "vault" { source = "../../../../modules/vault" vault_license_path = var.vault_license_path diff --git a/test/integration/infra/scale-testing/deployments/providers.tf b/test/integration/infra/scale-testing/deployments/providers.tf index b1ee971c8..9ca965411 100644 --- a/test/integration/infra/scale-testing/deployments/providers.tf +++ b/test/integration/infra/scale-testing/deployments/providers.tf @@ -1,34 +1,46 @@ terraform { required_providers { - kubernetes = { - source = "hashicorp/kubernetes" - version = "2.16.1" - } helm = { source = "hashicorp/helm" version = "2.13.1" } + kubernetes = { + source = "hashicorp/kubernetes" + version = "2.30.0" + } + aws = { + source = "hashicorp/aws" + version = "5.49.0" + } } } -data "terraform_remote_state" "eks" { - backend = "local" +provider "aws" { + region = var.region +} - config = { - path = "../../eks-cluster/state/terraform.tfstate" - } +data "aws_eks_cluster" "cluster" { + name = var.cluster_name } provider "kubernetes" { - host = data.terraform_remote_state.eks.outputs.cluster_endpoint - cluster_ca_certificate = base64decode(data.terraform_remote_state.eks.outputs.cluster_certificate_authority) - token = data.terraform_remote_state.eks.outputs.eks_cluster_token + host = data.aws_eks_cluster.cluster.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data) + exec { + api_version = "client.authentication.k8s.io/v1beta1" + args = ["eks", "get-token", "--cluster-name", data.aws_eks_cluster.cluster.name] + command = "aws" + } } provider "helm" { kubernetes { - host = data.terraform_remote_state.eks.outputs.cluster_endpoint - cluster_ca_certificate = base64decode(data.terraform_remote_state.eks.outputs.cluster_certificate_authority) - token = data.terraform_remote_state.eks.outputs.eks_cluster_token + host = data.aws_eks_cluster.cluster.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data) + exec { + api_version = "client.authentication.k8s.io/v1beta1" + args = ["eks", "get-token", "--cluster-name", data.aws_eks_cluster.cluster.name] + command = "aws" + } } } diff --git a/test/integration/infra/scale-testing/deployments/variables.tf b/test/integration/infra/scale-testing/deployments/variables.tf index ef4a2c926..11ed1bd60 100644 --- a/test/integration/infra/scale-testing/deployments/variables.tf +++ b/test/integration/infra/scale-testing/deployments/variables.tf @@ -1,116 +1,9 @@ # Copyright (c) HashiCorp, Inc. # SPDX-License-Identifier: BUSL-1.1 -variable "operator_namespace" { - default = "vault-secrets-operator-system" -} - -# The path to the local helm chart in our repository, this is used by helm to find the Chart.yaml -variable "operator_helm_chart_path" { - default = "../../../../../../chart" -} - -variable "enable_default_connection" { - type = bool - default = true -} - -variable "enable_default_auth_method" { - type = bool - default = true -} - -variable "k8s_vault_connection_address" { - default = "" -} - -variable "k8s_auth_default_mount" { - default = "" -} - -variable "vault_test_namespace" { - default = "" -} - -variable "operator_allowednamespaces" { - type = list(string) - default = [] -} - -variable "k8s_auth_default_role" { - default = "" -} - -variable "k8s_auth_default_token_audiences" { - type = list(string) - default = [] -} - -variable "operator_image_repo" { - default = "hashicorp/vault-secrets-operator" -} - -variable "operator_image_tag" { - default = "0.8.1" -} - -variable "cpu_limits" { - default = "" -} - -variable "memory_limits" { - default = "" -} - -variable "cpu_requests" { - default = "" -} - -variable "memory_requests" { - default = "" -} - -variable "client_cache_config" { - type = object({ - persistence_model = string - revoke_client_cache_on_uninstall = bool - storage_encryption = object({ - enabled = bool - vault_connection_ref = string - namespace = string - mount = string - transit_mount = string - key_name = string - method = string - kubernetes_auth_role = string - kubernetes_auth_service_account = string - kubernetes_auth_token_audiences = string - }) - }) - - default = { - persistence_model = "" - revoke_client_cache_on_uninstall = false - storage_encryption = { - enabled = false - vault_connection_ref = "" - namespace = "" - mount = "" - transit_mount = "" - key_name = "" - method = "" - kubernetes_auth_role = "" - kubernetes_auth_service_account = "" - kubernetes_auth_token_audiences = "" - } - } -} - -variable "manager_extra_args" { - type = list(string) - default = [ - "-zap-log-level=5" - ] +variable "cluster_name" { + description = "Name of the EKS cluster" + type = string } variable "vault_license_path" { @@ -137,18 +30,10 @@ variable "k8s_config_path" { default = "~/.kube/config" } -variable "vault_image_repo" { - default = "docker.mirror.hashicorp.services/hashicorp/vault" -} - variable "vault_image_repo_ent" { default = "docker.mirror.hashicorp.services/hashicorp/vault-enterprise" } -variable "vault_image_tag" { - default = "1.17" -} - variable "vault_image_tag_ent" { default = "1.17-ent" } @@ -162,13 +47,11 @@ variable "vault_chart_version" { default = "0.28.1" } -variable "install_kube_prometheus" { - type = bool - default = false +variable "region" { + description = "AWS region" + type = string + default = "us-east-2" } -variable "metrics_server_enabled" { - type = bool - default = true -} + diff --git a/test/integration/infra/scale-testing/eks-cluster/main.tf b/test/integration/infra/scale-testing/eks-cluster/main.tf index ac77cdfce..31bdb1f90 100644 --- a/test/integration/infra/scale-testing/eks-cluster/main.tf +++ b/test/integration/infra/scale-testing/eks-cluster/main.tf @@ -105,4 +105,14 @@ data "aws_eks_cluster" "cluster" { data "aws_eks_cluster_auth" "cluster" { name = module.eks.cluster_name depends_on = [module.eks.cluster_endpoint] -} \ No newline at end of file +} + +resource "local_file" "env_file" { + filename = "${path.module}/outputs.env" + content = <= 1.16.3, and since that // changes the app policy required we need to set a flag in the test // terraform @@ -292,23 +313,23 @@ func TestVaultStaticSecret(t *testing.T) { }, { name: "create-kv-v1", - create: 2, + create: kvv1Count, createTypes: []string{consts.KVSecretTypeV1}, }, { name: "create-kv-v2", - create: 1, + create: kvv2Count, createTypes: []string{consts.KVSecretTypeV2}, }, { name: "create-kv-v2-fixed-version", - create: 2, + create: kvv2FixedCount, createTypes: []string{consts.KVSecretTypeV2}, version: 1, }, { name: "create-both", - create: 2, + create: bothCount, createTypes: []string{consts.KVSecretTypeV1, consts.KVSecretTypeV2}, }, { @@ -324,7 +345,7 @@ func TestVaultStaticSecret(t *testing.T) { }, }, existing: getExisting(), - create: 2, + create: mixedBothCount, createTypes: []string{consts.KVSecretTypeV1, consts.KVSecretTypeV2}, }, { @@ -349,7 +370,7 @@ func TestVaultStaticSecret(t *testing.T) { } return vss }(), - create: 2, + create: eventsBothCount, createTypes: []string{consts.KVSecretTypeV1, consts.KVSecretTypeV2}, useEvents: true, },