Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add scale tests #916

Open
wants to merge 34 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 31 commits
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
b69f449
scale testing
Zlaticanin Aug 28, 2024
d8faecf
add OPERATOR_* env vars to output file terraform scale-testing
thyton Aug 28, 2024
89cc691
update makefile
Zlaticanin Sep 4, 2024
04ed9ab
update
Zlaticanin Sep 9, 2024
c2ef8ee
wip: dropped remoted state and tests passing
Zlaticanin Sep 10, 2024
f149d1b
add data sources to be able to get cluster outputs
Zlaticanin Sep 11, 2024
6585828
revert change and remove unnecessary code
Zlaticanin Sep 12, 2024
0c93f97
add modules/operator-common
thyton Sep 12, 2024
bc80ddf
add a make target to cleanup orphan port forward processes + refactor…
Zlaticanin Sep 12, 2024
7d090d7
update cleanup-port-forward and add a flag to skip HCPVSAPPS tests
Zlaticanin Sep 16, 2024
ff7e5f6
Add support to specify the number of CRs by type: VDS + VSS count
Zlaticanin Sep 18, 2024
0dd55cb
Extend vps to be able to specify count via env var
Zlaticanin Sep 18, 2024
8a989db
update cr names
Zlaticanin Sep 18, 2024
da4e8ab
delete operator-scale and use aliases for providers instead
Zlaticanin Sep 20, 2024
de3903c
make fmttf
Zlaticanin Sep 20, 2024
aee9feb
add default for cluster name
Zlaticanin Sep 20, 2024
f4f5cc9
add aws provider
Zlaticanin Sep 20, 2024
a55a5d7
Merge branch 'main' into VAULT-28156/add-scale-tests
Zlaticanin Sep 20, 2024
babc095
Add aws region
Zlaticanin Sep 20, 2024
37db0f6
add aws region
Zlaticanin Sep 23, 2024
1274adb
update aws region var
Zlaticanin Sep 23, 2024
9765c87
factor out providers files and use correct in testmain based on test …
Zlaticanin Sep 23, 2024
ae75817
make fmttf
Zlaticanin Sep 23, 2024
ab5fe95
move providers files into their own dir + add aws provider in our vau…
Zlaticanin Sep 24, 2024
15a5983
fix the providers
Zlaticanin Sep 24, 2024
4886e92
rename providers
Zlaticanin Sep 24, 2024
33dfbf5
update
Zlaticanin Sep 24, 2024
9caf197
update destination
Zlaticanin Sep 25, 2024
b15e2f9
update providers
Zlaticanin Sep 25, 2024
bf6d152
update with suggestions
Zlaticanin Oct 2, 2024
3d62123
Merge branch 'main' into VAULT-28156/add-scale-tests
Zlaticanin Oct 2, 2024
d4c1074
Update counts so we set default if count is <= 0
Zlaticanin Oct 7, 2024
a866860
update default counts + add conditional for postgres storage persistence
Zlaticanin Oct 7, 2024
87eb00b
update with suggestions
Zlaticanin Oct 10, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 40 additions & 2 deletions scale-testing.mk
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,19 @@
AWS_REGION ?= us-east-2
EKS_K8S_VERSION ?= 1.30

VERSION ?= 0.8.1
Zlaticanin marked this conversation as resolved.
Show resolved Hide resolved
INTEGRATION_TESTS_PARALLEL ?= true
Zlaticanin marked this conversation as resolved.
Show resolved Hide resolved
SKIP_HCPVSAPPS_TESTS ?= true
Zlaticanin marked this conversation as resolved.
Show resolved Hide resolved

# directories for cloud hosted k8s infrastructure for running tests
# root directory for all integration tests
TF_EKS_SRC_DIR ?= $(INTEGRATION_TEST_ROOT)/infra/scale-testing/eks-cluster
TF_EKS_STATE_DIR ?= $(TF_EKS_SRC_DIR)/state
TF_DEPLOY_SRC_DIR ?= $(INTEGRATION_TEST_ROOT)/infra/scale-testing/deployments
TF_DEPLOY_STATE_DIR ?= $(TF_DEPLOY_SRC_DIR)/state

SCALE_TESTS=1
Zlaticanin marked this conversation as resolved.
Show resolved Hide resolved

include ./Makefile
Zlaticanin marked this conversation as resolved.
Show resolved Hide resolved

.PHONY: create-eks
Expand All @@ -26,7 +32,7 @@ create-eks: ## Create a new EKS cluster
rm -f $(TF_EKS_STATE_DIR)/*.tfvars

.PHONY: deploy-workload
deploy-workload: set-vault-license ## Deploy the workload to the EKS cluster
deploy-workload: set-vault-license import-aws-vars ## Deploy the workload to the EKS cluster
@mkdir -p $(TF_DEPLOY_STATE_DIR)
ifeq ($(VAULT_ENTERPRISE), true)
## ensure that the license is *not* emitted to the console
Expand All @@ -35,9 +41,41 @@ endif
rm -f $(TF_DEPLOY_STATE_DIR)/*.tf
cp -v $(TF_DEPLOY_SRC_DIR)/*.tf $(TF_DEPLOY_STATE_DIR)/.
$(TERRAFORM) -chdir=$(TF_DEPLOY_STATE_DIR) init -upgrade
$(TERRAFORM) -chdir=$(TF_DEPLOY_STATE_DIR) apply -auto-approve || exit 1
$(TERRAFORM) -chdir=$(TF_DEPLOY_STATE_DIR) apply -auto-approve \
-var cluster_name=$(EKS_CLUSTER_NAME) || exit 1
rm -f $(TF_DEPLOY_STATE_DIR)/*.tfvars

.PHONY: import-aws-vars
import-aws-vars:
-include $(TF_EKS_STATE_DIR)/outputs.env

.PHONY: connect-cluster
connect-cluster: import-aws-vars ## Connect to the EKS cluster
Zlaticanin marked this conversation as resolved.
Show resolved Hide resolved
aws eks --region $(AWS_REGION) update-kubeconfig --name $(EKS_CLUSTER_NAME)

.PHONY: cleanup-port-forward
cleanup-port-forward: ## Kill orphan port-forward processes
@echo "Cleaning up orphan port-forward processes..."
@pgrep -f 'kubectl port-forward -n $(K8S_VAULT_NAMESPACE) statefulset/vault' | xargs -r kill -9 && \
echo "Port-forward processes terminated successfully." || \
echo "No port-forward processes found or an error occurred."

.PHONY: set image scale-tests
scale-tests: cleanup-port-forward set-image connect-cluster import-aws-vars
$(MAKE) port-forward &
SCALE_TESTS=true VAULT_ENTERPRISE=true ENT_TESTS=$(VAULT_ENTERPRISE) \
SUPPRESS_TF_OUTPUT=$(SUPPRESS_TF_OUTPUT) SKIP_CLEANUP=$(SKIP_CLEANUP) \
OPERATOR_IMAGE_REPO=$(IMAGE_TAG_BASE) OPERATOR_IMAGE_TAG=$(VERSION) \
OPERATOR_NAMESPACE=$(OPERATOR_NAMESPACE) \
VAULT_OIDC_DISC_URL=$(EKS_OIDC_URL) VAULT_OIDC_CA=false \
INTEGRATION_TESTS=true EKS_CLUSTER_NAME=$(EKS_CLUSTER_NAME) \
K8S_CLUSTER_CONTEXT=$(K8S_CLUSTER_CONTEXT) CGO_ENABLED=0 \
K8S_VAULT_NAMESPACE=$(K8S_VAULT_NAMESPACE) \
SKIP_AWS_TESTS=$(SKIP_AWS_TESTS) SKIP_AWS_STATIC_CREDS_TEST=$(SKIP_AWS_STATIC_CREDS_TEST) \
SKIP_GCP_TESTS=$(SKIP_GCP_TESTS) SKIP_HCPVSAPPS_TESTS=$(SKIP_HCPVSAPPS_TESTS) \
PARALLEL_INT_TESTS=$(INTEGRATION_TESTS_PARALLEL) \
go test github.com/hashicorp/vault-secrets-operator/test/integration/... $(TESTARGS) -timeout=30m

.PHONY: destroy-eks
destroy-eks: ## Destroy the EKS cluster
$(TERRAFORM) -chdir=$(TF_EKS_STATE_DIR) destroy -auto-approve \
Expand Down
16 changes: 0 additions & 16 deletions test/integration/infra/scale-testing/deployments/main.tf
Original file line number Diff line number Diff line change
@@ -1,22 +1,6 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: BUSL-1.1

module "vso-helm" {
source = "../../../../modules/vso-helm"
operator_namespace = var.operator_namespace
operator_image_repo = var.operator_image_repo
operator_image_tag = var.operator_image_tag
enable_default_connection = false
enable_default_auth_method = false
operator_helm_chart_path = var.operator_helm_chart_path
k8s_vault_connection_address = var.k8s_vault_connection_address

manager_extra_args = [
"-min-refresh-after-hvsa=3s",
"-zap-log-level=6"
]
}

module "vault" {
source = "../../../../modules/vault"
vault_license_path = var.vault_license_path
Expand Down
42 changes: 27 additions & 15 deletions test/integration/infra/scale-testing/deployments/providers.tf
Original file line number Diff line number Diff line change
@@ -1,34 +1,46 @@
terraform {
required_providers {
kubernetes = {
source = "hashicorp/kubernetes"
version = "2.16.1"
}
helm = {
source = "hashicorp/helm"
version = "2.13.1"
}
kubernetes = {
source = "hashicorp/kubernetes"
version = "2.30.0"
}
aws = {
source = "hashicorp/aws"
version = "5.49.0"
}
}
}

data "terraform_remote_state" "eks" {
backend = "local"
provider "aws" {
region = var.region
}

config = {
path = "../../eks-cluster/state/terraform.tfstate"
}
data "aws_eks_cluster" "cluster" {
name = var.cluster_name
}

provider "kubernetes" {
host = data.terraform_remote_state.eks.outputs.cluster_endpoint
cluster_ca_certificate = base64decode(data.terraform_remote_state.eks.outputs.cluster_certificate_authority)
token = data.terraform_remote_state.eks.outputs.eks_cluster_token
host = data.aws_eks_cluster.cluster.endpoint
cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data)
exec {
api_version = "client.authentication.k8s.io/v1beta1"
args = ["eks", "get-token", "--cluster-name", data.aws_eks_cluster.cluster.name]
command = "aws"
}
}

provider "helm" {
kubernetes {
host = data.terraform_remote_state.eks.outputs.cluster_endpoint
cluster_ca_certificate = base64decode(data.terraform_remote_state.eks.outputs.cluster_certificate_authority)
token = data.terraform_remote_state.eks.outputs.eks_cluster_token
host = data.aws_eks_cluster.cluster.endpoint
cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data)
exec {
api_version = "client.authentication.k8s.io/v1beta1"
args = ["eks", "get-token", "--cluster-name", data.aws_eks_cluster.cluster.name]
command = "aws"
}
}
}
133 changes: 8 additions & 125 deletions test/integration/infra/scale-testing/deployments/variables.tf
Zlaticanin marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
@@ -1,116 +1,9 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: BUSL-1.1

variable "operator_namespace" {
default = "vault-secrets-operator-system"
}

# The path to the local helm chart in our repository, this is used by helm to find the Chart.yaml
variable "operator_helm_chart_path" {
default = "../../../../../../chart"
}

variable "enable_default_connection" {
type = bool
default = true
}

variable "enable_default_auth_method" {
type = bool
default = true
}

variable "k8s_vault_connection_address" {
default = ""
}

variable "k8s_auth_default_mount" {
default = ""
}

variable "vault_test_namespace" {
default = ""
}

variable "operator_allowednamespaces" {
type = list(string)
default = []
}

variable "k8s_auth_default_role" {
default = ""
}

variable "k8s_auth_default_token_audiences" {
type = list(string)
default = []
}

variable "operator_image_repo" {
default = "hashicorp/vault-secrets-operator"
}

variable "operator_image_tag" {
default = "0.8.1"
}

variable "cpu_limits" {
default = ""
}

variable "memory_limits" {
default = ""
}

variable "cpu_requests" {
default = ""
}

variable "memory_requests" {
default = ""
}

variable "client_cache_config" {
type = object({
persistence_model = string
revoke_client_cache_on_uninstall = bool
storage_encryption = object({
enabled = bool
vault_connection_ref = string
namespace = string
mount = string
transit_mount = string
key_name = string
method = string
kubernetes_auth_role = string
kubernetes_auth_service_account = string
kubernetes_auth_token_audiences = string
})
})

default = {
persistence_model = ""
revoke_client_cache_on_uninstall = false
storage_encryption = {
enabled = false
vault_connection_ref = ""
namespace = ""
mount = ""
transit_mount = ""
key_name = ""
method = ""
kubernetes_auth_role = ""
kubernetes_auth_service_account = ""
kubernetes_auth_token_audiences = ""
}
}
}

variable "manager_extra_args" {
type = list(string)
default = [
"-zap-log-level=5"
]
variable "cluster_name" {
description = "Name of the EKS cluster"
type = string
}

variable "vault_license_path" {
Expand All @@ -137,18 +30,10 @@ variable "k8s_config_path" {
default = "~/.kube/config"
}

variable "vault_image_repo" {
default = "docker.mirror.hashicorp.services/hashicorp/vault"
}

variable "vault_image_repo_ent" {
default = "docker.mirror.hashicorp.services/hashicorp/vault-enterprise"
}

variable "vault_image_tag" {
default = "1.17"
}

variable "vault_image_tag_ent" {
default = "1.17-ent"
}
Expand All @@ -162,13 +47,11 @@ variable "vault_chart_version" {
default = "0.28.1"
}

variable "install_kube_prometheus" {
type = bool
default = false
variable "region" {
description = "AWS region"
type = string
default = "us-east-2"
}

variable "metrics_server_enabled" {
type = bool
default = true
}


12 changes: 11 additions & 1 deletion test/integration/infra/scale-testing/eks-cluster/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -105,4 +105,14 @@ data "aws_eks_cluster" "cluster" {
data "aws_eks_cluster_auth" "cluster" {
name = module.eks.cluster_name
depends_on = [module.eks.cluster_endpoint]
}
}

resource "local_file" "env_file" {
filename = "${path.module}/outputs.env"
content = <<EOT
EKS_OIDC_URL=${module.eks.cluster_oidc_issuer_url}
EKS_CLUSTER_NAME=${module.eks.cluster_name}
AWS_REGION=${var.region}
K8S_CLUSTER_CONTEXT=${module.eks.cluster_arn}
EOT
}
Loading