From 0f2d065bdd54ae26889dc0b83ee9f8544f185d66 Mon Sep 17 00:00:00 2001 From: Nandaja Varma Date: Mon, 20 Jun 2022 08:49:14 +0000 Subject: [PATCH] Add tests for external dependencies in AKS --- .werft/build.ts | 2 +- install/infra/terraform/aks/kubernetes.tf | 4 +-- install/infra/terraform/aks/local.tf | 4 +-- .../tools/azure-external-dns/main.tf | 13 ++++----- .../terraform/tools/issuer/azure/main.tf | 1 + install/tests/Makefile | 27 ++++++++++++++----- install/tests/main.tf | 9 ++++--- .../tests/manifests/kots-config-azure-db.yaml | 16 +++++++++++ .../manifests/kots-config-azure-registry.yaml | 16 +++++++++++ .../manifests/kots-config-azure-storage.yaml | 16 +++++++++++ install/tests/output.tf | 18 +++++++++++++ 11 files changed, 104 insertions(+), 22 deletions(-) create mode 100644 install/tests/manifests/kots-config-azure-db.yaml create mode 100644 install/tests/manifests/kots-config-azure-registry.yaml create mode 100644 install/tests/manifests/kots-config-azure-storage.yaml create mode 100644 install/tests/output.tf diff --git a/.werft/build.ts b/.werft/build.ts index 6f4911b4da2540..96eac5fbfb99f7 100644 --- a/.werft/build.ts +++ b/.werft/build.ts @@ -63,7 +63,7 @@ async function run(context: any) { if (!config.withPreview || config.publishRelease) { werft.phase("deploy", "not deploying"); console.log("running without preview environment or publish-release is set"); - return + return; } try { diff --git a/install/infra/terraform/aks/kubernetes.tf b/install/infra/terraform/aks/kubernetes.tf index 16629d7c3bfc53..0d712b59736a79 100644 --- a/install/infra/terraform/aks/kubernetes.tf +++ b/install/infra/terraform/aks/kubernetes.tf @@ -40,7 +40,7 @@ resource "azurerm_kubernetes_cluster" "k8s" { zones = [] enable_auto_scaling = true - min_count = 2 + min_count = 1 max_count = 10 orchestrator_version = data.azurerm_kubernetes_service_versions.k8s.latest_version node_labels = local.nodes.0.labels @@ -72,7 +72,7 @@ resource "azurerm_kubernetes_cluster_node_pool" "pools" { vm_size = local.machine enable_auto_scaling = true - min_count = 2 + min_count = 1 max_count = 10 orchestrator_version = data.azurerm_kubernetes_service_versions.k8s.latest_version node_labels = local.nodes[count.index + 1].labels diff --git a/install/infra/terraform/aks/local.tf b/install/infra/terraform/aks/local.tf index e11d87159206cc..7c5a1e53023cd6 100644 --- a/install/infra/terraform/aks/local.tf +++ b/install/infra/terraform/aks/local.tf @@ -8,13 +8,13 @@ locals { }) dns_enabled = var.domain_name != null name_format = join("-", [ - "gitpod", + "gitpod-test", "%s", # region "%s", # name local.workspace_name ]) name_format_global = join("-", [ - "gitpod", + "gitpod-test", "%s", # name local.workspace_name ]) diff --git a/install/infra/terraform/tools/azure-external-dns/main.tf b/install/infra/terraform/tools/azure-external-dns/main.tf index 61c3017c53073a..bc8a1d89dc845f 100644 --- a/install/infra/terraform/tools/azure-external-dns/main.tf +++ b/install/infra/terraform/tools/azure-external-dns/main.tf @@ -1,6 +1,7 @@ variable settings {} variable domain_name { default = "test"} variable kubeconfig { default = "conf"} +variable txt_owner_id { default = "nightly-test"} provider "helm" { kubernetes { @@ -50,12 +51,8 @@ resource "helm_release" "external_dns" { value = var.settings["azure.resourceGroup"] } - # TODO Add tags using dynamic block - # https://github.com/hashicorp/terraform/issues/22340 - # dynamic "set" { - # for_each = var.tags - # iterator = "tag" - # name = "podLabels[${index(var.tags, tag.key)}]" - # value = tag.value - # } + set { + name = "txt-owner-id" + value = var.txt_owner_id + } } diff --git a/install/infra/terraform/tools/issuer/azure/main.tf b/install/infra/terraform/tools/issuer/azure/main.tf index d2ad4fd71ce4fe..e76a6d9761de05 100644 --- a/install/infra/terraform/tools/issuer/azure/main.tf +++ b/install/infra/terraform/tools/issuer/azure/main.tf @@ -15,6 +15,7 @@ resource "kubernetes_manifest" "clusterissuer_gitpod" { "name" = "issuer-account-key" } "server" = "https://acme-v02.api.letsencrypt.org/directory" + # "server" = "https://acme-staging-v02.api.letsencrypt.org/directory" "solvers" = [ { "dns01" = { diff --git a/install/tests/Makefile b/install/tests/Makefile index 342818ef4aacbc..05b0b2eb81cb53 100644 --- a/install/tests/Makefile +++ b/install/tests/Makefile @@ -33,10 +33,11 @@ gke-standard-cluster: aks-standard-cluster: terraform init --upgrade && \ terraform workspace new $(TF_VAR_TEST_ID) || terraform workspace select $(TF_VAR_TEST_ID) && \ - terraform apply -target=module.aks -var kubeconfig=${KUBECONFIG} --auto-approve + terraform apply -target=module.aks -var k8s_flavor="aks" -var kubeconfig=${KUBECONFIG} --auto-approve .PHONY: ## azure-external-dns: Sets up external-dns with azure provider +azure-external-dns: azure-external-dns: terraform init --upgrade && \ terraform workspace new $(TF_VAR_TEST_ID) || terraform workspace select $(TF_VAR_TEST_ID) && \ @@ -111,10 +112,24 @@ get-config-gcp-db: yq m -i tmp_config.yml tmp_5_config.yml get-config-azure-storage: + export PASSWORD=$$(terraform output -json storage | yq r - 'password') && \ + export USERNAME=$$(terraform output -json storage | yq r - 'username') && \ + envsubst < ./manifests/kots-config-azure-storage.yaml > tmp_2_config.yml + yq m -i tmp_config.yml tmp_2_config.yml get-config-azure-db: + export DBHOST=$$(terraform output -json database | yq r - 'host') && \ + export DBPASS=$$(terraform output -json database | yq r - 'password') && \ + export DBUSER=$$(terraform output -json database | yq r - 'username') && \ + envsubst < ./manifests/kots-config-azure-db.yaml > tmp_2_config.yml + yq m -i tmp_config.yml tmp_2_config.yml get-config-azure-registry: + export SERVER=$$(terraform output -json registry | yq r - 'server') && \ + export PASSWORD=$$(terraform output -json registry | yq r - 'password') && \ + export USERNAME=$$(terraform output -json registry | yq r - 'username') && \ + envsubst < ./manifests/kots-config-azure-registry.yaml > tmp_2_config.yml + yq m -i tmp_config.yml tmp_2_config.yml storage ?= incluster registry ?= incluster @@ -158,7 +173,7 @@ check-gitpod-installation: delete-cm-setup check-kots-app check-env-sub-domain run-tests: ./tests.sh ${KUBECONFIG} -kots-upgrade: +kots-uprgade: @echo "Upgrade gitpod KOTS app to latest" kubectl kots upstream upgrade --kubeconfig=${KUBECONFIG} gitpod -n gitpod --deploy @@ -174,13 +189,13 @@ destroy-certmanager: select-workspace ls ${KUBECONFIG} && terraform destroy -target=module.certmanager -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file" destroy-gcpns: select-workspace - ls ${KUBECONFIG} && terraform destroy -target=module.add_gcp_nameservers -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file" + terraform destroy -target=module.add_gcp_nameservers -var kubeconfig=${KUBECONFIG} --auto-approve destroy-aks-edns: select-workspace - ls ${KUBECONFIG} && terraform destroy -target=module.azure-externaldns -var kubeconfig=${KUBECONFIG} --auto-approve + ls ${KUBECONFIG} && terraform destroy -target=module.azure-externaldns -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file" destroy-aks-issuer: select-workspace - ls ${KUBECONFIG} && terraform destroy -target=module.azure-issuer -var kubeconfig=${KUBECONFIG} --auto-approve + ls ${KUBECONFIG} && terraform destroy -target=module.azure-issuer -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file" destroy-gke: select-workspace terraform destroy -target=module.gke -var kubeconfig=${KUBECONFIG} --auto-approve @@ -193,7 +208,7 @@ destroy-aks: select-workspace get-results: @echo "If you have gotten this far, it means your setup succeeded" - @echo "The IP address of you setup is "https://$(TF_VAR_TEST_ID).gitpod-self-hosted.com"" + @echo "The IP address of you setup is "$(TF_VAR_TEST_ID).gitpod-self-hosted.com"" @echo "Following is the KUBECONFIG you can use to connect to the cluster:" @cat ${KUBECONFIG} diff --git a/install/tests/main.tf b/install/tests/main.tf index bc2b5e08a550bf..2766913b761872 100644 --- a/install/tests/main.tf +++ b/install/tests/main.tf @@ -1,6 +1,8 @@ variable "kubeconfig" { } variable "TEST_ID" { default = "nightly" } +variable "k8s_flavor" { default = "gke" } + # We store the state always in a GCS bucket terraform { backend "gcs" { @@ -45,9 +47,9 @@ module "aks" { domain_name = "${var.TEST_ID}.gitpod-self-hosted.com" enable_airgapped = false - enable_external_database = false - enable_external_registry = false - enable_external_storage = false + enable_external_database = true + enable_external_registry = true + enable_external_storage = true dns_enabled = true workspace_name = var.TEST_ID } @@ -73,6 +75,7 @@ module "azure-externaldns" { kubeconfig = var.kubeconfig settings = module.aks.external_dns_settings domain_name = "${var.TEST_ID}.gitpod-self-hosted.com" + txt_owner_id = var.TEST_ID } module "azure-issuer" { diff --git a/install/tests/manifests/kots-config-azure-db.yaml b/install/tests/manifests/kots-config-azure-db.yaml new file mode 100644 index 00000000000000..1e113404ae2907 --- /dev/null +++ b/install/tests/manifests/kots-config-azure-db.yaml @@ -0,0 +1,16 @@ +apiVersion: kots.io/v1beta1 +kind: ConfigValues +spec: + values: + db_incluster: + value: "0" + data: "db_incluster" + db_host: + value: ${DBHOST} + data: "db_host" + db_username: + value: ${DBUSER} + data: "db_username" + db_password: + value: ${DBPASS} + data: "db_password" diff --git a/install/tests/manifests/kots-config-azure-registry.yaml b/install/tests/manifests/kots-config-azure-registry.yaml new file mode 100644 index 00000000000000..1442501e3f07b5 --- /dev/null +++ b/install/tests/manifests/kots-config-azure-registry.yaml @@ -0,0 +1,16 @@ +apiVersion: kots.io/v1beta1 +kind: ConfigValues +spec: + values: + reg_incluster: + value: "0" + data: "reg_incluster" + reg_url: + value: ${SERVER} + data: "reg_url" + reg_username: + value: ${USERNAME} + data: "reg_username" + reg_password: + value: ${PASSWORD} + data: "reg_password" diff --git a/install/tests/manifests/kots-config-azure-storage.yaml b/install/tests/manifests/kots-config-azure-storage.yaml new file mode 100644 index 00000000000000..408473b8f4e5cf --- /dev/null +++ b/install/tests/manifests/kots-config-azure-storage.yaml @@ -0,0 +1,16 @@ +apiVersion: kots.io/v1beta1 +kind: ConfigValues +spec: + values: + store_provider: + value: azure + data: "store_provider" + store_region: + value: "northeurope" + data: "store_region" + store_azure_account_name: + value: ${USERNAME} + data: "store_azure_account_name" + store_azure_access_key: + value: ${PASSWORD} + data: "store_azure_access_key" diff --git a/install/tests/output.tf b/install/tests/output.tf new file mode 100644 index 00000000000000..3982885ebec276 --- /dev/null +++ b/install/tests/output.tf @@ -0,0 +1,18 @@ +locals { + cloud = var.k8s_flavor == "aks" ? module.aks : null +} + +output "storage" { + sensitive = true + value = try(lookup(local.cloud, "storage"), {}) +} + +output "registry" { + sensitive = true + value = try(lookup(local.cloud, "registry"), {}) +} + +output "database" { + sensitive = true + value = try(lookup(local.cloud, "database"), {}) +}