diff --git a/.werft/installer-tests.ts b/.werft/installer-tests.ts index 7534054ab83bfd..3f2c4b2cc3fb8c 100644 --- a/.werft/installer-tests.ts +++ b/.werft/installer-tests.ts @@ -33,12 +33,15 @@ interface TestConfig { CLOUD: string; } +const k8s_version: string = randK8sVersion(testConfig) +const os_version: string = randOsVersion() // applicable only for k3s + // Each of the TEST_CONFIGURATIONS define an integration test end-to-end // It should be a combination of multiple INFRA_PHASES, order of PHASES slice is important const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = { STANDARD_GKE_TEST: { CLOUD: "gcp", - DESCRIPTION: "Deploy Gitpod on GKE, with managed DNS, and run integration tests", + DESCRIPTION: `Deploy Gitpod on GKE(version ${k8s_version})`, PHASES: [ "STANDARD_GKE_CLUSTER", "CERT_MANAGER", @@ -52,8 +55,7 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = { STANDARD_K3S_TEST: { CLOUD: "gcp", // the cloud provider is still GCP DESCRIPTION: - "Deploy Gitpod on a K3s cluster, created on a GCP instance," + - " with managed DNS and run integrations tests", + `Deploy Gitpod on a K3s cluster(version ${k8s_version}), on a GCP instance with ubuntu ${os_version}`, PHASES: [ "STANDARD_K3S_CLUSTER_ON_GCP", "CERT_MANAGER", @@ -65,7 +67,7 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = { }, STANDARD_AKS_TEST: { CLOUD: "azure", - DESCRIPTION: "Deploy Gitpod on AKS, with managed DNS, and run integration tests", + DESCRIPTION: `Deploy Gitpod on AKS(version ${k8s_version})`, PHASES: [ "STANDARD_AKS_CLUSTER", "CERT_MANAGER", @@ -79,7 +81,7 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = { }, STANDARD_EKS_TEST: { CLOUD: "aws", - DESCRIPTION: "Create an EKS cluster", + DESCRIPTION: `Create an EKS cluster(version ${k8s_version})`, PHASES: [ "STANDARD_EKS_CLUSTER", "CERT_MANAGER", @@ -110,23 +112,23 @@ const cloud: string = config.CLOUD; const INFRA_PHASES: { [name: string]: InfraConfig } = { STANDARD_GKE_CLUSTER: { phase: "create-std-gke-cluster", - makeTarget: "gke-standard-cluster", - description: "Creating a GKE cluster with 1 nodepool each for workspace and server", + makeTarget: `gke-standard-cluster`, + description: `Creating a GCP GKE cluster(version: ${k8s_version}) with 1 nodepool each for workspace and server`, }, STANDARD_K3S_CLUSTER_ON_GCP: { phase: "create-std-k3s-cluster", - makeTarget: "k3s-standard-cluster", - description: "Creating a k3s cluster on GCP with 1 node", + makeTarget: `k3s-standard-cluster os_version=${os_version}`, + description: `Creating a k3s(version: ${k8s_version}) cluster on GCP with 1 node`, }, STANDARD_AKS_CLUSTER: { phase: "create-std-aks-cluster", - makeTarget: "aks-standard-cluster", - description: "Creating an aks cluster(azure)", + makeTarget: `aks-standard-cluster`, + description: `Creating an Azure AKS cluster(version: ${k8s_version})`, }, STANDARD_EKS_CLUSTER: { phase: "create-std-eks-cluster", - makeTarget: "eks-standard-cluster", - description: "Creating a EKS cluster with 1 nodepool each for workspace and server", + makeTarget: `eks-standard-cluster`, + description: `Creating a AWS EKS cluster(version: ${k8s_version}) with 1 nodepool each for workspace and server`, }, CERT_MANAGER: { phase: "setup-cert-manager", @@ -140,7 +142,7 @@ const INFRA_PHASES: { [name: string]: InfraConfig } = { }, GENERATE_KOTS_CONFIG: { phase: "generate-kots-config", - makeTarget: `generate-kots-config storage=${randomize()} registry=${randomize()} db=${randomize()}`, + makeTarget: `generate-kots-config storage=${randDeps()} registry=${randDeps()} db=${randDeps()}`, description: `Generate KOTS Config file`, }, CLUSTER_ISSUER: { @@ -335,7 +337,7 @@ function callMakeTargets(phase: string, description: string, makeTarget: string, werft.log(phase, `Calling ${makeTarget}`); // exporting cloud env var is important for the make targets - const response = exec(`export cloud=${cloud} && make -C ${makefilePath} ${makeTarget}`, { + const response = exec(`export TF_VAR_cluster_version=${k8s_version} cloud=${cloud} && make -C ${makefilePath} ${makeTarget}`, { slice: phase, dontCheckRc: true, }); @@ -347,24 +349,61 @@ function callMakeTargets(phase: string, description: string, makeTarget: string, werft.fail(phase, "Operation failed"); return response.code; } - werft.log(phase, `Phase failed`); + werft.log(phase, `'${description}' failed`); } else { - werft.log(phase, `Phase succeeded`); + werft.log(phase, `'${description}' succeeded`); werft.done(phase); } return response.code; } -function randomize(): string { - // in the follow-up PR we will add `${platform}-${resource}` as an option here to - // test against resource dependencies(storage, db, registry) for each cloud platform +function randomize(options: string[]): string { + return options[Math.floor(Math.random() * options.length)]; +} + +function randDeps(): string { var depOptions: string[] = ["incluster", "external"] + if(deps && depOptions.includes(deps)) { return deps } - return depOptions[Math.floor(Math.random() * depOptions.length)]; + return randomize(depOptions) +} + +function randK8sVersion(config: string): string { + var options: string[] = [] + switch(config) { + case "STANDARD_GKE_TEST": { + options = ["1.21", "1.22", "1.23"] + break; + } + case "STANDARD_AKS_TEST": { + options = ["1.21", "1.22", "1.23"] + break; + } + case "STANDARD_EKS_TEST": { + options = ["1.20", "1.21", "1.22"] + break; + } + case "STANDARD_K3S_TEST": { + options = ["v1.22.12+k3s1", "v1.23.9+k3s1", "v1.24.3+k3s1"] + break; + } + } + // in the follow-up PR we will add `${platform}-${resource}` as an option here to + // test against resource dependencies(storage, db, registry) for each cloud platform + + return randomize(options) +} + +function randOsVersion(): string { + // in the follow-up PR we will add `${platform}-${resource}` as an option here to + // test against resource dependencies(storage, db, registry) for each cloud platform + var options: string[] = ["2204", "2004", "1804"] + + return randomize(options) } function cleanup() { diff --git a/.werft/jobs/build/self-hosted-upgrade-tests.ts b/.werft/jobs/build/self-hosted-upgrade-tests.ts index 2d655239ed808f..34c49970ea9f58 100644 --- a/.werft/jobs/build/self-hosted-upgrade-tests.ts +++ b/.werft/jobs/build/self-hosted-upgrade-tests.ts @@ -46,7 +46,7 @@ export async function triggerUpgradeTests(werft: Werft, config: JobConfig, usern werft.phase(upgradeConfig.phase, upgradeConfig.description); - annotation = `${annotation} -a cluster=${phase}` + annotation = `${annotation} -a cluster=${phase} -a updateGitHubStatus=gitpod-io/gitpod` const testFile: string = ".werft/self-hosted-installer-tests.yaml"; diff --git a/install/infra/terraform/aks/kubernetes.tf b/install/infra/terraform/aks/kubernetes.tf index 0d712b59736a79..68e502c974e54c 100644 --- a/install/infra/terraform/aks/kubernetes.tf +++ b/install/infra/terraform/aks/kubernetes.tf @@ -1,8 +1,3 @@ -data "azurerm_kubernetes_service_versions" "k8s" { - location = azurerm_resource_group.gitpod.location - include_preview = false -} - resource "azurerm_role_assignment" "k8s" { count = var.dns_enabled ? 1 : 0 @@ -20,14 +15,14 @@ resource "azurerm_role_assignment" "k8s_reader" { } resource "azurerm_kubernetes_cluster" "k8s" { - name = format(local.name_format, local.location, "primary") - location = azurerm_resource_group.gitpod.location - resource_group_name = azurerm_resource_group.gitpod.name - dns_prefix = "gitpod" - tags = {} - api_server_authorized_ip_ranges = [] - - kubernetes_version = data.azurerm_kubernetes_service_versions.k8s.latest_version + name = format(local.name_format, local.location, "primary") + location = azurerm_resource_group.gitpod.location + resource_group_name = azurerm_resource_group.gitpod.name + dns_prefix = "gitpod" + tags = {} + api_server_authorized_ip_ranges = [] + + kubernetes_version = var.cluster_version http_application_routing_enabled = false default_node_pool { @@ -35,14 +30,14 @@ resource "azurerm_kubernetes_cluster" "k8s" { vm_size = local.machine - node_taints = [] - tags = {} - zones = [] + node_taints = [] + tags = {} + zones = [] enable_auto_scaling = true min_count = 1 max_count = 10 - orchestrator_version = data.azurerm_kubernetes_service_versions.k8s.latest_version + orchestrator_version = var.cluster_version node_labels = local.nodes.0.labels type = "VirtualMachineScaleSets" @@ -50,7 +45,7 @@ resource "azurerm_kubernetes_cluster" "k8s" { } identity { - type = "SystemAssigned" + type = "SystemAssigned" identity_ids = [] } @@ -74,7 +69,7 @@ resource "azurerm_kubernetes_cluster_node_pool" "pools" { enable_auto_scaling = true min_count = 1 max_count = 10 - orchestrator_version = data.azurerm_kubernetes_service_versions.k8s.latest_version + orchestrator_version = var.cluster_version node_labels = local.nodes[count.index + 1].labels vnet_subnet_id = azurerm_subnet.network.id } diff --git a/install/infra/terraform/aks/variables.tf b/install/infra/terraform/aks/variables.tf index 94dabc09e6f98d..159182d4c8c542 100644 --- a/install/infra/terraform/aks/variables.tf +++ b/install/infra/terraform/aks/variables.tf @@ -1,8 +1,12 @@ // Common variables variable "kubeconfig" { - default = "./kubeconfig" + default = "./kubeconfig" +} +variable "cluster_version" { + description = "kubernetes version of to create the cluster with" } + variable "dns_enabled" {} variable "domain_name" {} variable "enable_airgapped" {} @@ -14,6 +18,5 @@ variable "workspace_name" { // Azure-specific variables variable "location" { - default = "northeurope" - + default = "northeurope" } diff --git a/install/infra/terraform/eks/kubernetes.tf b/install/infra/terraform/eks/kubernetes.tf index e48d92592eef17..5294e46a4ab2db 100644 --- a/install/infra/terraform/eks/kubernetes.tf +++ b/install/infra/terraform/eks/kubernetes.tf @@ -2,30 +2,30 @@ module "vpc" { source = "terraform-aws-modules/vpc/aws" version = "3.12.0" - name = "vpc-${var.cluster_name}" - cidr = var.vpc_cidr - azs = var.vpc_availability_zones - private_subnets = [var.private_primary_subnet_cidr, var.private_secondary_subnet_cidr] - public_subnets = [var.public_primary_subnet_cidr, var.public_secondary_subnet_cidr, var.public_db_subnet_cidr_1, var.public_db_subnet_cidr_2] - enable_nat_gateway = true + name = "vpc-${var.cluster_name}" + cidr = var.vpc_cidr + azs = var.vpc_availability_zones + private_subnets = [var.private_primary_subnet_cidr, var.private_secondary_subnet_cidr] + public_subnets = [var.public_primary_subnet_cidr, var.public_secondary_subnet_cidr, var.public_db_subnet_cidr_1, var.public_db_subnet_cidr_2] + enable_nat_gateway = true enable_dns_hostnames = true } resource "aws_security_group" "nodes" { - name = "nodes-sg-${var.cluster_name}" + name = "nodes-sg-${var.cluster_name}" vpc_id = module.vpc.vpc_id ingress { - from_port = 0 - to_port = 0 - protocol = "-1" + from_port = 0 + to_port = 0 + protocol = "-1" cidr_blocks = ["0.0.0.0/0"] } egress { - from_port = 0 - to_port = 0 - protocol = "-1" + from_port = 0 + to_port = 0 + protocol = "-1" cidr_blocks = ["0.0.0.0/0"] } } @@ -34,10 +34,10 @@ module "eks" { source = "terraform-aws-modules/eks/aws" version = "18.8.1" - cluster_name = var.cluster_name - cluster_version = "1.22" + cluster_name = var.cluster_name + cluster_version = var.cluster_version - cluster_endpoint_public_access = true + cluster_endpoint_public_access = true vpc_id = module.vpc.vpc_id subnet_ids = module.vpc.public_subnets @@ -58,7 +58,7 @@ module "eks" { iam_role_attach_cni_policy = true ami_id = var.image_id enable_bootstrap_user_data = true - vpc_security_group_ids = [aws_security_group.nodes.id] + vpc_security_group_ids = [aws_security_group.nodes.id] } eks_managed_node_groups = { @@ -144,7 +144,7 @@ module "vpc_cni_irsa" { } resource "null_resource" "kubeconfig" { - depends_on = [ module.eks ] + depends_on = [module.eks] provisioner "local-exec" { command = "aws eks update-kubeconfig --region ${var.region} --name ${var.cluster_name} --kubeconfig ${var.kubeconfig}" } diff --git a/install/infra/terraform/eks/variables.tf b/install/infra/terraform/eks/variables.tf index 8b7528ef17c567..192ba4d3398ffc 100644 --- a/install/infra/terraform/eks/variables.tf +++ b/install/infra/terraform/eks/variables.tf @@ -2,6 +2,13 @@ variable "cluster_name" { type = string description = "EKS cluster name." } + +variable "cluster_version" { + type = string + description = "Kubernetes version to create the cluster with" + default = "1.22" +} + variable "kubeconfig" { type = string description = "Path to the kubeconfig file" @@ -9,31 +16,31 @@ variable "kubeconfig" { } variable "image_id" { - type = string + type = string description = "AMI Image ID specific to the region" // latest ubuntu image for 1.22 k8s for eu-west-1 region, refer https://cloud-images.ubuntu.com/docs/aws/eks/ default = "ami-0793b4124359a6ad7" } variable "service_machine_type" { - type = string + type = string description = "Machine type for service workload node pool" - default = "m6i.xlarge" + default = "m6i.xlarge" } variable "workspace_machine_type" { - type = string + type = string description = "Machine type for workspace workload node pool" - default = "m6i.2xlarge" + default = "m6i.2xlarge" } variable "region" { - type = string + type = string default = "eu-west-1" } variable "vpc_availability_zones" { - type = list(string) + type = list(string) default = ["eu-west-1c", "eu-west-1b"] } diff --git a/install/infra/terraform/gke/main.tf b/install/infra/terraform/gke/main.tf index 664e7297a2cfbb..cb9b7afc561688 100644 --- a/install/infra/terraform/gke/main.tf +++ b/install/infra/terraform/gke/main.tf @@ -39,7 +39,7 @@ resource "google_compute_subnetwork" "subnet" { } resource "google_container_cluster" "gitpod-cluster" { - name = "c${var.name}" + name = "gitpod-${var.name}" location = var.zone == null ? var.region : var.zone cluster_autoscaling { @@ -58,7 +58,7 @@ resource "google_container_cluster" "gitpod-cluster" { } } - min_master_version = var.kubernetes_version + min_master_version = var.cluster_version # the default nodepool is used as the services nodepool remove_default_node_pool = false node_config { @@ -82,7 +82,7 @@ resource "google_container_cluster" "gitpod-cluster" { } } - initial_node_count = 1 + initial_node_count = 1 release_channel { channel = "UNSPECIFIED" } @@ -115,7 +115,7 @@ resource "google_container_node_pool" "workspaces" { name = "workspaces-${var.name}" location = google_container_cluster.gitpod-cluster.location cluster = google_container_cluster.gitpod-cluster.name - version = var.kubernetes_version // kubernetes version + version = var.cluster_version // kubernetes version initial_node_count = 1 max_pods_per_node = 110 @@ -153,26 +153,26 @@ resource "google_container_node_pool" "workspaces" { } resource "google_sql_database_instance" "gitpod" { - name = "sql-${var.name}" + name = "sql-${var.name}" database_version = "MYSQL_5_7" - region = "${var.region}" + region = var.region settings { - tier = "db-n1-standard-2" + tier = "db-n1-standard-2" } deletion_protection = false } resource "google_sql_database" "database" { - name = "gitpod" - instance = "${google_sql_database_instance.gitpod.name}" - charset = "utf8" - collation = "utf8_general_ci" + name = "gitpod" + instance = google_sql_database_instance.gitpod.name + charset = "utf8" + collation = "utf8_general_ci" } resource "google_sql_user" "users" { - name = "gitpod" - instance = "${google_sql_database_instance.gitpod.name}" - password = "gitpod" + name = "gitpod" + instance = google_sql_database_instance.gitpod.name + password = "gitpod" } module "gke_auth" { @@ -182,7 +182,7 @@ module "gke_auth" { project_id = var.project location = google_container_cluster.gitpod-cluster.location - cluster_name = "c${var.name}" + cluster_name = "gitpod-${var.name}" } resource "local_file" "kubeconfig" { diff --git a/install/infra/terraform/gke/variables.tf b/install/infra/terraform/gke/variables.tf index db80ab6a9bbfcc..c07d3588a514b0 100644 --- a/install/infra/terraform/gke/variables.tf +++ b/install/infra/terraform/gke/variables.tf @@ -20,7 +20,7 @@ variable "zone" { default = null } -variable "kubernetes_version" { +variable "cluster_version" { type = string description = "Kubernetes version to be setup" default = "1.22.8-gke.201" diff --git a/install/infra/terraform/k3s/main.tf b/install/infra/terraform/k3s/main.tf index 4a5916e02d1962..2184ca39bfd6c5 100644 --- a/install/infra/terraform/k3s/main.tf +++ b/install/infra/terraform/k3s/main.tf @@ -73,7 +73,7 @@ resource "google_compute_instance" "k3s_master_instance" { boot_disk { initialize_params { - image = "ubuntu-2004-focal-v20220419" + image = var.image_id size = 100 type = "pd-ssd" } @@ -118,6 +118,7 @@ resource "null_resource" "k3sup_install" { --context k3s \ --user gitpod \ --local-path ${var.kubeconfig} \ + --k3s-version ${var.cluster_version} \ --k3s-extra-args=" --disable=traefik --node-label=gitpod.io/workload_meta=true --node-label=gitpod.io/workload_ide=true --node-label=gitpod.io/workload_workspace_services=true --node-label=gitpod.io/workload_workspace_regular=true --node-label=gitpod.io/workload_workspace_headless=true" \ EOT } @@ -125,7 +126,7 @@ resource "null_resource" "k3sup_install" { resource "google_dns_record_set" "gitpod-dns" { provider = google.dns - count = (var.domain_name == null) || (var.managed_dns_zone == null ) ? 0 : 1 + count = (var.domain_name == null) || (var.managed_dns_zone == null) ? 0 : 1 name = "${var.domain_name}." managed_zone = var.managed_dns_zone project = var.dns_project == null ? var.gcp_project : var.dns_project @@ -137,7 +138,7 @@ resource "google_dns_record_set" "gitpod-dns" { resource "google_dns_record_set" "gitpod-dns-1" { provider = google.dns - count = (var.domain_name == null) || (var.managed_dns_zone == null ) ? 0 : 1 + count = (var.domain_name == null) || (var.managed_dns_zone == null) ? 0 : 1 name = "ws.${var.domain_name}." managed_zone = var.managed_dns_zone project = var.dns_project == null ? var.gcp_project : var.dns_project @@ -149,7 +150,7 @@ resource "google_dns_record_set" "gitpod-dns-1" { resource "google_dns_record_set" "gitpod-dns-2" { provider = google.dns - count = (var.domain_name == null) || (var.managed_dns_zone == null ) ? 0 : 1 + count = (var.domain_name == null) || (var.managed_dns_zone == null) ? 0 : 1 name = "*.${var.domain_name}." managed_zone = var.managed_dns_zone project = var.dns_project == null ? var.gcp_project : var.dns_project @@ -161,7 +162,7 @@ resource "google_dns_record_set" "gitpod-dns-2" { resource "google_dns_record_set" "gitpod-dns-3" { provider = google.dns - count = (var.domain_name == null) || (var.managed_dns_zone == null ) ? 0 : 1 + count = (var.domain_name == null) || (var.managed_dns_zone == null) ? 0 : 1 name = "*.ws.${var.domain_name}." managed_zone = var.managed_dns_zone project = var.dns_project == null ? var.gcp_project : var.dns_project @@ -172,26 +173,26 @@ resource "google_dns_record_set" "gitpod-dns-3" { } resource "google_sql_database_instance" "gitpod" { - name = "sql-${var.name}" + name = "sql-${var.name}" database_version = "MYSQL_5_7" - region = "${var.gcp_region}" + region = var.gcp_region settings { - tier = "db-n1-standard-2" + tier = "db-n1-standard-2" } deletion_protection = false } resource "google_sql_database" "database" { - name = "gitpod" - instance = "${google_sql_database_instance.gitpod.name}" - charset = "utf8" - collation = "utf8_general_ci" + name = "gitpod" + instance = google_sql_database_instance.gitpod.name + charset = "utf8" + collation = "utf8_general_ci" } resource "google_sql_user" "users" { - name = "gitpod" - instance = "${google_sql_database_instance.gitpod.name}" - password = "gitpod" + name = "gitpod" + instance = google_sql_database_instance.gitpod.name + password = "gitpod" } data "local_file" "kubeconfig" { diff --git a/install/infra/terraform/k3s/variables.tf b/install/infra/terraform/k3s/variables.tf index 26096d197bf097..5a4b889c5b5047 100644 --- a/install/infra/terraform/k3s/variables.tf +++ b/install/infra/terraform/k3s/variables.tf @@ -27,6 +27,16 @@ variable "name" { default = "k3s" } +variable "image_id" { + description = "Node image ID to be used to provision EC2 instances" + default = "ubuntu-2004-focal-v20220419" +} + +variable "cluster_version" { + description = "Kubernetes version to use to provision the cluster" + default = "v1.22.12+k3s1" +} + variable "dns_sa_creds" { description = "Credentials with DNS admin rights to the project with managed DNS record" default = "" diff --git a/install/tests/Makefile b/install/tests/Makefile index 8876f4e734ea6a..73f35eeaaea74c 100644 --- a/install/tests/Makefile +++ b/install/tests/Makefile @@ -17,6 +17,11 @@ ifndef cloud $(error cloud is not defined) endif +check-env-cluster-version: +ifndef TF_VAR_cluster_version + $(error TF_VAR_cluster_version is not defined) +endif + .PHONY: help all: help help: Makefile @@ -41,12 +46,13 @@ k3s-kubeconfig: sync-kubeconfig gcp-kubeconfig: gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS} --project=sh-automated-tests export KUBECONFIG=${KUBECONFIG} && \ - gcloud container clusters get-credentials c${TF_VAR_TEST_ID} --zone europe-west1-d --project sh-automated-tests || $(MAKE) sync-kubeconfig || echo "No cluster present" + gcloud container clusters get-credentials gitpod-${TF_VAR_TEST_ID} --zone europe-west1-d --project sh-automated-tests || $(MAKE) sync-kubeconfig || echo "No cluster present" ## azure-kubeconfig: Get the kubeconfig configuration for Azure AKS azure-kubeconfig: export KUBECONFIG=${KUBECONFIG} && \ - az aks get-credentials --name gitpod-test-nor-primary-${TF_VAR_TEST_ID} --resource-group gitpod-test-nor-${TF_VAR_TEST_ID} --file ${KUBECONFIG} || echo "No cluster present" + export resource=$$(echo "$$TF_VAR_TEST_ID" | sed "s/[\\W\\-]//") && \ + az aks get-credentials --name gitpod-test-nor-primary-$$resource --resource-group gitpod-test-nor-$$resource --file ${KUBECONFIG} || echo "No cluster present" ## aws-kubeconfig: Get the kubeconfig configuration for AWS EKS aws-kubeconfig: @@ -56,23 +62,30 @@ aws-kubeconfig: .PHONY: ## gke-standard-cluster: Creates a zonal GKE cluster -gke-standard-cluster: +gke-standard-cluster: check-env-cluster-version terraform init --upgrade && \ terraform workspace new $(TF_VAR_TEST_ID) || $(MAKE) select-workspace && \ terraform apply -target=module.gke -var kubeconfig=${KUBECONFIG} --auto-approve @echo "Done creating GKE cluster" +ami_id_120 := "ami-0ecb917eb4fbf4387" + +ami_id_121 := "ami-0d57fb01036fac543" + +ami_id_122 := "ami-0b306cb7e98db98e4" + .PHONY: ## eks-standard-cluster: Creates an EKS cluster -eks-standard-cluster: +eks-standard-cluster: ami_id = $(if $(ami_id_${TF_VAR_cluster_version//.}),$(ami_id_${TF_VAR_cluster_version//.}),$(ami_id_122)) +eks-standard-cluster: check-env-cluster-version terraform init --upgrade && \ terraform workspace new $(TF_VAR_TEST_ID) || $(MAKE) select-workspace && \ - terraform apply -target=module.eks -var kubeconfig=${KUBECONFIG} --auto-approve + terraform apply -target=module.eks -var kubeconfig=${KUBECONFIG} -var eks_node_image_id=${ami_id} --auto-approve @echo "Done creating EKS cluster" .PHONY: ## aks-standard-cluster: Creates an AKS cluster -aks-standard-cluster: +aks-standard-cluster: check-env-cluster-version terraform init --upgrade && \ terraform workspace new $(TF_VAR_TEST_ID) || $(MAKE) select-workspace && \ terraform apply -target=module.aks -var kubeconfig=${KUBECONFIG} --auto-approve @@ -94,12 +107,20 @@ cluster-issuer: check-env-cloud terraform apply -target=module.$(cloud)-issuer -var kubeconfig=${KUBECONFIG} --auto-approve @echo "Done creating cluster issuer" +image_id_1804 := "ubuntu-1804-bionic-v20220712" + +image_id_2004 := "ubuntu-2004-focal-v20220712" + +image_id_2204 := "ubuntu-2204-jammy-v20220712a" + +os_version ?= "2004" .PHONY: ## k3s-standard-cluster: Creates a K3S cluster on GCP with one master and 1 worker node -k3s-standard-cluster: +k3s-standard-cluster: image_id = $(if $(image_id_$(os_version)),$(image_id_$(os_version)),$(image_id_2004)) +k3s-standard-cluster: check-env-cluster-version terraform init --upgrade && \ terraform workspace new $(TF_VAR_TEST_ID) || $(MAKE) select-workspace && \ - terraform apply -target=module.k3s -var kubeconfig=${KUBECONFIG} --auto-approve && \ + terraform apply -target=module.k3s -var kubeconfig=${KUBECONFIG} -var k3s_node_image_id=${image_id} --auto-approve && \ $(MAKE) upload-kubeconfig-to-gcp # we upload the file to GCP since we cannot retrieve the file against without SSHing to the master @echo "Done creating k3s cluster" @@ -241,8 +262,13 @@ delete-cm-setup: sleeptime=$(if $(time_to_sleep_$(cloud)),$(time_to_sleep_$(clou delete-cm-setup: sleep 180 && kubectl --kubeconfig=${KUBECONFIG} delete pods --all -n cert-manager && sleep ${sleeptime}; +gitpod-debug-info: + @echo "Gitpod is not ready" + @kubectl get pods -n gitpod + @kubectl get certificate -n gitpod + check-kots-app: - kubectl kots get --kubeconfig=${KUBECONFIG} app gitpod -n gitpod | grep gitpod | awk '{print $$2}' | grep "ready" || { echo "Gitpod is not ready"; exit 1; } + kubectl kots get --kubeconfig=${KUBECONFIG} app gitpod -n gitpod | grep gitpod | awk '{print $$2}' | grep "ready" || { $(MAKE) gitpod-debug-info; exit 1; } check-gitpod-installation: delete-cm-setup check-kots-app check-env-sub-domain @echo "Curling http://${TF_VAR_TEST_ID}.tests.gitpod-self-hosted.com/api/version" @@ -293,7 +319,7 @@ destroy-cluster: destroy-gcp destroy-aws destroy-azure destroy-kubeconfig: gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS} --project=sh-automated-tests gsutil rm gs://nightly-tests/tf-state/${TF_VAR_TEST_ID}-kubeconfig || echo "No kubeconfig" - rm ${KUBECONFIG} + rm ${KUBECONFIG} || echo "No kubeconfig" select-workspace: terraform workspace select $(TF_VAR_TEST_ID) diff --git a/install/tests/main.tf b/install/tests/main.tf index cf4219d6c74638..d4196b3af4b7a0 100644 --- a/install/tests/main.tf +++ b/install/tests/main.tf @@ -1,4 +1,4 @@ -variable "kubeconfig" { } +variable "kubeconfig" {} variable "TEST_ID" { default = "nightly" } # We store the state always in a GCS bucket @@ -11,18 +11,31 @@ terraform { variable "project" { default = "sh-automated-tests" } variable "sa_creds" { default = null } -variable "dns_sa_creds" {default = null } +variable "dns_sa_creds" { default = null } + +variable "eks_node_image_id" { + default = "ami-0793b4124359a6ad7" // this AMI is regional +} + +variable "k3s_node_image_id" { + default = null +} + +variable "cluster_version" { + default = "1.22" +} module "gke" { # source = "github.com/gitpod-io/gitpod//install/infra/terraform/gke?ref=main" # we can later use tags here source = "../infra/terraform/gke" # we can later use tags here - name = var.TEST_ID - project = var.project - credentials = var.sa_creds - kubeconfig = var.kubeconfig - region = "europe-west1" - zone = "europe-west1-d" + name = var.TEST_ID + project = var.project + credentials = var.sa_creds + kubeconfig = var.kubeconfig + region = "europe-west1" + zone = "europe-west1-d" + cluster_version = var.cluster_version } module "k3s" { @@ -37,17 +50,19 @@ module "k3s" { dns_project = "dns-for-playgrounds" managed_dns_zone = "tests-gitpod-self-hosted-com" domain_name = "${var.TEST_ID}.tests.gitpod-self-hosted.com" + cluster_version = var.cluster_version + image_id = var.k3s_node_image_id } module "gcp-issuer" { - source = "../infra/terraform/tools/issuer" - kubeconfig = var.kubeconfig - issuer_name = "cloudDNS" + source = "../infra/terraform/tools/issuer" + kubeconfig = var.kubeconfig + issuer_name = "cloudDNS" cert_manager_issuer = { - project = "dns-for-playgrounds" + project = "dns-for-playgrounds" serviceAccountSecretRef = { name = "clouddns-dns01-solver" - key = "keys.json" + key = "keys.json" } } } @@ -55,7 +70,7 @@ module "gcp-issuer" { module "aks" { # source = "github.com/gitpod-io/gitpod//install/infra/terraform/aks?ref=main" # we can later use tags here - source = "../infra/terraform/aks" + source = "../infra/terraform/aks" domain_name = "${var.TEST_ID}.tests.gitpod-self-hosted.com" enable_airgapped = false @@ -65,6 +80,7 @@ module "aks" { dns_enabled = true workspace_name = var.TEST_ID kubeconfig = var.kubeconfig + cluster_version = var.cluster_version } module "eks" { @@ -73,13 +89,14 @@ module "eks" { cluster_name = var.TEST_ID region = "eu-west-1" vpc_availability_zones = ["eu-west-1c", "eu-west-1b"] - image_id = "ami-0793b4124359a6ad7" // this AMI is regional + image_id = var.eks_node_image_id kubeconfig = var.kubeconfig + cluster_version = var.cluster_version } module "certmanager" { # source = "github.com/gitpod-io/gitpod//install/infra/terraform/tools/cert-manager?ref=main" - source = "../infra/terraform/tools/cert-manager" + source = "../infra/terraform/tools/cert-manager" kubeconfig = var.kubeconfig credentials = var.dns_sa_creds