From 84e477a6e438ff1c992e288c3c4ec190a21a6739 Mon Sep 17 00:00:00 2001
From: Nandaja Varma <nandaja.varma@gmail.com>
Date: Thu, 23 Jun 2022 09:47:08 +0000
Subject: [PATCH] add DNS code

---
 .werft/installer-tests.ts                     |   7 +-
 install/infra/terraform/aks/output.tf         |  34 ++-
 install/infra/terraform/eks/dns.tf            |  76 +++++++
 install/infra/terraform/eks/kubernetes.tf     | 203 ------------------
 install/infra/terraform/eks/network.tf        | 103 ---------
 install/infra/terraform/eks/output.tf         |  30 +++
 install/infra/terraform/eks/variables.tf      |  24 ---
 .../terraform/tools/external-dns/main.tf      |  40 +---
 install/tests/Makefile                        |  58 ++---
 install/tests/main.tf                         |  45 ++--
 10 files changed, 191 insertions(+), 429 deletions(-)
 create mode 100644 install/infra/terraform/eks/dns.tf
 delete mode 100644 install/infra/terraform/eks/kubernetes.tf
 delete mode 100644 install/infra/terraform/eks/network.tf
 create mode 100644 install/infra/terraform/eks/output.tf
 delete mode 100644 install/infra/terraform/eks/variables.tf

diff --git a/.werft/installer-tests.ts b/.werft/installer-tests.ts
index bee652d807b197..c1432ba1fd52ef 100644
--- a/.werft/installer-tests.ts
+++ b/.werft/installer-tests.ts
@@ -214,7 +214,7 @@ const INFRA_PHASES: { [name: string]: InfraConfig } = {
     },
     DESTROY: {
         phase: "destroy",
-        makeTarget: "cleanup",
+        makeTarget: `cleanup cloud=${cloud}`,
         description: "Destroy the created infrastucture",
     },
     RESULTS: {
@@ -279,7 +279,10 @@ function cleanup() {
     const phase = "destroy-infrastructure";
     werft.phase(phase, "Destroying all the created resources");
 
-    const response = exec(`make -C ${makefilePath} cleanup`, { slice: "run-terrafrom-destroy", dontCheckRc: true });
+    const response = exec(`make -C ${makefilePath} cleanup cloud=${cloud}`, {
+        slice: "run-terrafrom-destroy",
+        dontCheckRc: true,
+    });
 
     // if the destroy command fail, we check if any resources are pending to be removed
     // if nothing is yet to be cleaned, we return with success
diff --git a/install/infra/terraform/aks/output.tf b/install/infra/terraform/aks/output.tf
index 0ea0d4bf2f8b0e..83c60ee2205c81 100644
--- a/install/infra/terraform/aks/output.tf
+++ b/install/infra/terraform/aks/output.tf
@@ -36,14 +36,32 @@ output "external_dns_secrets" {
 }
 
 output "external_dns_settings" {
-  value = {
-    provider                            = "azure"
-    "azure.resourceGroup"               = azurerm_resource_group.gitpod.name
-    "azure.subscriptionId"              = data.azurerm_client_config.current.subscription_id
-    "azure.tenantId"                    = data.azurerm_client_config.current.tenant_id
-    "azure.useManagedIdentityExtension" = true
-    "azure.userAssignedIdentityID"      = azurerm_kubernetes_cluster.k8s.kubelet_identity.0.client_id
-  }
+  value = [
+    {
+      "name": "provider",
+      "value": "azure"
+    },
+    {
+      "name": "azure.resourceGroup",
+      "value": azurerm_resource_group.gitpod.name,
+    },
+    {
+      "name": "azure.subscriptionId",
+      "value": data.azurerm_client_config.current.subscription_id,
+    },
+    {
+      "name": "azure.tenantId",
+      "value": data.azurerm_client_config.current.tenant_id,
+    },
+    {
+      "name": "azure.useManagedIdentityExtension",
+      "value": true
+    },
+    {
+      "name": "azure.userAssignedIdentityID",
+      "value": azurerm_kubernetes_cluster.k8s.kubelet_identity.0.client_id
+    },
+  ]
 }
 
 output "k8s_connection" {
diff --git a/install/infra/terraform/eks/dns.tf b/install/infra/terraform/eks/dns.tf
new file mode 100644
index 00000000000000..ce4d78201c4740
--- /dev/null
+++ b/install/infra/terraform/eks/dns.tf
@@ -0,0 +1,76 @@
+variable "domain_name" {}
+variable "cluster_name" {}
+
+terraform {
+  required_providers {
+    aws = {
+        version = " ~> 3.0"
+        source = "registry.terraform.io/hashicorp/aws"
+    }
+  }
+}
+
+provider "aws" {
+  region  = "eu-west-1"
+}
+
+resource "aws_route53_zone" "gitpod" {
+  name = var.domain_name
+
+  tags = {
+    Environment = "test"
+  }
+}
+
+resource "aws_iam_policy" "gitpod" {
+  name = "role-${var.cluster_name}"
+
+  # Terraform's "jsonencode" function converts a
+  # Terraform expression result to valid JSON syntax.
+  policy = jsonencode({
+    Version = "2012-10-17",
+    Statement = [
+        {
+            Effect = "Allow",
+            Action = [
+                "route53:ChangeResourceRecordSets"
+            ],
+            Resource = [
+                "arn:aws:route53:::hostedzone/*"
+            ]
+        },
+        {
+            Effect = "Allow",
+            Action = [
+                "route53:ListHostedZones",
+                "route53:ListResourceRecordSets"
+            ],
+            Resource = [ "*" ]
+        }
+    ],
+  })
+}
+
+resource "aws_iam_role" "gitpod" {
+  name = "iam-route53-${var.cluster_name}"
+
+  assume_role_policy = <<POLICY
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": {
+        "Service": "ec2.amazonaws.com"
+      },
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}
+POLICY
+}
+
+resource "aws_iam_role_policy_attachment" "route53" {
+  policy_arn = resource.aws_iam_policy.gitpod.arn
+  role       = aws_iam_role.gitpod.name
+}
diff --git a/install/infra/terraform/eks/kubernetes.tf b/install/infra/terraform/eks/kubernetes.tf
deleted file mode 100644
index 08900f4f0b624f..00000000000000
--- a/install/infra/terraform/eks/kubernetes.tf
+++ /dev/null
@@ -1,203 +0,0 @@
-terraform {
-  required_providers {
-    kubectl = {
-      source  = "gavinbunney/kubectl"
-      version = ">= 1.7.0"
-    }
-    aws = {
-        version = " ~> 3.0"
-        source = "registry.terraform.io/hashicorp/aws"
-    }
-  }
-}
-
-resource "aws_iam_role" "eks_cluster" {
-  depends_on = [data.aws_subnet_ids.subnet_ids]
-  name = "iam-${var.cluster_name}"
-
-  assume_role_policy = <<POLICY
-{
-  "Version": "2012-10-17",
-  "Statement": [
-    {
-      "Effect": "Allow",
-      "Principal": {
-        "Service": "eks.amazonaws.com"
-      },
-      "Action": "sts:AssumeRole"
-    }
-  ]
-}
-POLICY
-}
-
-resource "aws_iam_role_policy_attachment" "AmazonEKSClusterPolicy" {
-  policy_arn = "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy"
-  role       = aws_iam_role.eks_cluster.name
-}
-
-resource "aws_iam_role_policy_attachment" "AmazonEKSServicePolicy" {
-  policy_arn = "arn:aws:iam::aws:policy/AmazonEKSServicePolicy"
-  role       = aws_iam_role.eks_cluster.name
-}
-
-resource "aws_eks_cluster" "aws_eks" {
-  name     = var.cluster_name
-  role_arn = aws_iam_role.eks_cluster.arn
-
-  vpc_config {
-    subnet_ids = data.aws_subnet_ids.subnet_ids.ids
-  }
-
-  tags = {
-    Name = "EKS_tuto"
-  }
-
-  depends_on = [
-    aws_iam_role.eks_cluster,
- ]
-}
-
-data "aws_eks_cluster" "cluster" {
-  depends_on = [
-    aws_eks_cluster.aws_eks,
-  ]
-  name = resource.aws_eks_cluster.aws_eks.id
-}
-
-data "aws_eks_cluster_auth" "cluster" {
-  depends_on = [
-    aws_eks_cluster.aws_eks,
-  ]
-  name = resource.aws_eks_cluster.aws_eks.id
-}
-
-
-resource "aws_iam_role" "eks_nodes" {
-  name = "iam-ng-${var.cluster_name}"
-
-  assume_role_policy = <<POLICY
-{
-  "Version": "2012-10-17",
-  "Statement": [
-    {
-      "Effect": "Allow",
-      "Principal": {
-        "Service": "ec2.amazonaws.com"
-      },
-      "Action": "sts:AssumeRole"
-    }
-  ]
-}
-POLICY
-}
-
-resource "aws_iam_role_policy_attachment" "AmazonEKSWorkerNodePolicy" {
-  policy_arn = "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy"
-  role       = aws_iam_role.eks_nodes.name
-}
-
-resource "aws_iam_role_policy_attachment" "AmazonEKS_CNI_Policy" {
-  policy_arn = "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy"
-  role       = aws_iam_role.eks_nodes.name
-}
-
-locals {
-  map_roles = <<ROLES
-- rolearn: ${aws_iam_role.eks_nodes.arn}
-  username: system:node:{{EC2PrivateDNSName}}
-  groups:
-    - system:bootstrappers
-    - system:nodes
-ROLES
-}
-
-resource "aws_iam_role_policy_attachment" "AmazonSSMManagedInstanceCore" {
-  policy_arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"
-  role       = aws_iam_role.eks_nodes.name
-}
-
-resource "aws_iam_role_policy_attachment" "AmazonEC2ContainerRegistryReadOnly" {
-  policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly"
-  role       = aws_iam_role.eks_nodes.name
-}
-
-resource "aws_iam_role_policy_attachment" "EC2InstanceProfileForImageBuilderECRContainerBuilds" {
-  policy_arn = "arn:aws:iam::aws:policy/EC2InstanceProfileForImageBuilderECRContainerBuilds"
-  role       = aws_iam_role.eks_nodes.name
-}
-
-resource "aws_launch_template" "eks" {
-  name = "${var.cluster_name}-template"
-  update_default_version = true
-  block_device_mappings {
-    device_name = "/dev/sda1"
-    ebs {
-      volume_size = 100
-    }
-  }
-  credit_specification {
-    cpu_credits = "standard"
-  }
-  ebs_optimized                        = true
-  # AMI generated with packer (is private)
-  image_id                             = "ami-0f08b4b1a4fd3ebe3"
-  network_interfaces {
-    associate_public_ip_address = false
-  }
-}
-
-resource "aws_eks_node_group" "workspace" {
-  cluster_name    = aws_eks_cluster.aws_eks.name
-  node_group_name = "ngw-${var.cluster_name}"
-  node_role_arn   = aws_iam_role.eks_nodes.arn
-  subnet_ids      = data.aws_subnet_ids.subnet_ids.ids
-  instance_types  = ["m6i.2xlarge"]
-  labels = {
-      "gitpod.io/workload_workspace_services" = true
-      "gitpod.io/workload_workspace_regular"  = true
-      "gitpod.io/workload_workspace_headless" = true
-      "gitpod.io/workload_meta" = true
-      "gitpod.io/workload_ide"  = true
-  }
-
-  scaling_config {
-    desired_size = 1
-    max_size     = 10
-    min_size     = 1
-  }
-
-  # Ensure that IAM Role permissions are created before and deleted after EKS Node Group handling.
-  # Otherwise, EKS will not be able to properly delete EC2 Instances and Elastic Network Interfaces.
-  depends_on = [
-    resource.aws_iam_role_policy_attachment.AmazonSSMManagedInstanceCore,
-    resource.aws_iam_role_policy_attachment.AmazonEC2ContainerRegistryReadOnly,
-    resource.aws_iam_role_policy_attachment.EC2InstanceProfileForImageBuilderECRContainerBuilds,
-  ]
-
-  launch_template {
-    id = resource.aws_launch_template.eks.id
-    version = aws_launch_template.eks.latest_version
-  }
-}
-
-provider "kubectl" {
-  host                   = resource.aws_eks_cluster.aws_eks.endpoint
-  cluster_ca_certificate = base64decode(resource.aws_eks_cluster.aws_eks.certificate_authority[0].data)
-  token                  = data.aws_eks_cluster_auth.cluster.token
-  config_path            = var.kubeconfig
-}
-
-output host {
-  value = resource.aws_eks_cluster.aws_eks.endpoint
-}
-
-output ca {
-  sensitive = true
-  value = base64decode(resource.aws_eks_cluster.aws_eks.certificate_authority[0].data)
-}
-
-output token {
-  sensitive = true
-  value = data.aws_eks_cluster_auth.cluster.token
-}
diff --git a/install/infra/terraform/eks/network.tf b/install/infra/terraform/eks/network.tf
deleted file mode 100644
index f9a616abda997d..00000000000000
--- a/install/infra/terraform/eks/network.tf
+++ /dev/null
@@ -1,103 +0,0 @@
-# get all available AZs in our region
-data "aws_availability_zones" "available_azs" {
-  state = "available"
-}
-
-# reserve Elastic IP to be used in our NAT gateway
-resource "aws_eip" "nat_gw_elastic_ip" {
-  vpc = true
-
-  tags = {
-    Name = "${var.cluster_name}-nat-eip"
-  }
-}
-
-# create VPC using the official AWS module
-module "vpc" {
-  source  = "terraform-aws-modules/vpc/aws"
-  version = "3.14.0"
-
-  name = "${var.cluster_name}-vpc"
-  cidr = var.main_network_block
-  azs  = data.aws_availability_zones.available_azs.names
-
-  private_subnets = [
-    # this loop will create a one-line list as ["10.0.0.0/20", "10.0.16.0/20", "10.0.32.0/20", ...]
-    # with a length depending on how many Zones are available
-    for zone_id in data.aws_availability_zones.available_azs.zone_ids :
-    cidrsubnet(var.main_network_block, var.subnet_prefix_extension, tonumber(substr(zone_id, length(zone_id) - 1, 1)) - 1)
-  ]
-
-  public_subnets = [
-    # this loop will create a one-line list as ["10.0.128.0/20", "10.0.144.0/20", "10.0.160.0/20", ...]
-    # with a length depending on how many Zones are available
-    # there is a zone Offset variable, to make sure no collisions are present with private subnet blocks
-    for zone_id in data.aws_availability_zones.available_azs.zone_ids :
-    cidrsubnet(var.main_network_block, var.subnet_prefix_extension, tonumber(substr(zone_id, length(zone_id) - 1, 1)) + var.zone_offset - 1)
-  ]
-
-  # enable single NAT Gateway to save some money
-  # WARNING: this could create a single point of failure, since we are creating a NAT Gateway in one AZ only
-  # feel free to change these options if you need to ensure full Availability without the need of running 'terraform apply'
-  # reference: https://registry.terraform.io/modules/terraform-aws-modules/vpc/aws/2.44.0#nat-gateway-scenarios
-  enable_nat_gateway     = true
-  single_nat_gateway     = true
-  one_nat_gateway_per_az = false
-  enable_dns_hostnames   = true
-  reuse_nat_ips          = true
-  external_nat_ip_ids    = [aws_eip.nat_gw_elastic_ip.id]
-
-  # add VPC/Subnet tags required by EKS
-  tags = {
-    "kubernetes.io/cluster/${var.cluster_name}" = "shared"
-  }
-  public_subnet_tags = {
-    "kubernetes.io/cluster/${var.cluster_name}" = "shared"
-    "kubernetes.io/role/elb"                    = "1"
-  }
-  private_subnet_tags = {
-    "kubernetes.io/cluster/${var.cluster_name}" = "shared"
-    "kubernetes.io/role/internal-elb"           = "1"
-  }
-}
-
-# create security group to be used later by the ingress ALB
-resource "aws_security_group" "alb" {
-  name   = "${var.cluster_name}-alb"
-  vpc_id = module.vpc.vpc_id
-
-  ingress {
-    description      = "http"
-    from_port        = 80
-    to_port          = 80
-    protocol         = "tcp"
-    cidr_blocks      = ["0.0.0.0/0"]
-    ipv6_cidr_blocks = ["::/0"]
-  }
-
-  ingress {
-    description      = "https"
-    from_port        = 443
-    to_port          = 443
-    protocol         = "tcp"
-    cidr_blocks      = ["0.0.0.0/0"]
-    ipv6_cidr_blocks = ["::/0"]
-  }
-
-  egress {
-    from_port        = 0
-    to_port          = 0
-    protocol         = "-1"
-    cidr_blocks      = ["0.0.0.0/0"]
-    ipv6_cidr_blocks = ["::/0"]
-  }
-
-  tags = {
-    "Name" = "${var.cluster_name}-alb"
-  }
-}
-
-data "aws_subnet_ids" "subnet_ids" {
-  depends_on = [module.vpc]
-  vpc_id     = module.vpc.vpc_id
-}
diff --git a/install/infra/terraform/eks/output.tf b/install/infra/terraform/eks/output.tf
new file mode 100644
index 00000000000000..7771d5332be311
--- /dev/null
+++ b/install/infra/terraform/eks/output.tf
@@ -0,0 +1,30 @@
+data "aws_caller_identity" "current" {}
+
+output "external_dns_settings" {
+  value = [
+    {
+        "name" = "provider",
+        "value" = "aws"
+    },
+    {
+        "name" = "aws.region",
+        "value" = "eu-west-1"
+    },
+    {
+        "name" = "aws.roleArn",
+        "value" = resource.aws_iam_role.gitpod.arn
+    }
+    ]
+}
+
+output "cert_manager_issuer" {
+  value = try({
+        region = "eu-west-1"
+        role   = resource.aws_iam_role.gitpod.arn
+        hostedZoneID  = resource.aws_route53_zone.gitpod.zone_id
+  }, {})
+}
+
+output "domain_nameservers" {
+  value = resource.aws_route53_zone.gitpod.name_servers
+}
diff --git a/install/infra/terraform/eks/variables.tf b/install/infra/terraform/eks/variables.tf
deleted file mode 100644
index c86eefa03d7639..00000000000000
--- a/install/infra/terraform/eks/variables.tf
+++ /dev/null
@@ -1,24 +0,0 @@
-variable "cluster_name" {
-  type        = string
-  description = "EKS cluster name."
-}
-variable "kubeconfig" {
-  type        = string
-  description = "Path to the kubeconfig file"
-  default     = "kubeconfig"
-}
-variable "main_network_block" {
-  type        = string
-  description = "Base CIDR block to be used in our VPC."
-  default     = "10.0.0.0/16"
-}
-variable "subnet_prefix_extension" {
-  type        = number
-  description = "CIDR block bits extension to calculate CIDR blocks of each subnetwork."
-  default     = 8
-}
-variable "zone_offset" {
-  type        = number
-  description = "CIDR block bits extension offset to calculate Public subnets, avoiding collisions with Private subnets."
-  default     = 21
-}
diff --git a/install/infra/terraform/tools/external-dns/main.tf b/install/infra/terraform/tools/external-dns/main.tf
index a02a989a141138..86354e1e4ad567 100644
--- a/install/infra/terraform/tools/external-dns/main.tf
+++ b/install/infra/terraform/tools/external-dns/main.tf
@@ -1,7 +1,7 @@
 variable settings {}
 variable domain_name { default = "test"}
 variable kubeconfig { default = "conf"}
-variable provider { default = "azure"}
+variable txt_owner_id { default = "nightly-test" }
 
 provider "helm" {
   kubernetes {
@@ -21,10 +21,9 @@ resource "helm_release" "external_dns" {
     name  = "domainFilters[0]"
     value = var.domain_name
   }
-
   set {
-    name  = "provider"
-    value = var.provider
+    name = "txt-owner-id"
+    value = var.txt_owner_id
   }
 
   dynamic "set" {
@@ -34,37 +33,4 @@ resource "helm_release" "external_dns" {
       value = setting.value["value"]
     }
   }
-  # set {
-  #   name  = "azure.userAssignedIdentityID"
-  #   value = var.settings["azure.userAssignedIdentityID"]
-  # }
-
-  # set {
-  #   name  = "azure.useManagedIdentityExtension"
-  #   value = var.settings["azure.useManagedIdentityExtension"]
-  # }
-
-  # set {
-  #   name  = "azure.tenantId"
-  #   value = var.settings["azure.tenantId"]
-  # }
-
-  # set {
-  #   name  = "azure.subscriptionId"
-  #   value = var.settings["azure.subscriptionId"]
-  # }
-
-  # set {
-  #   name  = "azure.resourceGroup"
-  #   value = var.settings["azure.resourceGroup"]
-  # }
-
-  # TODO Add tags using dynamic block
-  # https://github.com/hashicorp/terraform/issues/22340
-  #  dynamic "set" {
-  #    for_each = var.tags
-  #    iterator = "tag"
-  #    name     = "podLabels[${index(var.tags, tag.key)}]"
-  #    value    = tag.value
-  #  }
 }
diff --git a/install/tests/Makefile b/install/tests/Makefile
index 0fb75492f7f32a..29ae037fe9113a 100644
--- a/install/tests/Makefile
+++ b/install/tests/Makefile
@@ -50,8 +50,8 @@ add-ns-record:
 	terraform apply -target=module.add_gcp_nameservers  -var kubeconfig=${KUBECONFIG} --auto-approve
 
 .PHONY:
-## azure-issuer: Creates a cluster issuer with AD access
-azure-issuer:
+## cluster-issuer: Creates a cluster issuer for the correspondign provider
+cluster-issuer:
 	terraform init --upgrade && \
 	terraform workspace new $(TF_VAR_TEST_ID) || terraform workspace select $(TF_VAR_TEST_ID) && \
 	terraform apply -target=module.azure-issuer  -var kubeconfig=${KUBECONFIG} --auto-approve
@@ -78,12 +78,12 @@ managed-dns: check-env-sub-domain
 	terraform apply -target=module.clouddns-externaldns -var kubeconfig=${KUBECONFIG} --auto-approve && \
     kubectl --kubeconfig=${KUBECONFIG} apply -f ${CLUSTER_ISSUER_CLOUD_DNS}
 
-provider ?= "azure"
+cloud ?= "azure"
 .PHONY:
 ## external-dns: Installs external-dns
 external-dns: check-env-sub-domain
 	terraform workspace select $(TF_VAR_TEST_ID) && \
-	terraform apply -target=module.externaldns -var provider=${provider} -var kubeconfig=${KUBECONFIG} --auto-approve
+	terraform apply -target=module.$(cloud)-externaldns -var kubeconfig=${KUBECONFIG} --auto-approve
 
 .PHONY:
 ## get-kubeconfig: Returns KUBECONFIG of a just created cluster
@@ -117,19 +117,6 @@ get-config-gcp-db:
 	envsubst '$${TF_VAR_TEST_ID}' < tmp_4_config.yml > tmp_5_config.yml
 	yq m -i tmp_config.yml tmp_5_config.yml
 
-get-config-azure-storage:
-	export PASSWORD=$$(terraform output -json storage | yq r - 'password') && \
-	export USERNAME=$$(terraform output -json storage | yq r - 'username') && \
-	envsubst < ./manifests/kots-config-azure-storage.yaml > tmp_2_config.yml
-	yq m -i tmp_config.yml tmp_2_config.yml
-
-get-config-azure-db:
-	export DBHOST=$$(terraform output -json database | yq r - 'host') && \
-	export DBPASS=$$(terraform output -json database | yq r - 'password') && \
-	export DBUSER=$$(terraform output -json database | yq r - 'username') && \
-	envsubst < ./manifests/kots-config-azure-db.yaml > tmp_2_config.yml
-	yq m -i tmp_config.yml tmp_2_config.yml
-
 get-config-azure-registry:
 	export SERVER=$$(terraform output -json registry | yq r - 'server') && \
 	export PASSWORD=$$(terraform output -json registry | yq r - 'password') && \
@@ -150,13 +137,6 @@ get-config-azure-db:
 	envsubst < ./manifests/kots-config-azure-db.yaml > tmp_2_config.yml
 	yq m -i tmp_config.yml tmp_2_config.yml
 
-get-config-azure-registry:
-	export SERVER=$$(terraform output -json registry | yq r - 'server') && \
-	export PASSWORD=$$(terraform output -json registry | yq r - 'password') && \
-	export USERNAME=$$(terraform output -json registry | yq r - 'username') && \
-	envsubst < ./manifests/kots-config-azure-registry.yaml > tmp_2_config.yml
-	yq m -i tmp_config.yml tmp_2_config.yml
-
 storage ?= incluster
 registry ?= incluster
 db ?= incluster
@@ -203,30 +183,30 @@ kots-uprgade:
 	@echo "Upgrade gitpod KOTS app to latest"
 	kubectl kots upstream upgrade --kubeconfig=${KUBECONFIG} gitpod -n gitpod --deploy
 
-cleanup: destroy-gcp-externaldns destroy-gcpns destroy-aks-edns destroy-aks-issuer destroy-certmanager destroy-clusters
+cloud ?= gcp
+cleanup: destroy-$(cloud)
 
 select-workspace:
 	terraform workspace select $(TF_VAR_TEST_ID)
 
-destroy-gcp-externaldns: select-workspace
+destroy-gcp: select-workspace
 	ls ${KUBECONFIG} && terraform destroy -target=module.clouddns-externaldns -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file"
-
-destroy-certmanager: select-workspace
 	ls ${KUBECONFIG} && terraform destroy -target=module.certmanager -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file"
+	terraform destroy -target=module.k3s -var kubeconfig=${KUBECONFIG} --auto-approve
+	terraform destroy -target=module.gke -var kubeconfig=${KUBECONFIG} --auto-approve
 
-destroy-gcpns: select-workspace
-	terraform destroy -target=module.add_gcp_nameservers -var kubeconfig=${KUBECONFIG} --auto-approve
-
-destroy-edns: select-workspace
-	ls ${KUBECONFIG} && terraform destroy -target=module.externaldns -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file"
+destroy-aws: select-workspace
+	terraform destroy -target=module.aws-add-dns-record -var kubeconfig=${KUBECONFIG} --auto-approve
+	ls ${KUBECONFIG} && terraform destroy -target=module.aws-issuer -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file"
+	ls ${KUBECONFIG} && terraform destroy -target=module.aws-externaldns -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file"
+	ls ${KUBECONFIG} && terraform destroy -target=module.certmanager -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file"
+	terraform destroy -target=module.eks -var kubeconfig=${KUBECONFIG} --auto-approve
 
-destroy-aks-issuer: select-workspace
+destroy-azure: select-workspace
 	ls ${KUBECONFIG} && terraform destroy -target=module.azure-issuer -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file"
-
-destroy-clusters: select-workspace
-	terraform destroy -target=module.gke -var kubeconfig=${KUBECONFIG} --auto-approve
-	terraform destroy -target=module.eks -var kubeconfig=${KUBECONFIG} --auto-approve
-	terraform destroy -target=module.k3s -var kubeconfig=${KUBECONFIG} --auto-approve
+	terraform destroy -target=module.azure-add-dns-record -var kubeconfig=${KUBECONFIG} --auto-approve
+	ls ${KUBECONFIG} && terraform destroy -target=module.azure-externaldns -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file"
+	ls ${KUBECONFIG} && terraform destroy -target=module.certmanager -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file"
 	[ -z "$$ARM_SUBSCRIPTION_ID" ] || terraform destroy -target=module.aks -var kubeconfig=${KUBECONFIG} --auto-approve
 
 get-results:
diff --git a/install/tests/main.tf b/install/tests/main.tf
index aca038e6b96f3a..86c1e116a605da 100644
--- a/install/tests/main.tf
+++ b/install/tests/main.tf
@@ -56,8 +56,8 @@ module "aks" {
 
 module "eks" {
   source = "../infra/terraform/eks"
+  domain_name              = "${var.TEST_ID}.gitpod-self-hosted.com"
   cluster_name = var.TEST_ID
-  kubeconfig   = var.kubeconfig
 }
 
 module "certmanager" {
@@ -75,25 +75,35 @@ module "clouddns-externaldns" {
   credentials    = var.dns_sa_creds
 }
 
-variable "cloud" { default = "azure" }
+module "azure-externaldns" {
+  source       = "../infra/terraform/tools/external-dns"
+  kubeconfig   = var.kubeconfig
+  settings     = module.aks.external_dns_settings
+  domain_name  = "${var.TEST_ID}.gitpod-self-hosted.com"
+  txt_owner_id = var.TEST_ID
+}
 
-module "externaldns" {
-  source      = "../infra/terraform/tools/external-dns"
-  kubeconfig  = var.kubeconfig
-  settings    = module.aks.external_dns_settings
-  domain_name = "${var.TEST_ID}.gitpod-self-hosted.com"
-  txt_owner_id   = var.TEST_ID
-  cloud       = var.cloud
+module "aws-externaldns" {
+  source       = "../infra/terraform/tools/external-dns"
+  kubeconfig   = var.kubeconfig
+  settings     = module.eks.external_dns_settings
+  domain_name  = "${var.TEST_ID}.gitpod-self-hosted.com"
+  txt_owner_id = var.TEST_ID
 }
 
 module "azure-issuer" {
-  source = "../infra/terraform/tools/issuer/azure"
-  kubeconfig  = var.kubeconfig
+  source              = "../infra/terraform/tools/issuer/azure"
+  kubeconfig          = var.kubeconfig
   cert_manager_issuer = module.aks.cert_manager_issuer
 }
 
-module "add_gcp_nameservers" {
-  # source = "github.com/gitpod-io/gitpod//install/infra/terraform/tools/cloud-dns-ns?ref=main"
+module "aws-issuer" {
+  source              = "../infra/terraform/tools/issuer/azure"
+  kubeconfig          = var.kubeconfig
+  cert_manager_issuer = module.eks.cert_manager_issuer
+}
+
+module "azure-add-dns-record" {
   source           = "../infra/terraform/tools/cloud-dns-ns"
   credentials      = var.dns_sa_creds
   nameservers      = module.aks.domain_nameservers
@@ -101,3 +111,12 @@ module "add_gcp_nameservers" {
   managed_dns_zone = "gitpod-self-hosted-com"
   domain_name      = "${var.TEST_ID}.gitpod-self-hosted.com"
 }
+
+module "aws-add-dns-record" {
+  source           = "../infra/terraform/tools/cloud-dns-ns"
+  credentials      = var.dns_sa_creds
+  nameservers      = module.eks.domain_nameservers
+  dns_project      = "dns-for-playgrounds"
+  managed_dns_zone = "gitpod-self-hosted-com"
+  domain_name      = "${var.TEST_ID}.gitpod-self-hosted.com"
+}