From b9828eae671d673baaef5851ddc83f63fd681cd3 Mon Sep 17 00:00:00 2001
From: Nandaja Varma <nandaja.varma@gmail.com>
Date: Thu, 16 Jun 2022 19:50:54 +0000
Subject: [PATCH] [installer] Add EKS installer test

---
 .werft/eks-installer-tests.yaml               |  85 +++++++
 .werft/installer-tests.ts                     |  22 ++
 install/infra/terraform/eks/kubernetes.tf     | 232 ++++++++++++++++++
 install/infra/terraform/eks/network.tf        | 103 ++++++++
 install/infra/terraform/eks/variables.tf      |  24 ++
 install/tests/Makefile                        |  22 +-
 .../install/infra/terraform/eks/providers.tf  |   0
 install/tests/main.tf                         |   7 +-
 8 files changed, 485 insertions(+), 10 deletions(-)
 create mode 100644 .werft/eks-installer-tests.yaml
 create mode 100644 install/infra/terraform/eks/kubernetes.tf
 create mode 100644 install/infra/terraform/eks/network.tf
 create mode 100644 install/infra/terraform/eks/variables.tf
 create mode 100644 install/tests/install/infra/terraform/eks/providers.tf

diff --git a/.werft/eks-installer-tests.yaml b/.werft/eks-installer-tests.yaml
new file mode 100644
index 00000000000000..96c376ebf465d2
--- /dev/null
+++ b/.werft/eks-installer-tests.yaml
@@ -0,0 +1,85 @@
+# debug using `werft run github -f -s .werft/installer-tests.ts -j .werft/eks-installer-tests.yaml -a debug=true`
+pod:
+  serviceAccount: werft
+  affinity:
+  nodeAffinity:
+    requiredDuringSchedulingIgnoredDuringExecution:
+      nodeSelectorTerms:
+      - matchExpressions:
+        - key: dev/workload
+          operator: In
+          values:
+          - "builds"
+  securityContext:
+    runAsUser: 0
+  volumes:
+  - name: sh-playground-sa-perm
+    secret:
+      secretName: sh-playground-sa-perm
+  - name: sh-playground-dns-perm
+    secret:
+      secretName: sh-playground-dns-perm
+  - name: sh-aks-perm
+    secret:
+      secretName: aks-credentials
+  containers:
+  - name: nightly-test
+    image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:cw-werft-cred.0
+    workingDir: /workspace
+    imagePullPolicy: Always
+    volumeMounts:
+    - name: sh-playground-sa-perm
+      mountPath: /mnt/secrets/sh-playground-sa-perm
+    - name: sh-playground-dns-perm # this sa is used for the DNS management
+      mountPath: /mnt/secrets/sh-playground-dns-perm
+    env:
+    - name: AWS_ACCESS_KEY_ID
+      valueFrom:
+        secretKeyRef:
+          name: aws-credentials
+          key: aws-access-key
+    - name: AWS_SECRET_ACCESS_KEY
+      valueFrom:
+        secretKeyRef:
+          name: aws-credentials
+          key: aws-secret-key
+    - name: AWS_REGION
+      valueFrom:
+        secretKeyRef:
+          name: aws-credentials
+          key: aws-region
+    - name: WERFT_HOST
+      value: "werft.werft.svc.cluster.local:7777"
+    - name: GOOGLE_APPLICATION_CREDENTIALS
+      value: "/mnt/secrets/sh-playground-sa-perm/sh-sa.json"
+    - name: WERFT_K8S_NAMESPACE
+      value: "werft"
+    - name: WERFT_K8S_LABEL
+      value: "component=werft"
+    - name: TF_VAR_sa_creds
+      value: "/mnt/secrets/sh-playground-sa-perm/sh-sa.json"
+    - name: TF_VAR_dns_sa_creds
+      value: "/mnt/secrets/sh-playground-dns-perm/sh-dns-sa.json"
+    - name: NODENAME
+      valueFrom:
+        fieldRef:
+          fieldPath: spec.nodeName
+    command:
+      - bash
+      - -c
+      - |
+        sleep 1
+        set -Eeuo pipefail
+
+        sudo chown -R gitpod:gitpod /workspace
+        sudo apt update && apt install gettext-base
+
+        export TF_VAR_TEST_ID=$(echo $RANDOM | md5sum | head -c 5; echo)
+
+        (cd .werft && yarn install && mv node_modules ..) | werft log slice prep
+        printf '{{ toJson . }}' > context.json
+
+        npx ts-node .werft/installer-tests.ts "STANDARD_EKS_TEST"
+# The bit below makes this a cron job
+# plugins:
+#   cron: "15 3 * * *"
diff --git a/.werft/installer-tests.ts b/.werft/installer-tests.ts
index 6f7e00794277a3..9597eade921264 100644
--- a/.werft/installer-tests.ts
+++ b/.werft/installer-tests.ts
@@ -39,6 +39,11 @@ const INFRA_PHASES: { [name: string]: InfraConfig } = {
         makeTarget: "aks-standard-cluster",
         description: "Creating an aks cluster(azure)",
     },
+    STANDARD_EKS_CLUSTER: {
+        phase: "create-std-eks-cluster",
+        makeTarget: "eks-standard-cluster",
+        description: "Creating a EKS cluster with 1 nodepool each for workspace and server",
+    },
     CERT_MANAGER: {
         phase: "setup-cert-manager",
         makeTarget: "cert-manager",
@@ -174,6 +179,23 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = {
             "DESTROY",
         ],
     },
+    STANDARD_EKS_TEST: {
+        DESCRIPTION: "Create an EKS cluster",
+        PHASES: [
+            "STANDARD_EKS_CLUSTER",
+            "CERT_MANAGER",
+            // TODO phases are:
+            // 1) register domains in AWS, associate with route53
+            // 2) add the associated ns record to gcp(since we use gitpod-self-hsoted.com domain)
+            // 3) create cluster issuer with route53 as solver
+            "GENERATE_KOTS_CONFIG",
+            "INSTALL_GITPOD",
+            // "CHECK_INSTALLATION",
+            // "RUN_INTEGRATION_TESTS",
+            "RESULTS",
+            "DESTROY",
+        ],
+    },
     STANDARD_K3S_PREVIEW: {
         DESCRIPTION: "Create a SH Gitpod preview environment on a K3s cluster, created on a GCP instance",
         PHASES: [
diff --git a/install/infra/terraform/eks/kubernetes.tf b/install/infra/terraform/eks/kubernetes.tf
new file mode 100644
index 00000000000000..a38bba1b1e4b53
--- /dev/null
+++ b/install/infra/terraform/eks/kubernetes.tf
@@ -0,0 +1,232 @@
+terraform {
+  required_providers {
+    kubectl = {
+      source  = "gavinbunney/kubectl"
+      version = ">= 1.7.0"
+    }
+    aws = {
+        version = " ~> 3.0"
+        source = "registry.terraform.io/hashicorp/aws"
+    }
+  }
+}
+
+resource "aws_iam_role" "eks_cluster" {
+  depends_on = [data.aws_subnet_ids.subnet_ids]
+  name = "iam-${var.cluster_name}"
+
+  assume_role_policy = <<POLICY
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": {
+        "Service": "eks.amazonaws.com"
+      },
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}
+POLICY
+}
+
+resource "aws_iam_role_policy_attachment" "AmazonEKSClusterPolicy" {
+  policy_arn = "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy"
+  role       = aws_iam_role.eks_cluster.name
+}
+
+resource "aws_iam_role_policy_attachment" "AmazonEKSServicePolicy" {
+  policy_arn = "arn:aws:iam::aws:policy/AmazonEKSServicePolicy"
+  role       = aws_iam_role.eks_cluster.name
+}
+
+resource "aws_eks_cluster" "aws_eks" {
+  name     = var.cluster_name
+  role_arn = aws_iam_role.eks_cluster.arn
+
+  vpc_config {
+    subnet_ids = data.aws_subnet_ids.subnet_ids.ids
+  }
+
+  tags = {
+    Name = "EKS_tuto"
+  }
+
+  depends_on = [
+    aws_iam_role.eks_cluster,
+ ]
+}
+
+data "aws_eks_cluster" "cluster" {
+  depends_on = [
+    aws_eks_cluster.aws_eks,
+  ]
+  name = resource.aws_eks_cluster.aws_eks.id
+}
+
+data "aws_eks_cluster_auth" "cluster" {
+  depends_on = [
+    aws_eks_cluster.aws_eks,
+  ]
+  name = resource.aws_eks_cluster.aws_eks.id
+}
+
+
+resource "aws_iam_role" "eks_nodes" {
+  name = "iam-ng-${var.cluster_name}"
+
+  assume_role_policy = <<POLICY
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": {
+        "Service": "ec2.amazonaws.com"
+      },
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}
+POLICY
+}
+
+resource "aws_iam_role_policy_attachment" "AmazonEKSWorkerNodePolicy" {
+  policy_arn = "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy"
+  role       = aws_iam_role.eks_nodes.name
+}
+
+resource "aws_iam_role_policy_attachment" "AmazonEKS_CNI_Policy" {
+  policy_arn = "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy"
+  role       = aws_iam_role.eks_nodes.name
+}
+
+locals {
+  map_roles = <<ROLES
+- rolearn: ${aws_iam_role.eks_nodes.arn}
+  username: system:node:{{EC2PrivateDNSName}}
+  groups:
+    - system:bootstrappers
+    - system:nodes
+ROLES
+}
+
+resource "aws_iam_role_policy_attachment" "AmazonSSMManagedInstanceCore" {
+  policy_arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"
+  role       = aws_iam_role.eks_nodes.name
+}
+
+resource "aws_iam_role_policy_attachment" "AmazonEC2ContainerRegistryReadOnly" {
+  policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly"
+  role       = aws_iam_role.eks_nodes.name
+}
+
+resource "aws_iam_role_policy_attachment" "EC2InstanceProfileForImageBuilderECRContainerBuilds" {
+  policy_arn = "arn:aws:iam::aws:policy/EC2InstanceProfileForImageBuilderECRContainerBuilds"
+  role       = aws_iam_role.eks_nodes.name
+}
+
+resource "aws_launch_template" "eks" {
+  name = "${var.cluster_name}-template"
+  update_default_version = true
+  block_device_mappings {
+    device_name = "/dev/sda1"
+    ebs {
+      volume_size = 100
+    }
+  }
+  credit_specification {
+    cpu_credits = "standard"
+  }
+  ebs_optimized                        = true
+  # AMI generated with packer (is private)
+  image_id                             = "ami-0f08b4b1a4fd3ebe3"
+  network_interfaces {
+    associate_public_ip_address = false
+  }
+}
+
+resource "aws_eks_node_group" "workspace" {
+  cluster_name    = aws_eks_cluster.aws_eks.name
+  node_group_name = "ngw-${var.cluster_name}"
+  node_role_arn   = aws_iam_role.eks_nodes.arn
+  subnet_ids      = data.aws_subnet_ids.subnet_ids.ids
+  instance_types  = ["m6i.2xlarge"]
+  labels = {
+      "gitpod.io/workload_workspace_services" = true
+      "gitpod.io/workload_workspace_regular"  = true
+      "gitpod.io/workload_workspace_headless" = true
+  }
+
+  scaling_config {
+    desired_size = 1
+    max_size     = 10
+    min_size     = 1
+  }
+
+  # Ensure that IAM Role permissions are created before and deleted after EKS Node Group handling.
+  # Otherwise, EKS will not be able to properly delete EC2 Instances and Elastic Network Interfaces.
+  depends_on = [
+    resource.aws_iam_role_policy_attachment.AmazonSSMManagedInstanceCore,
+    resource.aws_iam_role_policy_attachment.AmazonEC2ContainerRegistryReadOnly,
+    resource.aws_iam_role_policy_attachment.EC2InstanceProfileForImageBuilderECRContainerBuilds,
+  ]
+
+  launch_template {
+    id = resource.aws_launch_template.eks.id
+    version = aws_launch_template.eks.latest_version
+  }
+}
+
+resource "aws_eks_node_group" "services" {
+  cluster_name    = aws_eks_cluster.aws_eks.name
+  node_group_name = "ngs-${var.cluster_name}"
+  node_role_arn   = aws_iam_role.eks_nodes.arn
+  subnet_ids      = data.aws_subnet_ids.subnet_ids.ids
+  instance_types  = ["m6i.xlarge"]
+  labels = {
+      "gitpod.io/workload_meta" = true
+      "gitpod.io/workload_ide"  = true
+  }
+
+  scaling_config {
+    desired_size = 1
+    max_size     = 10
+    min_size     = 1
+  }
+
+  # Ensure that IAM Role permissions are created before and deleted after EKS Node Group handling.
+  # Otherwise, EKS will not be able to properly delete EC2 Instances and Elastic Network Interfaces.
+  depends_on = [
+    resource.aws_iam_role_policy_attachment.AmazonSSMManagedInstanceCore,
+    resource.aws_iam_role_policy_attachment.AmazonEC2ContainerRegistryReadOnly,
+    resource.aws_iam_role_policy_attachment.EC2InstanceProfileForImageBuilderECRContainerBuilds,
+  ]
+
+  launch_template {
+    id = resource.aws_launch_template.eks.id
+    version = aws_launch_template.eks.latest_version
+  }
+}
+
+provider "kubectl" {
+  host                   = resource.aws_eks_cluster.aws_eks.endpoint
+  cluster_ca_certificate = base64decode(resource.aws_eks_cluster.aws_eks.certificate_authority[0].data)
+  token                  = data.aws_eks_cluster_auth.cluster.token
+  config_path            = var.kubeconfig
+}
+
+output host {
+  value = resource.aws_eks_cluster.aws_eks.endpoint
+}
+
+output ca {
+  sensitive = true
+  value = base64decode(resource.aws_eks_cluster.aws_eks.certificate_authority[0].data)
+}
+
+output token {
+  sensitive = true
+  value = data.aws_eks_cluster_auth.cluster.token
+}
diff --git a/install/infra/terraform/eks/network.tf b/install/infra/terraform/eks/network.tf
new file mode 100644
index 00000000000000..f9a616abda997d
--- /dev/null
+++ b/install/infra/terraform/eks/network.tf
@@ -0,0 +1,103 @@
+# get all available AZs in our region
+data "aws_availability_zones" "available_azs" {
+  state = "available"
+}
+
+# reserve Elastic IP to be used in our NAT gateway
+resource "aws_eip" "nat_gw_elastic_ip" {
+  vpc = true
+
+  tags = {
+    Name = "${var.cluster_name}-nat-eip"
+  }
+}
+
+# create VPC using the official AWS module
+module "vpc" {
+  source  = "terraform-aws-modules/vpc/aws"
+  version = "3.14.0"
+
+  name = "${var.cluster_name}-vpc"
+  cidr = var.main_network_block
+  azs  = data.aws_availability_zones.available_azs.names
+
+  private_subnets = [
+    # this loop will create a one-line list as ["10.0.0.0/20", "10.0.16.0/20", "10.0.32.0/20", ...]
+    # with a length depending on how many Zones are available
+    for zone_id in data.aws_availability_zones.available_azs.zone_ids :
+    cidrsubnet(var.main_network_block, var.subnet_prefix_extension, tonumber(substr(zone_id, length(zone_id) - 1, 1)) - 1)
+  ]
+
+  public_subnets = [
+    # this loop will create a one-line list as ["10.0.128.0/20", "10.0.144.0/20", "10.0.160.0/20", ...]
+    # with a length depending on how many Zones are available
+    # there is a zone Offset variable, to make sure no collisions are present with private subnet blocks
+    for zone_id in data.aws_availability_zones.available_azs.zone_ids :
+    cidrsubnet(var.main_network_block, var.subnet_prefix_extension, tonumber(substr(zone_id, length(zone_id) - 1, 1)) + var.zone_offset - 1)
+  ]
+
+  # enable single NAT Gateway to save some money
+  # WARNING: this could create a single point of failure, since we are creating a NAT Gateway in one AZ only
+  # feel free to change these options if you need to ensure full Availability without the need of running 'terraform apply'
+  # reference: https://registry.terraform.io/modules/terraform-aws-modules/vpc/aws/2.44.0#nat-gateway-scenarios
+  enable_nat_gateway     = true
+  single_nat_gateway     = true
+  one_nat_gateway_per_az = false
+  enable_dns_hostnames   = true
+  reuse_nat_ips          = true
+  external_nat_ip_ids    = [aws_eip.nat_gw_elastic_ip.id]
+
+  # add VPC/Subnet tags required by EKS
+  tags = {
+    "kubernetes.io/cluster/${var.cluster_name}" = "shared"
+  }
+  public_subnet_tags = {
+    "kubernetes.io/cluster/${var.cluster_name}" = "shared"
+    "kubernetes.io/role/elb"                    = "1"
+  }
+  private_subnet_tags = {
+    "kubernetes.io/cluster/${var.cluster_name}" = "shared"
+    "kubernetes.io/role/internal-elb"           = "1"
+  }
+}
+
+# create security group to be used later by the ingress ALB
+resource "aws_security_group" "alb" {
+  name   = "${var.cluster_name}-alb"
+  vpc_id = module.vpc.vpc_id
+
+  ingress {
+    description      = "http"
+    from_port        = 80
+    to_port          = 80
+    protocol         = "tcp"
+    cidr_blocks      = ["0.0.0.0/0"]
+    ipv6_cidr_blocks = ["::/0"]
+  }
+
+  ingress {
+    description      = "https"
+    from_port        = 443
+    to_port          = 443
+    protocol         = "tcp"
+    cidr_blocks      = ["0.0.0.0/0"]
+    ipv6_cidr_blocks = ["::/0"]
+  }
+
+  egress {
+    from_port        = 0
+    to_port          = 0
+    protocol         = "-1"
+    cidr_blocks      = ["0.0.0.0/0"]
+    ipv6_cidr_blocks = ["::/0"]
+  }
+
+  tags = {
+    "Name" = "${var.cluster_name}-alb"
+  }
+}
+
+data "aws_subnet_ids" "subnet_ids" {
+  depends_on = [module.vpc]
+  vpc_id     = module.vpc.vpc_id
+}
diff --git a/install/infra/terraform/eks/variables.tf b/install/infra/terraform/eks/variables.tf
new file mode 100644
index 00000000000000..c86eefa03d7639
--- /dev/null
+++ b/install/infra/terraform/eks/variables.tf
@@ -0,0 +1,24 @@
+variable "cluster_name" {
+  type        = string
+  description = "EKS cluster name."
+}
+variable "kubeconfig" {
+  type        = string
+  description = "Path to the kubeconfig file"
+  default     = "kubeconfig"
+}
+variable "main_network_block" {
+  type        = string
+  description = "Base CIDR block to be used in our VPC."
+  default     = "10.0.0.0/16"
+}
+variable "subnet_prefix_extension" {
+  type        = number
+  description = "CIDR block bits extension to calculate CIDR blocks of each subnetwork."
+  default     = 8
+}
+variable "zone_offset" {
+  type        = number
+  description = "CIDR block bits extension offset to calculate Public subnets, avoiding collisions with Private subnets."
+  default     = 21
+}
diff --git a/install/tests/Makefile b/install/tests/Makefile
index 2aad355945f8df..2cab841f4ebd0b 100644
--- a/install/tests/Makefile
+++ b/install/tests/Makefile
@@ -28,6 +28,13 @@ gke-standard-cluster:
 	terraform workspace new $(TF_VAR_TEST_ID) || terraform workspace select $(TF_VAR_TEST_ID) && \
 	terraform apply -target=module.gke -var kubeconfig=${KUBECONFIG} --auto-approve
 
+.PHONY:
+## eks-standard-cluster: Creates an EKS cluster
+eks-standard-cluster:
+	terraform init --upgrade && \
+	terraform workspace new $(TF_VAR_TEST_ID) || terraform workspace select $(TF_VAR_TEST_ID) && \
+	terraform apply -target=module.eks -var kubeconfig=${KUBECONFIG} --auto-approve
+
 .PHONY:
 ## aks-standard-cluster: Creates an AKS cluster
 aks-standard-cluster:
@@ -129,7 +136,7 @@ kots-uprgade:
 	@echo "Upgrade gitpod KOTS app to latest"
 	kubectl kots upstream upgrade --kubeconfig=${KUBECONFIG} gitpod -n gitpod --deploy
 
-cleanup: destroy-gcp-externaldns destroy-gcpns destroy-aks-edns destroy-aks-issuer destroy-certmanager destroy-k3s destroy-gke destroy-aks
+cleanup: destroy-gcp-externaldns destroy-gcpns destroy-aks-edns destroy-aks-issuer destroy-certmanager destroy-clusters
 
 select-workspace:
 	terraform workspace select $(TF_VAR_TEST_ID)
@@ -144,19 +151,16 @@ destroy-gcpns: select-workspace
 	ls ${KUBECONFIG} && terraform destroy -target=module.add_gcp_nameservers -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file"
 
 destroy-aks-edns: select-workspace
-	ls ${KUBECONFIG} && terraform destroy -target=module.azure-externaldns -var kubeconfig=${KUBECONFIG} --auto-approve
+	ls ${KUBECONFIG} && terraform destroy -target=module.azure-externaldns -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file"
 
 destroy-aks-issuer: select-workspace
-	ls ${KUBECONFIG} && terraform destroy -target=module.azure-issuer -var kubeconfig=${KUBECONFIG} --auto-approve
+	ls ${KUBECONFIG} && terraform destroy -target=module.azure-issuer -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file"
 
-destroy-gke: select-workspace
+destroy-clusters: select-workspace
 	terraform destroy -target=module.gke -var kubeconfig=${KUBECONFIG} --auto-approve
-
-destroy-k3s: select-workspace
+	terraform destroy -target=module.eks -var kubeconfig=${KUBECONFIG} --auto-approve
 	terraform destroy -target=module.k3s -var kubeconfig=${KUBECONFIG} --auto-approve
-
-destroy-aks: select-workspace
-	terraform destroy -target=module.aks -var kubeconfig=${KUBECONFIG} --auto-approve
+	[ -z "$$ARM_SUBSCRIPTION_ID" ] || terraform destroy -target=module.aks -var kubeconfig=${KUBECONFIG} --auto-approve
 
 get-results:
 	@echo "If you have gotten this far, it means your setup succeeded"
diff --git a/install/tests/install/infra/terraform/eks/providers.tf b/install/tests/install/infra/terraform/eks/providers.tf
new file mode 100644
index 00000000000000..e69de29bb2d1d6
diff --git a/install/tests/main.tf b/install/tests/main.tf
index bc2b5e08a550bf..a3a2a496616110 100644
--- a/install/tests/main.tf
+++ b/install/tests/main.tf
@@ -52,6 +52,12 @@ module "aks" {
   workspace_name           = var.TEST_ID
 }
 
+module "eks" {
+  source = "../infra/terraform/eks"
+  cluster_name = var.TEST_ID
+  kubeconfig   = var.kubeconfig
+}
+
 module "certmanager" {
   # source = "github.com/gitpod-io/gitpod//install/infra/terraform/tools/cert-manager?ref=main"
   source = "../infra/terraform/tools/cert-manager"
@@ -65,7 +71,6 @@ module "externaldns" {
   source = "../infra/terraform/tools/external-dns"
   kubeconfig     = var.kubeconfig
   credentials    = var.dns_sa_creds
-  txt_owner_id   = var.TEST_ID
 }
 
 module "azure-externaldns" {