From 8bbd0e79babb1039adac1fe82443d83e46485852 Mon Sep 17 00:00:00 2001 From: Nandaja Varma Date: Thu, 16 Jun 2022 19:50:54 +0000 Subject: [PATCH] [installer] Add EKS installer test --- .werft/eks-installer-tests.yaml | 88 +++++++++++ .werft/installer-tests.ts | 70 ++++++--- install/infra/terraform/aks/output.tf | 34 ++++- install/infra/terraform/eks/database.tf | 40 +++++ install/infra/terraform/eks/dns.tf | 114 ++++++++++++++ install/infra/terraform/eks/kubernetes.tf | 142 ++++++++++++++++++ install/infra/terraform/eks/output.tf | 72 +++++++++ install/infra/terraform/eks/providers.tf | 16 ++ install/infra/terraform/eks/registry.tf | 12 ++ install/infra/terraform/eks/storage.tf | 37 +++++ install/infra/terraform/eks/variables.tf | 70 +++++++++ .../README.md | 0 .../tools/cloud-dns-external-dns/main.tf | 74 +++++++++ .../variables.tf | 0 .../terraform/tools/external-dns/main.tf | 76 +++------- .../tools/issuer/{azure => }/main.tf | 14 +- .../tools/issuer/{azure => }/variables.tf | 8 + install/tests/Makefile | 123 +++++++++------ .../install/infra/terraform/eks/providers.tf | 0 install/tests/main.tf | 87 ++++++++--- .../tests/manifests/kots-config-aws-db.yaml | 16 ++ .../manifests/kots-config-aws-registry.yaml | 16 ++ .../manifests/kots-config-aws-storage.yaml | 22 +++ install/tests/output.tf | 27 +++- 24 files changed, 999 insertions(+), 159 deletions(-) create mode 100644 .werft/eks-installer-tests.yaml create mode 100644 install/infra/terraform/eks/database.tf create mode 100644 install/infra/terraform/eks/dns.tf create mode 100644 install/infra/terraform/eks/kubernetes.tf create mode 100644 install/infra/terraform/eks/output.tf create mode 100644 install/infra/terraform/eks/providers.tf create mode 100644 install/infra/terraform/eks/registry.tf create mode 100644 install/infra/terraform/eks/storage.tf create mode 100644 install/infra/terraform/eks/variables.tf rename install/infra/terraform/tools/{external-dns => cloud-dns-external-dns}/README.md (100%) create mode 100644 install/infra/terraform/tools/cloud-dns-external-dns/main.tf rename install/infra/terraform/tools/{external-dns => cloud-dns-external-dns}/variables.tf (100%) rename install/infra/terraform/tools/issuer/{azure => }/main.tf (67%) rename install/infra/terraform/tools/issuer/{azure => }/variables.tf (62%) create mode 100644 install/tests/install/infra/terraform/eks/providers.tf create mode 100644 install/tests/manifests/kots-config-aws-db.yaml create mode 100644 install/tests/manifests/kots-config-aws-registry.yaml create mode 100644 install/tests/manifests/kots-config-aws-storage.yaml diff --git a/.werft/eks-installer-tests.yaml b/.werft/eks-installer-tests.yaml new file mode 100644 index 00000000000000..eaf803b836f308 --- /dev/null +++ b/.werft/eks-installer-tests.yaml @@ -0,0 +1,88 @@ +# debug using `werft run github -f -s .werft/installer-tests.ts -j .werft/eks-installer-tests.yaml -a debug=true` +pod: + serviceAccount: werft + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: dev/workload + operator: In + values: + - "builds" + securityContext: + runAsUser: 0 + volumes: + - name: sh-playground-sa-perm + secret: + secretName: sh-playground-sa-perm + - name: sh-playground-dns-perm + secret: + secretName: sh-playground-dns-perm + - name: sh-aks-perm + secret: + secretName: aks-credentials + containers: + - name: nightly-test + image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:cw-werft-cred.0 + workingDir: /workspace + imagePullPolicy: Always + volumeMounts: + - name: sh-playground-sa-perm + mountPath: /mnt/secrets/sh-playground-sa-perm + - name: sh-playground-dns-perm # this sa is used for the DNS management + mountPath: /mnt/secrets/sh-playground-dns-perm + env: + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: aws-credentials + key: aws-access-key + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: aws-credentials + key: aws-secret-key + - name: AWS_REGION + valueFrom: + secretKeyRef: + name: aws-credentials + key: aws-region + - name: WERFT_HOST + value: "werft.werft.svc.cluster.local:7777" + - name: GOOGLE_APPLICATION_CREDENTIALS + value: "/mnt/secrets/sh-playground-sa-perm/sh-sa.json" + - name: WERFT_K8S_NAMESPACE + value: "werft" + - name: WERFT_K8S_LABEL + value: "component=werft" + - name: TF_VAR_sa_creds + value: "/mnt/secrets/sh-playground-sa-perm/sh-sa.json" + - name: TF_VAR_dns_sa_creds + value: "/mnt/secrets/sh-playground-dns-perm/sh-dns-sa.json" + - name: NODENAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + command: + - bash + - -c + - | + sleep 1 + set -Eeuo pipefail + + sudo chown -R gitpod:gitpod /workspace + sudo apt update && apt install gettext-base + + export TF_VAR_TEST_ID="$(echo $RANDOM | md5sum | head -c 5; echo)" + + (cd .werft && yarn install && mv node_modules ..) | werft log slice prep + printf '{{ toJson . }}' > context.json + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + unzip awscliv2.zip + sudo ./aws/install + + npx ts-node .werft/installer-tests.ts "STANDARD_EKS_TEST" +# The bit below makes this a cron job +# plugins: +# cron: "15 3 * * *" diff --git a/.werft/installer-tests.ts b/.werft/installer-tests.ts index 35290735f948f2..8e9bf01d0f4be3 100644 --- a/.werft/installer-tests.ts +++ b/.werft/installer-tests.ts @@ -35,6 +35,7 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = { "STANDARD_GKE_CLUSTER", "CERT_MANAGER", "GCP_MANAGED_DNS", + "CLUSTER_ISSUER", "GENERATE_KOTS_CONFIG", "INSTALL_GITPOD", "CHECK_INSTALLATION", @@ -49,6 +50,7 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = { PHASES: [ "STANDARD_GKE_CLUSTER", "CERT_MANAGER", + "CLUSTER_ISSUER", "GCP_MANAGED_DNS", "GENERATE_KOTS_CONFIG", "INSTALL_GITPOD", @@ -66,11 +68,12 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = { PHASES: [ "STANDARD_K3S_CLUSTER_ON_GCP", "CERT_MANAGER", + "CLUSTER_ISSUER", "GENERATE_KOTS_CONFIG", "INSTALL_GITPOD", + "RESULTS", "CHECK_INSTALLATION", "RUN_INTEGRATION_TESTS", - "RESULTS", "DESTROY", ], }, @@ -80,6 +83,7 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = { PHASES: [ "STANDARD_K3S_CLUSTER_ON_GCP", "CERT_MANAGER", + "CLUSTER_ISSUER", "GENERATE_KOTS_CONFIG", "INSTALL_GITPOD", "CHECK_INSTALLATION", @@ -92,8 +96,8 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = { PHASES: [ "STANDARD_AKS_CLUSTER", "CERT_MANAGER", - "AZURE_ISSUER", - "AZURE_EXTERNALDNS", + "CLUSTER_ISSUER", + "EXTERNALDNS", "ADD_NS_RECORD", "GENERATE_KOTS_CONFIG", "INSTALL_GITPOD", @@ -103,6 +107,23 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = { "DESTROY", ], }, + STANDARD_EKS_TEST: { + CLOUD: "aws", + DESCRIPTION: "Create an EKS cluster", + PHASES: [ + "STANDARD_EKS_CLUSTER", + "CERT_MANAGER", + "EXTERNALDNS", + "CLUSTER_ISSUER", + "ADD_NS_RECORD", + "GENERATE_KOTS_CONFIG", + "RESULTS", + "INSTALL_GITPOD", + "CHECK_INSTALLATION", + "RUN_INTEGRATION_TESTS", + "DESTROY", + ], + }, }; const config: TestConfig = TEST_CONFIGURATIONS[testConfig]; @@ -128,6 +149,11 @@ const INFRA_PHASES: { [name: string]: InfraConfig } = { makeTarget: "aks-standard-cluster", description: "Creating an aks cluster(azure)", }, + STANDARD_EKS_CLUSTER: { + phase: "create-std-eks-cluster", + makeTarget: "eks-standard-cluster", + description: "Creating a EKS cluster with 1 nodepool each for workspace and server", + }, CERT_MANAGER: { phase: "setup-cert-manager", makeTarget: "cert-manager", @@ -146,19 +172,19 @@ const INFRA_PHASES: { [name: string]: InfraConfig } = { )} db=${randomize("db", cloud)}`, description: `Generate KOTS Config file`, }, - AZURE_ISSUER: { - phase: "setup-azure-cluster-issuer", - makeTarget: "azure-issuer", - description: "Deploys ClusterIssuer for azure", + CLUSTER_ISSUER: { + phase: "setup-cluster-issuer", + makeTarget: `cluster-issuer cloud=${cloud}`, + description: `Deploys ClusterIssuer for ${cloud}`, }, - AZURE_EXTERNALDNS: { - phase: "azure-external-dns", - makeTarget: "azure-external-dns", - description: "Deploys external-dns with azure provider", + EXTERNALDNS: { + phase: "external-dns", + makeTarget: `external-dns cloud=${cloud}`, + description: `Deploys external-dns with ${cloud} provider`, }, ADD_NS_RECORD: { phase: "add-ns-record", - makeTarget: "add-ns-record", + makeTarget: `add-ns-record cloud=${cloud}`, description: "Adds NS record for subdomain under gitpod-self-hosted.com", }, INSTALL_GITPOD_IGNORE_PREFLIGHTS: { @@ -189,7 +215,7 @@ const INFRA_PHASES: { [name: string]: InfraConfig } = { }, DESTROY: { phase: "destroy", - makeTarget: "cleanup", + makeTarget: `cleanup cloud=${cloud}`, description: "Destroy the created infrastucture", }, RESULTS: { @@ -224,23 +250,24 @@ export async function installerTests(config: TestConfig) { } function callMakeTargets(phase: string, description: string, makeTarget: string) { - werft.phase(phase, `${description}`); - werft.log(phase, `calling ${makeTarget}`); + werft.phase(phase, description); const response = exec(`make -C ${makefilePath} ${makeTarget}`, { - slice: "call-make-target", + slice: phase, dontCheckRc: true, }); if (response.code) { console.error(`Error: ${response.stderr}`); werft.fail(phase, "Operation failed"); - } else { - werft.log(phase, response.stdout.toString()); - werft.done(phase); + return response.code; } + werft.log(phase, response.stdout.toString()); + werft.done(phase); + return response.code; + } function randomize(resource: string, platform: string): string { @@ -254,7 +281,10 @@ function cleanup() { const phase = "destroy-infrastructure"; werft.phase(phase, "Destroying all the created resources"); - const response = exec(`make -C ${makefilePath} cleanup`, { slice: "run-terrafrom-destroy", dontCheckRc: true }); + const response = exec(`make -C ${makefilePath} cleanup cloud=${cloud}`, { + slice: "run-terrafrom-destroy", + dontCheckRc: true, + }); // if the destroy command fail, we check if any resources are pending to be removed // if nothing is yet to be cleaned, we return with success diff --git a/install/infra/terraform/aks/output.tf b/install/infra/terraform/aks/output.tf index 0ea0d4bf2f8b0e..83c60ee2205c81 100644 --- a/install/infra/terraform/aks/output.tf +++ b/install/infra/terraform/aks/output.tf @@ -36,14 +36,32 @@ output "external_dns_secrets" { } output "external_dns_settings" { - value = { - provider = "azure" - "azure.resourceGroup" = azurerm_resource_group.gitpod.name - "azure.subscriptionId" = data.azurerm_client_config.current.subscription_id - "azure.tenantId" = data.azurerm_client_config.current.tenant_id - "azure.useManagedIdentityExtension" = true - "azure.userAssignedIdentityID" = azurerm_kubernetes_cluster.k8s.kubelet_identity.0.client_id - } + value = [ + { + "name": "provider", + "value": "azure" + }, + { + "name": "azure.resourceGroup", + "value": azurerm_resource_group.gitpod.name, + }, + { + "name": "azure.subscriptionId", + "value": data.azurerm_client_config.current.subscription_id, + }, + { + "name": "azure.tenantId", + "value": data.azurerm_client_config.current.tenant_id, + }, + { + "name": "azure.useManagedIdentityExtension", + "value": true + }, + { + "name": "azure.userAssignedIdentityID", + "value": azurerm_kubernetes_cluster.k8s.kubelet_identity.0.client_id + }, + ] } output "k8s_connection" { diff --git a/install/infra/terraform/eks/database.tf b/install/infra/terraform/eks/database.tf new file mode 100644 index 00000000000000..4d759db54906c9 --- /dev/null +++ b/install/infra/terraform/eks/database.tf @@ -0,0 +1,40 @@ +resource "aws_db_subnet_group" "gitpod_subnets" { + name = "db-sg-${var.cluster_name}" + subnet_ids = [module.vpc.public_subnets[2], module.vpc.public_subnets[3]] +} + +resource "aws_security_group" "rdssg" { + name = "dh-sg-${var.cluster_name}" + vpc_id = module.vpc.vpc_id + + ingress { + from_port = 0 + to_port = 3306 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } +} + +resource "aws_db_instance" "gitpod" { + allocated_storage = 10 + max_allocated_storage = 100 + engine = "mysql" + engine_version = "5.7" + instance_class = "db.t3.micro" + vpc_security_group_ids = [ aws_security_group.rdssg.id ] + identifier = "db-${var.cluster_name}" + name = "gitpod" + username = "gitpod" + password = "gitpod-qwat" + parameter_group_name = "default.mysql5.7" + db_subnet_group_name = aws_db_subnet_group.gitpod_subnets.name + skip_final_snapshot = true + publicly_accessible = true +} diff --git a/install/infra/terraform/eks/dns.tf b/install/infra/terraform/eks/dns.tf new file mode 100644 index 00000000000000..1c1c62e8a39626 --- /dev/null +++ b/install/infra/terraform/eks/dns.tf @@ -0,0 +1,114 @@ +resource "aws_route53_zone" "gitpod" { + force_destroy = true + name = var.domain_name + + tags = { + Environment = "test" + } +} + +# This creates an SSL certificate +resource "aws_acm_certificate" "gitpod" { + domain_name = var.domain_name + subject_alternative_names = ["*.ws.${var.domain_name}", "ws.${var.domain_name}", "*.${var.domain_name}"] + validation_method = "DNS" +} + +# This is a DNS record for the ACM certificate validation to prove we own the domain +# +# This example, we make an assumption that the certificate is for a single domain name so can just use the first value of the +# domain_validation_options. It allows the terraform to apply without having to be targeted. +# This is somewhat less complex than the example at https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/acm_certificate_validation +# - that above example, won't apply without targeting + +resource "aws_route53_record" "cert_validation" { + allow_overwrite = true + name = tolist(aws_acm_certificate.gitpod.domain_validation_options)[0].resource_record_name + type = tolist(aws_acm_certificate.gitpod.domain_validation_options)[0].resource_record_type + zone_id = "${resource.aws_route53_zone.gitpod.zone_id}" + records = [tolist(aws_acm_certificate.gitpod.domain_validation_options)[0].resource_record_value] + ttl = 60 +} + +resource "aws_route53_record" "cert_validation_1" { + allow_overwrite = true + name = tolist(aws_acm_certificate.gitpod.domain_validation_options)[1].resource_record_name + type = tolist(aws_acm_certificate.gitpod.domain_validation_options)[1].resource_record_type + zone_id = "${resource.aws_route53_zone.gitpod.zone_id}" + records = [tolist(aws_acm_certificate.gitpod.domain_validation_options)[1].resource_record_value] + ttl = 60 +} + +resource "aws_route53_record" "cert_validation_2" { + allow_overwrite = true + name = tolist(aws_acm_certificate.gitpod.domain_validation_options)[2].resource_record_name + type = tolist(aws_acm_certificate.gitpod.domain_validation_options)[2].resource_record_type + zone_id = "${resource.aws_route53_zone.gitpod.zone_id}" + records = [tolist(aws_acm_certificate.gitpod.domain_validation_options)[2].resource_record_value] + ttl = 60 +} + +resource "aws_route53_record" "cert_validation_3" { + allow_overwrite = true + name = tolist(aws_acm_certificate.gitpod.domain_validation_options)[3].resource_record_name + type = tolist(aws_acm_certificate.gitpod.domain_validation_options)[3].resource_record_type + zone_id = "${resource.aws_route53_zone.gitpod.zone_id}" + records = [tolist(aws_acm_certificate.gitpod.domain_validation_options)[3].resource_record_value] + ttl = 6 +} + +resource "aws_iam_policy" "gitpod" { + name = "role-${var.cluster_name}" + + # Terraform's "jsonencode" function converts a + # Terraform expression result to valid JSON syntax. + policy = jsonencode({ + Version = "2012-10-17", + Statement = [ + { + Effect = "Allow", + Action = [ + "route53:ChangeResourceRecordSets" + ], + Resource = [ + "arn:aws:route53:::hostedzone/*" + ] + }, + { + Effect = "Allow", + Action = [ + "route53:GetChange", + ], + Resource = [ + "arn:aws:route53:::change/*" + ] + }, + { + Effect = "Allow", + Action = [ + "route53:ListHostedZones", + "route53:ListResourceRecordSets" + ], + Resource = [ "*" ] + } + ], + }) +} + +resource "aws_iam_user_policy_attachment" "test-attach" { + user = aws_iam_user.edns.name + policy_arn = aws_iam_policy.gitpod.arn +} + +resource "aws_iam_user" "edns" { + name = "${var.cluster_name}-external-dns" + force_destroy = true + + tags = { + env = "test" + } +} + +resource "aws_iam_access_key" "edns" { + user = aws_iam_user.edns.name +} diff --git a/install/infra/terraform/eks/kubernetes.tf b/install/infra/terraform/eks/kubernetes.tf new file mode 100644 index 00000000000000..090d498fc1e71f --- /dev/null +++ b/install/infra/terraform/eks/kubernetes.tf @@ -0,0 +1,142 @@ +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "3.12.0" + + name = "vpc-${var.cluster_name}" + cidr = var.vpc_cidr + azs = var.vpc_availability_zones + private_subnets = [var.private_primary_subnet_cidr, var.private_secondary_subnet_cidr] + public_subnets = [var.public_primary_subnet_cidr, var.public_secondary_subnet_cidr, var.public_db_subnet_cidr_1, var.public_db_subnet_cidr_2] + enable_nat_gateway = true + enable_dns_hostnames = true +} + +resource "aws_security_group" "nodes" { + name = "nodes-sg-${var.cluster_name}" + vpc_id = module.vpc.vpc_id + + ingress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } +} + +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "18.8.1" + + cluster_name = var.cluster_name + cluster_version = "1.22" + + cluster_endpoint_public_access = true + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.public_subnets + + cluster_addons = { + coredns = { + resolve_conflicts = "OVERWRITE" + } + kube-proxy = {} + vpc-cni = { + resolve_conflicts = "OVERWRITE" + service_account_role_arn = module.vpc_cni_irsa.iam_role_arn + } + } + + eks_managed_node_group_defaults = { + disk_size = 150 + ami_type = "CUSTOM" + iam_role_attach_cni_policy = true + ami_id = var.image_id + enable_bootstrap_user_data = true + vpc_security_group_ids = [aws_security_group.nodes.id] + } + + eks_managed_node_groups = { + Services = { + enable_bootstrap_user_data = true + instance_types = [var.service_machine_type] + name = "service-${var.cluster_name}" + subnet_ids = module.vpc.public_subnets + min_size = 1 + max_size = 10 + desired_size = 1 + labels = { + "gitpod.io/workload_meta" = true + "gitpod.io/workload_ide" = true + } + pre_bootstrap_user_data = <<-EOT + #!/bin/bash + set -ex + cat <<-EOF > /etc/profile.d/bootstrap.sh + export CONTAINER_RUNTIME="containerd" + export USE_MAX_PODS=false + EOF + # Source extra environment variables in bootstrap script + sed -i '/^set -o errexit/a\\nsource /etc/profile.d/bootstrap.sh' /etc/eks/bootstrap.sh + EOT + } + + Workspaces = { + instance_types = [var.workspace_machine_type] + name = "ws-${var.cluster_name}" + subnet_ids = module.vpc.public_subnets + min_size = 1 + max_size = 10 + desired_size = 1 + enable_bootstrap_user_data = true + labels = { + "gitpod.io/workload_workspace_services" = true + "gitpod.io/workload_workspace_regular" = true + "gitpod.io/workload_workspace_headless" = true + } + pre_bootstrap_user_data = <<-EOT + #!/bin/bash + set -ex + cat <<-EOF > /etc/profile.d/bootstrap.sh + export CONTAINER_RUNTIME="containerd" + export USE_MAX_PODS=false + EOF + # Source extra environment variables in bootstrap script + sed -i '/^set -o errexit/a\\nsource /etc/profile.d/bootstrap.sh' /etc/eks/bootstrap.sh + EOT + } + } +} + +module "vpc_cni_irsa" { + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + version = "~> 4.12" + + role_name_prefix = "VPC-CNI-IRSA" + attach_vpc_cni_policy = true + vpc_cni_enable_ipv4 = true + + oidc_providers = { + main = { + provider_arn = module.eks.oidc_provider_arn + namespace_service_accounts = ["kube-system:aws-node"] + } + } +} + +resource "null_resource" "kubeconfig" { + depends_on = [ module.eks ] + provisioner "local-exec" { + command = "aws eks update-kubeconfig --region ${var.region} --name ${var.cluster_name} --kubeconfig ${var.kubeconfig}" + } + + lifecycle { + create_before_destroy = true + } +} diff --git a/install/infra/terraform/eks/output.tf b/install/infra/terraform/eks/output.tf new file mode 100644 index 00000000000000..151443ca504021 --- /dev/null +++ b/install/infra/terraform/eks/output.tf @@ -0,0 +1,72 @@ +output "external_dns_settings" { + value = [ + { + "name" = "provider", + "value" = "aws" + }, + { + "name" = "aws.region", + "value" = "eu-west-1" + }, + { + "name" = "aws.credentials.secretKey", + "value" = aws_iam_access_key.edns.secret + }, + { + "name" = "aws.credentials.accessKey", + "value" = aws_iam_access_key.edns.id + } + ] +} + +output "secretAccessKey" { + sensitive = true + value = try("${aws_iam_access_key.edns.secret}", "") +} + +output "cert_manager_issuer" { + value = try({ + region = "eu-west-1" + secretAccessKeySecretRef = { + name = "route53-credentials" + key = "secret-access-key" + } + + hostedZoneID = aws_route53_zone.gitpod.zone_id + accessKeyID = aws_iam_access_key.edns.id + }, {}) +} + +output "domain_nameservers" { + value = formatlist("%s.", resource.aws_route53_zone.gitpod.name_servers) +} + +output "database" { + sensitive = true + value = try({ + host = "${aws_db_instance.gitpod.address}" + password = "gitpod-qwat" # FIXME hardcoded at this point + port = 3306 + username = "${aws_db_instance.gitpod.username}" + }, {}) +} + +output "registry" { + sensitive = true + value = try({ + server = aws_ecr_repository.gitpod.repository_url + username = data.aws_ecr_authorization_token.gitpod.user_name + password = data.aws_ecr_authorization_token.gitpod.password + }, {}) +} + +output "storage" { + sensitive = true + value = try({ + access_key_id = aws_iam_access_key.bucket_storage_user.id + secret_access_key = aws_iam_access_key.bucket_storage_user.secret + region = aws_s3_bucket.gitpod-storage.region + bucket_name = aws_s3_bucket.gitpod-storage.id + endpoint = "s3.amazonaws.com" + }, {}) +} diff --git a/install/infra/terraform/eks/providers.tf b/install/infra/terraform/eks/providers.tf new file mode 100644 index 00000000000000..7e1676c2f73eca --- /dev/null +++ b/install/infra/terraform/eks/providers.tf @@ -0,0 +1,16 @@ +terraform { + required_providers { + aws = { + version = " ~> 3.0" + source = "registry.terraform.io/hashicorp/aws" + } + helm = { + source = "hashicorp/helm" + version = "2.5.1" + } + } +} + +provider "aws" { + region = var.region +} diff --git a/install/infra/terraform/eks/registry.tf b/install/infra/terraform/eks/registry.tf new file mode 100644 index 00000000000000..9756b6ed4b8858 --- /dev/null +++ b/install/infra/terraform/eks/registry.tf @@ -0,0 +1,12 @@ +resource "aws_ecr_repository" "gitpod" { + name = "registry-${var.cluster_name}" + image_tag_mutability = "MUTABLE" + + image_scanning_configuration { + scan_on_push = false + } +} + +data "aws_ecr_authorization_token" "gitpod" { + registry_id = aws_ecr_repository.gitpod.registry_id +} diff --git a/install/infra/terraform/eks/storage.tf b/install/infra/terraform/eks/storage.tf new file mode 100644 index 00000000000000..772187015ab190 --- /dev/null +++ b/install/infra/terraform/eks/storage.tf @@ -0,0 +1,37 @@ +resource "aws_s3_bucket" "gitpod-storage" { + force_destroy = true + bucket = "bucket-${var.cluster_name}" + acl = "private" + + versioning { + enabled = true + } +} + +data "aws_iam_policy_document" "s3_policy" { + statement { + actions = ["s3:*"] + resources = ["*"] + effect = "Allow" + } +} + +resource "aws_iam_policy" "policy" { + name = "spolicy-${var.cluster_name}" + description = "s3 storage bucket policy" + policy = data.aws_iam_policy_document.s3_policy.json +} + +resource "aws_iam_user" "bucket_storage" { + name = "suser-${var.cluster_name}" + +} + +resource "aws_iam_user_policy_attachment" "attachment" { + user = aws_iam_user.bucket_storage.name + policy_arn = aws_iam_policy.policy.arn +} + +resource "aws_iam_access_key" "bucket_storage_user" { + user = aws_iam_user.bucket_storage.name +} diff --git a/install/infra/terraform/eks/variables.tf b/install/infra/terraform/eks/variables.tf new file mode 100644 index 00000000000000..ccbc32f5207217 --- /dev/null +++ b/install/infra/terraform/eks/variables.tf @@ -0,0 +1,70 @@ +variable "cluster_name" { + type = string + description = "EKS cluster name." + default = "nan-eks-3" +} +variable "kubeconfig" { + type = string + description = "Path to the kubeconfig file" + default = "kubeconfig" +} + +variable "image_id" { + type = string + description = "AMI Image ID specific to the region" + // latest ubuntu image for 1.22 k8s for eu-west-1 region, refer https://cloud-images.ubuntu.com/docs/aws/eks/ + default = "ami-0793b4124359a6ad7" +} + +variable "service_machine_type" { + type = string + description = "Machine type for service workload node pool" + default = "m6i.xlarge" +} + +variable "workspace_machine_type" { + type = string + description = "Machine type for workspace workload node pool" + default = "m6i.2xlarge" +} + +variable "region" { + type = string + default = "eu-west-1" +} + +variable "vpc_availability_zones" { + type = list(string) + default = ["eu-west-1c", "eu-west-1b"] +} + +variable "domain_name" { +} + +variable "vpc_cidr" { + default = "10.100.0.0/16" +} + +variable "private_primary_subnet_cidr" { + default = "10.100.160.0/19" +} + +variable "private_secondary_subnet_cidr" { + default = "10.100.128.0/19" +} + +variable "public_primary_subnet_cidr" { + default = "10.100.64.0/18" +} + +variable "public_secondary_subnet_cidr" { + default = "10.100.0.0/18" +} + +variable "public_db_subnet_cidr_1" { + default = "10.100.192.0/19" +} + +variable "public_db_subnet_cidr_2" { + default = "10.100.224.0/19" +} diff --git a/install/infra/terraform/tools/external-dns/README.md b/install/infra/terraform/tools/cloud-dns-external-dns/README.md similarity index 100% rename from install/infra/terraform/tools/external-dns/README.md rename to install/infra/terraform/tools/cloud-dns-external-dns/README.md diff --git a/install/infra/terraform/tools/cloud-dns-external-dns/main.tf b/install/infra/terraform/tools/cloud-dns-external-dns/main.tf new file mode 100644 index 00000000000000..bb719f259abc5e --- /dev/null +++ b/install/infra/terraform/tools/cloud-dns-external-dns/main.tf @@ -0,0 +1,74 @@ +provider "kubernetes" { + config_path = var.kubeconfig +} + +data local_file "gcp_credentials" { + filename = var.credentials +} + +provider "google" { + credentials = var.credentials + project = var.gcp_project + region = var.gcp_region + zone = var.gcp_zone +} + +provider "helm" { + kubernetes { + config_path = var.kubeconfig + } +} + +#create namespace for external-dns +resource "kubernetes_namespace" "external_dns" { + metadata { + name = "external-dns" + } +} + +resource "kubernetes_secret" "external_dns" { + depends_on = [ + kubernetes_namespace.external_dns + ] + metadata { + name = "external-dns" + namespace = "external-dns" + } + data = { + "credentials.json" = data.local_file.gcp_credentials.content + } +} + +resource "helm_release" "external-dns" { + depends_on = [ + kubernetes_secret.external_dns, + kubernetes_namespace.external_dns + ] + name = "external-dns" + namespace = "external-dns" + create_namespace = true + chart = "external-dns" + repository = "https://charts.bitnami.com/bitnami" + cleanup_on_fail = true + replace = true + set { + name = "provider" + value = "google" + } + set { + name = "google.project" + value = var.gcp_project + } + set { + name = "logFormat" + value = "json" + } + set { + name = "google.serviceAccountSecret" + value = "external-dns" + } + set { + name = "txt-owner-id" + value = var.txt_owner_id + } +} diff --git a/install/infra/terraform/tools/external-dns/variables.tf b/install/infra/terraform/tools/cloud-dns-external-dns/variables.tf similarity index 100% rename from install/infra/terraform/tools/external-dns/variables.tf rename to install/infra/terraform/tools/cloud-dns-external-dns/variables.tf diff --git a/install/infra/terraform/tools/external-dns/main.tf b/install/infra/terraform/tools/external-dns/main.tf index bb719f259abc5e..688f1ae6d1f00d 100644 --- a/install/infra/terraform/tools/external-dns/main.tf +++ b/install/infra/terraform/tools/external-dns/main.tf @@ -1,17 +1,7 @@ -provider "kubernetes" { - config_path = var.kubeconfig -} - -data local_file "gcp_credentials" { - filename = var.credentials -} - -provider "google" { - credentials = var.credentials - project = var.gcp_project - region = var.gcp_region - zone = var.gcp_zone -} +variable settings {} +variable domain_name { default = "test"} +variable kubeconfig { default = "conf"} +variable txt_owner_id { default = "nightly-test" } provider "helm" { kubernetes { @@ -19,56 +9,28 @@ provider "helm" { } } -#create namespace for external-dns -resource "kubernetes_namespace" "external_dns" { - metadata { - name = "external-dns" - } -} - -resource "kubernetes_secret" "external_dns" { - depends_on = [ - kubernetes_namespace.external_dns - ] - metadata { - name = "external-dns" - namespace = "external-dns" - } - data = { - "credentials.json" = data.local_file.gcp_credentials.content - } -} - -resource "helm_release" "external-dns" { - depends_on = [ - kubernetes_secret.external_dns, - kubernetes_namespace.external_dns - ] +# External DNS Deployment using Helm +resource "helm_release" "external_dns" { name = "external-dns" + repository = "https://charts.bitnami.com" + chart = "external-dns" namespace = "external-dns" create_namespace = true - chart = "external-dns" - repository = "https://charts.bitnami.com/bitnami" - cleanup_on_fail = true - replace = true - set { - name = "provider" - value = "google" - } - set { - name = "google.project" - value = var.gcp_project - } - set { - name = "logFormat" - value = "json" - } + set { - name = "google.serviceAccountSecret" - value = "external-dns" + name = "domainFilters[0]" + value = var.domain_name } set { name = "txt-owner-id" value = var.txt_owner_id } + + dynamic "set" { + for_each = var.settings + content { + name = set.value["name"] + value = set.value["value"] + } + } } diff --git a/install/infra/terraform/tools/issuer/azure/main.tf b/install/infra/terraform/tools/issuer/main.tf similarity index 67% rename from install/infra/terraform/tools/issuer/azure/main.tf rename to install/infra/terraform/tools/issuer/main.tf index e76a6d9761de05..079b6cb12ecd35 100644 --- a/install/infra/terraform/tools/issuer/azure/main.tf +++ b/install/infra/terraform/tools/issuer/main.tf @@ -2,6 +2,18 @@ provider "kubernetes" { config_path = var.kubeconfig } +resource "kubernetes_secret" "dns_solver" { + count = var.secretAccessKey == null ? 0 : 1 + metadata { + name = "route53-credentials" + namespace = "cert-manager" + } + data = { + "secret-access-key" = var.secretAccessKey + } +} + + resource "kubernetes_manifest" "clusterissuer_gitpod" { manifest = { "apiVersion" = "cert-manager.io/v1" @@ -19,7 +31,7 @@ resource "kubernetes_manifest" "clusterissuer_gitpod" { "solvers" = [ { "dns01" = { - "azureDNS" = var.cert_manager_issuer + "${var.issuer_name}" = "${var.cert_manager_issuer}" } } ] diff --git a/install/infra/terraform/tools/issuer/azure/variables.tf b/install/infra/terraform/tools/issuer/variables.tf similarity index 62% rename from install/infra/terraform/tools/issuer/azure/variables.tf rename to install/infra/terraform/tools/issuer/variables.tf index 2960f621b2a55e..c9622796efa386 100644 --- a/install/infra/terraform/tools/issuer/azure/variables.tf +++ b/install/infra/terraform/tools/issuer/variables.tf @@ -6,3 +6,11 @@ variable "kubeconfig" { variable "cert_manager_issuer" { default = null } + +variable "secretAccessKey" { + default = null +} + +variable "issuer_name" { + default = "route53" +} diff --git a/install/tests/Makefile b/install/tests/Makefile index 05b0b2eb81cb53..80e689b6709427 100644 --- a/install/tests/Makefile +++ b/install/tests/Makefile @@ -29,33 +29,38 @@ gke-standard-cluster: terraform apply -target=module.gke -var kubeconfig=${KUBECONFIG} --auto-approve .PHONY: -## aks-standard-cluster: Creates an AKS cluster -aks-standard-cluster: +## eks-standard-cluster: Creates an EKS cluster +eks-standard-cluster: terraform init --upgrade && \ terraform workspace new $(TF_VAR_TEST_ID) || terraform workspace select $(TF_VAR_TEST_ID) && \ - terraform apply -target=module.aks -var k8s_flavor="aks" -var kubeconfig=${KUBECONFIG} --auto-approve + terraform apply -target=module.eks -var kubeconfig=${KUBECONFIG} --auto-approve + @echo "Done creating EKS cluster" .PHONY: -## azure-external-dns: Sets up external-dns with azure provider -azure-external-dns: -azure-external-dns: +## aks-standard-cluster: Creates an AKS cluster +aks-standard-cluster: terraform init --upgrade && \ terraform workspace new $(TF_VAR_TEST_ID) || terraform workspace select $(TF_VAR_TEST_ID) && \ - terraform apply -target=module.azure-externaldns -var kubeconfig=${KUBECONFIG} --auto-approve + terraform apply -target=module.aks -var kubeconfig=${KUBECONFIG} --auto-approve + @echo "Done creating AKS cluster" +cloud ?= azure .PHONY: ## add-ns-record: Adds NS record for subdomain under gitpod-selfhosted.com add-ns-record: terraform init --upgrade && \ terraform workspace new $(TF_VAR_TEST_ID) || terraform workspace select $(TF_VAR_TEST_ID) && \ - terraform apply -target=module.add_gcp_nameservers -var kubeconfig=${KUBECONFIG} --auto-approve + terraform apply -target=module.$(cloud)-add-dns-record -var kubeconfig=${KUBECONFIG} --auto-approve + @echo "Done adding NS record" +cloud ?= azure .PHONY: -## azure-issuer: Creates a cluster issuer with AD access -azure-issuer: +## cluster-issuer: Creates a cluster issuer for the correspondign provider +cluster-issuer: terraform init --upgrade && \ terraform workspace new $(TF_VAR_TEST_ID) || terraform workspace select $(TF_VAR_TEST_ID) && \ - terraform apply -target=module.azure-issuer -var kubeconfig=${KUBECONFIG} --auto-approve + terraform apply -target=module.$(cloud)-issuer -var kubeconfig=${KUBECONFIG} --auto-approve + @echo "Done creating cluster issuer" .PHONY: ## k3s-standard-cluster: Creates a K3S cluster on GCP with one master and 1 worker node @@ -63,21 +68,29 @@ k3s-standard-cluster: terraform init --upgrade && \ terraform workspace new $(TF_VAR_TEST_ID) || terraform workspace select $(TF_VAR_TEST_ID) && \ terraform apply -target=module.k3s -var kubeconfig=${KUBECONFIG} --auto-approve - -CLUSTER_ISSUER_CLOUD_DNS := "./manifests/gcp-issuer.yaml" + @echo "Done creating k3s cluster" .PHONY: ## cert-manager: Installs cert-manager, optionally create secret for cloud-dns access cert-manager: terraform workspace select $(TF_VAR_TEST_ID) && \ terraform apply -target=module.certmanager -var kubeconfig=${KUBECONFIG} --auto-approve + @echo "Done installing cert-manager" .PHONY: ## managed-dns: Installs external-dns, and setup up CloudDNS access managed-dns: check-env-sub-domain terraform workspace select $(TF_VAR_TEST_ID) && \ - terraform apply -target=module.externaldns -var kubeconfig=${KUBECONFIG} --auto-approve && \ - kubectl --kubeconfig=${KUBECONFIG} apply -f ${CLUSTER_ISSUER_CLOUD_DNS} + terraform apply -target=module.clouddns-externaldns -var kubeconfig=${KUBECONFIG} --auto-approve + @echo "Done created GCP managed DNS" + +cloud ?= "azure" +.PHONY: +## external-dns: Installs external-dns +external-dns: check-env-sub-domain + terraform workspace select $(TF_VAR_TEST_ID) && \ + terraform apply -target=module.$(cloud)-externaldns -var kubeconfig=${KUBECONFIG} --auto-approve + @echo "Done creating externaldns for ${cloud}" .PHONY: ## get-kubeconfig: Returns KUBECONFIG of a just created cluster @@ -111,24 +124,47 @@ get-config-gcp-db: envsubst '$${TF_VAR_TEST_ID}' < tmp_4_config.yml > tmp_5_config.yml yq m -i tmp_config.yml tmp_5_config.yml +get-config-azure-registry: + export SERVER=$$(terraform output -json azure_registry | yq r - 'server') && \ + export PASSWORD=$$(terraform output -json azure_registry | yq r - 'password') && \ + export USERNAME=$$(terraform output -json azure_registry | yq r - 'username') && \ + envsubst < ./manifests/kots-config-azure-registry.yaml > tmp_2_config.yml + yq m -i tmp_config.yml tmp_2_config.yml + get-config-azure-storage: - export PASSWORD=$$(terraform output -json storage | yq r - 'password') && \ - export USERNAME=$$(terraform output -json storage | yq r - 'username') && \ + export PASSWORD=$$(terraform output -json azure_storage | yq r - 'password') && \ + export USERNAME=$$(terraform output -json azure_storage | yq r - 'username') && \ envsubst < ./manifests/kots-config-azure-storage.yaml > tmp_2_config.yml yq m -i tmp_config.yml tmp_2_config.yml get-config-azure-db: - export DBHOST=$$(terraform output -json database | yq r - 'host') && \ - export DBPASS=$$(terraform output -json database | yq r - 'password') && \ - export DBUSER=$$(terraform output -json database | yq r - 'username') && \ + export DBHOST=$$(terraform output -json azure_database | yq r - 'host') && \ + export DBPASS=$$(terraform output -json azure_database | yq r - 'password') && \ + export DBUSER=$$(terraform output -json azure_database | yq r - 'username') && \ envsubst < ./manifests/kots-config-azure-db.yaml > tmp_2_config.yml yq m -i tmp_config.yml tmp_2_config.yml -get-config-azure-registry: - export SERVER=$$(terraform output -json registry | yq r - 'server') && \ - export PASSWORD=$$(terraform output -json registry | yq r - 'password') && \ - export USERNAME=$$(terraform output -json registry | yq r - 'username') && \ - envsubst < ./manifests/kots-config-azure-registry.yaml > tmp_2_config.yml +get-config-aws-db: + export DBHOST=$$(terraform output -json aws_database | yq r - 'host') && \ + export DBPASS=$$(terraform output -json aws_database | yq r - 'password') && \ + export DBUSER=$$(terraform output -json aws_database | yq r - 'username') && \ + envsubst < ./manifests/kots-config-aws-db.yaml > tmp_2_config.yml + yq m -i tmp_config.yml tmp_2_config.yml + +get-config-aws-storage: + export REGION=$$(terraform output -json aws_storage | yq r - 'region') && \ + export ENDPOINT=$$(terraform output -json aws_storage | yq r - 'endpoint') && \ + export BUCKET=$$(terraform output -json aws_storage | yq r - 'bucket_name') && \ + export S3_ACCESS_KEY_ID=$$(terraform output -json aws_storage | yq r - 'access_key_id') && \ + export S3_SECRET_ACCESS_KEY=$$(terraform output -json aws_storage | yq r - 'secret_access_key') && \ + envsubst < ./manifests/kots-config-aws-storage.yaml > tmp_2_config.yml + yq m -i tmp_config.yml tmp_2_config.yml + +get-config-aws-registry: + export SERVER=$$(terraform output -json aws_registry | yq r - 'server' | cut -d / -f 1) && \ + export PASSWORD=$$(terraform output -json aws_registry | yq r - 'password') && \ + export USERNAME=$$(terraform output -json aws_registry | yq r - 'username') && \ + envsubst < ./manifests/kots-config-aws-registry.yaml > tmp_2_config.yml yq m -i tmp_config.yml tmp_2_config.yml storage ?= incluster @@ -177,34 +213,31 @@ kots-uprgade: @echo "Upgrade gitpod KOTS app to latest" kubectl kots upstream upgrade --kubeconfig=${KUBECONFIG} gitpod -n gitpod --deploy -cleanup: destroy-gcp-externaldns destroy-gcpns destroy-aks-edns destroy-aks-issuer destroy-certmanager destroy-k3s destroy-gke destroy-aks +cloud ?= gcp +cleanup: destroy-$(cloud) select-workspace: terraform workspace select $(TF_VAR_TEST_ID) -destroy-gcp-externaldns: select-workspace - ls ${KUBECONFIG} && terraform destroy -target=module.externaldns -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file" - -destroy-certmanager: select-workspace +destroy-gcp: select-workspace + ls ${KUBECONFIG} && terraform destroy -target=module.clouddns-externaldns -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file" ls ${KUBECONFIG} && terraform destroy -target=module.certmanager -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file" - -destroy-gcpns: select-workspace - terraform destroy -target=module.add_gcp_nameservers -var kubeconfig=${KUBECONFIG} --auto-approve - -destroy-aks-edns: select-workspace - ls ${KUBECONFIG} && terraform destroy -target=module.azure-externaldns -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file" - -destroy-aks-issuer: select-workspace - ls ${KUBECONFIG} && terraform destroy -target=module.azure-issuer -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file" - -destroy-gke: select-workspace + terraform destroy -target=module.k3s -var kubeconfig=${KUBECONFIG} --auto-approve terraform destroy -target=module.gke -var kubeconfig=${KUBECONFIG} --auto-approve -destroy-k3s: select-workspace - terraform destroy -target=module.k3s -var kubeconfig=${KUBECONFIG} --auto-approve +destroy-aws: select-workspace + terraform destroy -target=module.aws-add-dns-record -var kubeconfig=${KUBECONFIG} --auto-approve + ls ${KUBECONFIG} && terraform destroy -target=module.aws-issuer -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file" + ls ${KUBECONFIG} && terraform destroy -target=module.aws-externaldns -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file" + ls ${KUBECONFIG} && terraform destroy -target=module.certmanager -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file" + terraform destroy -target=module.eks -var kubeconfig=${KUBECONFIG} --auto-approve -destroy-aks: select-workspace - terraform destroy -target=module.aks -var kubeconfig=${KUBECONFIG} --auto-approve +destroy-azure: select-workspace + ls ${KUBECONFIG} && terraform destroy -target=module.azure-issuer -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file" + terraform destroy -target=module.azure-add-dns-record -var kubeconfig=${KUBECONFIG} --auto-approve + ls ${KUBECONFIG} && terraform destroy -target=module.azure-externaldns -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file" + ls ${KUBECONFIG} && terraform destroy -target=module.certmanager -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file" + [ -z "$$ARM_SUBSCRIPTION_ID" ] || terraform destroy -target=module.aks -var kubeconfig=${KUBECONFIG} --auto-approve get-results: @echo "If you have gotten this far, it means your setup succeeded" diff --git a/install/tests/install/infra/terraform/eks/providers.tf b/install/tests/install/infra/terraform/eks/providers.tf new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/install/tests/main.tf b/install/tests/main.tf index 2766913b761872..38e84b448a9b8d 100644 --- a/install/tests/main.tf +++ b/install/tests/main.tf @@ -1,8 +1,6 @@ variable "kubeconfig" { } variable "TEST_ID" { default = "nightly" } -variable "k8s_flavor" { default = "gke" } - # We store the state always in a GCS bucket terraform { backend "gcs" { @@ -41,9 +39,23 @@ module "k3s" { domain_name = "${var.TEST_ID}.gitpod-self-hosted.com" } +module "gcp-issuer" { + source = "../infra/terraform/tools/issuer" + kubeconfig = var.kubeconfig + issuer_name = "cloudDNS" + cert_manager_issuer = { + project = "dns-for-playgrounds" + serviceAccountSecretRef = { + name = "clouddns-dns01-solver" + key = "keys.json" + } + } +} + + module "aks" { # source = "github.com/gitpod-io/gitpod//install/infra/terraform/aks?ref=main" # we can later use tags here - source = "../infra/terraform/aks" + source = "../infra/terraform/aks" domain_name = "${var.TEST_ID}.gitpod-self-hosted.com" enable_airgapped = false @@ -52,40 +64,66 @@ module "aks" { enable_external_storage = true dns_enabled = true workspace_name = var.TEST_ID + kubeconfig = var.kubeconfig +} + +module "eks" { + source = "../infra/terraform/eks" + domain_name = "${var.TEST_ID}.gitpod-self-hosted.com" + cluster_name = var.TEST_ID + region = "eu-west-1" + vpc_availability_zones = ["eu-west-1c", "eu-west-1b"] + image_id = "ami-0793b4124359a6ad7" // this AMI is regional + kubeconfig = var.kubeconfig } module "certmanager" { # source = "github.com/gitpod-io/gitpod//install/infra/terraform/tools/cert-manager?ref=main" - source = "../infra/terraform/tools/cert-manager" + source = "../infra/terraform/tools/cert-manager" - kubeconfig = var.kubeconfig - credentials = var.dns_sa_creds + kubeconfig = var.kubeconfig + credentials = var.dns_sa_creds } -module "externaldns" { +module "clouddns-externaldns" { # source = "github.com/gitpod-io/gitpod//install/infra/terraform/tools/external-dns?ref=main" - source = "../infra/terraform/tools/external-dns" - kubeconfig = var.kubeconfig - credentials = var.dns_sa_creds - txt_owner_id = var.TEST_ID + source = "../infra/terraform/tools/cloud-dns-external-dns" + kubeconfig = var.kubeconfig + credentials = var.dns_sa_creds } module "azure-externaldns" { - source = "../infra/terraform/tools/azure-external-dns" - kubeconfig = var.kubeconfig - settings = module.aks.external_dns_settings - domain_name = "${var.TEST_ID}.gitpod-self-hosted.com" - txt_owner_id = var.TEST_ID + source = "../infra/terraform/tools/external-dns" + kubeconfig = var.kubeconfig + settings = module.aks.external_dns_settings + domain_name = "${var.TEST_ID}.gitpod-self-hosted.com" + txt_owner_id = var.TEST_ID +} + +module "aws-externaldns" { + source = "../infra/terraform/tools/external-dns" + kubeconfig = var.kubeconfig + settings = module.eks.external_dns_settings + domain_name = "${var.TEST_ID}.gitpod-self-hosted.com" + txt_owner_id = var.TEST_ID } module "azure-issuer" { - source = "../infra/terraform/tools/issuer/azure" - kubeconfig = var.kubeconfig + source = "../infra/terraform/tools/issuer" + kubeconfig = var.kubeconfig cert_manager_issuer = module.aks.cert_manager_issuer + issuer_name = "azureDNS" +} + +module "aws-issuer" { + source = "../infra/terraform/tools/issuer" + kubeconfig = var.kubeconfig + cert_manager_issuer = module.eks.cert_manager_issuer + secretAccessKey = module.eks.secretAccessKey + issuer_name = "route53" } -module "add_gcp_nameservers" { - # source = "github.com/gitpod-io/gitpod//install/infra/terraform/tools/cloud-dns-ns?ref=main" +module "azure-add-dns-record" { source = "../infra/terraform/tools/cloud-dns-ns" credentials = var.dns_sa_creds nameservers = module.aks.domain_nameservers @@ -93,3 +131,12 @@ module "add_gcp_nameservers" { managed_dns_zone = "gitpod-self-hosted-com" domain_name = "${var.TEST_ID}.gitpod-self-hosted.com" } + +module "aws-add-dns-record" { + source = "../infra/terraform/tools/cloud-dns-ns" + credentials = var.dns_sa_creds + nameservers = module.eks.domain_nameservers + dns_project = "dns-for-playgrounds" + managed_dns_zone = "gitpod-self-hosted-com" + domain_name = "${var.TEST_ID}.gitpod-self-hosted.com" +} diff --git a/install/tests/manifests/kots-config-aws-db.yaml b/install/tests/manifests/kots-config-aws-db.yaml new file mode 100644 index 00000000000000..1e113404ae2907 --- /dev/null +++ b/install/tests/manifests/kots-config-aws-db.yaml @@ -0,0 +1,16 @@ +apiVersion: kots.io/v1beta1 +kind: ConfigValues +spec: + values: + db_incluster: + value: "0" + data: "db_incluster" + db_host: + value: ${DBHOST} + data: "db_host" + db_username: + value: ${DBUSER} + data: "db_username" + db_password: + value: ${DBPASS} + data: "db_password" diff --git a/install/tests/manifests/kots-config-aws-registry.yaml b/install/tests/manifests/kots-config-aws-registry.yaml new file mode 100644 index 00000000000000..1442501e3f07b5 --- /dev/null +++ b/install/tests/manifests/kots-config-aws-registry.yaml @@ -0,0 +1,16 @@ +apiVersion: kots.io/v1beta1 +kind: ConfigValues +spec: + values: + reg_incluster: + value: "0" + data: "reg_incluster" + reg_url: + value: ${SERVER} + data: "reg_url" + reg_username: + value: ${USERNAME} + data: "reg_username" + reg_password: + value: ${PASSWORD} + data: "reg_password" diff --git a/install/tests/manifests/kots-config-aws-storage.yaml b/install/tests/manifests/kots-config-aws-storage.yaml new file mode 100644 index 00000000000000..af7896eff483d5 --- /dev/null +++ b/install/tests/manifests/kots-config-aws-storage.yaml @@ -0,0 +1,22 @@ +apiVersion: kots.io/v1beta1 +kind: ConfigValues +spec: + values: + store_provider: + value: "s3" + data: "store_provider" + store_region: + value: "${REGION}" + data: "store_region" + store_s3_endpoint: + value: "${ENDPOINT}" + data: "store_s3_endpoint" + store_s3_bucket: + value: "${BUCKET}" + data: "store_s3_bucket" + store_s3_access_key_id: + value: "${S3_ACCESS_KEY_ID}" + data: "store_s3_access_key_id" + store_s3_secret_access_key: + value: "${S3_SECRET_ACCESS_KEY}" + data: "store_s3_secret_access_key" diff --git a/install/tests/output.tf b/install/tests/output.tf index 3982885ebec276..7af88c204d704a 100644 --- a/install/tests/output.tf +++ b/install/tests/output.tf @@ -1,18 +1,29 @@ -locals { - cloud = var.k8s_flavor == "aks" ? module.aks : null +output "aws_storage" { + sensitive = true + value = try(module.eks.storage, null) +} + +output "aws_registry" { + sensitive = true + value = try(module.eks.registry, null) +} + +output "aws_database" { + sensitive = true + value = try(module.eks.database, null) } -output "storage" { +output "azure_database" { sensitive = true - value = try(lookup(local.cloud, "storage"), {}) + value = try(module.aks.database, null) } -output "registry" { +output "azure_registry" { sensitive = true - value = try(lookup(local.cloud, "registry"), {}) + value = try(module.aks.registry, null) } -output "database" { +output "azure_storage" { sensitive = true - value = try(lookup(local.cloud, "database"), {}) + value = try(module.aks.storage, null) }