From c2a4a3798f17d2098d467d7ce5b7097e34fa1331 Mon Sep 17 00:00:00 2001 From: Tarun Pothulapati Date: Mon, 26 Sep 2022 09:52:13 +0000 Subject: [PATCH 1/2] [single-cluster/aws] Make `self-signed` work Fixes https://github.com/gitpod-io/gitpod/issues/12979 This PR adds a new `post_bootstrap_user_data` variable to the EKS terraform modules that changes the containerd config to accept new self signed SSL Certificates, and restarting `containerd` to make the same work. Signed-off-by: Tarun Pothulapati --- install/infra/modules/eks/kubernetes.tf | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/install/infra/modules/eks/kubernetes.tf b/install/infra/modules/eks/kubernetes.tf index 3de56a3770f727..c1a3e7ffd84ccd 100644 --- a/install/infra/modules/eks/kubernetes.tf +++ b/install/infra/modules/eks/kubernetes.tf @@ -103,6 +103,18 @@ module "eks" { enable_bootstrap_user_data = true vpc_security_group_ids = [aws_security_group.nodes.id] ebs_optimized = true + + post_bootstrap_user_data = <<-EOT + #!/bin/bash + cat << CONFIG >> /etc/containerd/config.toml + + [plugins."io.containerd.grpc.v1.cri".registry] + config_path = "/etc/containerd/certs.d" + + CONFIG + + service containerd restart + EOT } eks_managed_node_groups = { @@ -144,7 +156,7 @@ module "eks" { export CONTAINER_RUNTIME="containerd" export USE_MAX_PODS=false EOF - # Source extra environment 5ariables in bootstrap script + # Source extra environment variables in bootstrap script sed -i '/^set -o errexit/a\\nsource /etc/profile.d/bootstrap.sh' /etc/eks/bootstrap.sh EOT } @@ -170,7 +182,7 @@ module "eks" { desired_size = 2 enable_bootstrap_user_data = true labels = { - "gitpod.io/workload_workspace_regular" = true + "gitpod.io/workload_workspace_regular" = true } tags = { @@ -262,14 +274,14 @@ resource "null_resource" "kubeconfig" { data "aws_iam_policy_document" "eks_policy" { statement { - actions = [ + actions = [ "eks:DescribeCluster", "eks:ListClusters" ] resources = [ "*", ] - effect = "Allow" + effect = "Allow" } } @@ -281,7 +293,7 @@ resource "aws_iam_policy" "eks_policy" { resource "aws_iam_user" "eks_user" { force_destroy = true - name = "eks-user-${var.cluster_name}" + name = "eks-user-${var.cluster_name}" } resource "aws_iam_user_policy_attachment" "eks_attachment" { @@ -290,5 +302,5 @@ resource "aws_iam_user_policy_attachment" "eks_attachment" { } resource "aws_iam_access_key" "eks_user_key" { - user = aws_iam_user.eks_user.name + user = aws_iam_user.eks_user.name } From 523110a3e638aeac8c3e338810d4d1528097c594 Mon Sep 17 00:00:00 2001 From: Tarun Pothulapati Date: Tue, 27 Sep 2022 06:01:26 +0000 Subject: [PATCH 2/2] upload custom CA into terraform state Signed-off-by: Tarun Pothulapati --- .werft/eks-installer-tests.yaml | 2 ++ .werft/installer-tests.ts | 11 +++++++---- install/tests/Makefile | 6 ++++++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/.werft/eks-installer-tests.yaml b/.werft/eks-installer-tests.yaml index 92ae6276c92d56..6d8090664b5f11 100644 --- a/.werft/eks-installer-tests.yaml +++ b/.werft/eks-installer-tests.yaml @@ -89,6 +89,8 @@ pod: value: "/mnt/secrets/sh-playground-sa-perm/sh-sa.json" - name: TF_VAR_dns_sa_creds value: "/mnt/secrets/sh-playground-dns-perm/sh-dns-sa.json" + - name: TF_VAR_sa_creds + value: "/mnt/secrets/sh-playground-sa-perm/sh-sa.json" - name: NODENAME valueFrom: fieldRef: diff --git a/.werft/installer-tests.ts b/.werft/installer-tests.ts index 62a549f341fe6a..ed55f79bf0dbb9 100644 --- a/.werft/installer-tests.ts +++ b/.werft/installer-tests.ts @@ -347,6 +347,12 @@ export async function installerTests(config: TestConfig) { console.error("Failed to send message to Slack", error); }); + if (selfSigned === "true") { + exec( + `werft log result -d "Custom CA Certificate store underd GCP project 'sh-automated-tests'" url "gs://nightly-tests/tf-state/${process.env["TF_VAR_TEST_ID"]}-ca.pem"`, + ); + } + exec( `werft log result -d "Terraform state" url "Terraform state file name is ${process.env["TF_VAR_TEST_ID"]}"`, ); @@ -384,10 +390,7 @@ function runIntegrationTests() { function callMakeTargets(phase: string, description: string, makeTarget: string, failable: boolean = false) { werft.log(phase, `Calling ${makeTarget}`); // exporting cloud env var is important for the make targets - var env = `export TF_VAR_cluster_version=${k8s_version} cloud=${cloud} TF_VAR_domain=${baseDomain} TF_VAR_gcp_zone=${gcpDnsZone}`; - if (selfSigned) { - env = env.concat(` self_signed=${selfSigned}`) - } + const env = `export TF_VAR_cluster_version=${k8s_version} cloud=${cloud} TF_VAR_domain=${baseDomain} TF_VAR_gcp_zone=${gcpDnsZone}`; const response = exec( `${env} && make -C ${makefilePath} ${makeTarget}`, diff --git a/install/tests/Makefile b/install/tests/Makefile index af7a4dd9d3de5f..bbc0455fb2c19c 100644 --- a/install/tests/Makefile +++ b/install/tests/Makefile @@ -314,6 +314,9 @@ self-signed-config: envsubst < ./manifests/kots-config-self-signed.yaml > tmp_2_config.yml yq m -i tmp_config.yml tmp_2_config.yml + # upload the Custom CA Cert into tf-state + gsutil cp ./ca.pem gs://nightly-tests/tf-state/${TF_VAR_TEST_ID}-ca.pem + storage-config-incluster: @echo "Nothing to do" @@ -449,6 +452,9 @@ destroy-kubeconfig: gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS} --project=sh-automated-tests gsutil rm gs://nightly-tests/tf-state/${TF_VAR_TEST_ID}-kubeconfig || echo "No kubeconfig" gsutil rm gs://nightly-tests/tf-state/${TF_VAR_TEST_ID}-creds || echo "No credentials file" +ifeq (true,$(self_signed)) + gsutil rm gs://nightly-tests/tf-state/${TF_VAR_TEST_ID}-ca.pem || echo "No custom CA cert file" +endif rm ${KUBECONFIG} || echo "No kubeconfig" select-workspace: