use secret manager to store auth token

small updates and improvements
GoogleCloudPlatform · May 13, 2024 · e13aefe · e13aefe
1 parent 1edb25a
commit e13aefe
Show file tree

Hide file tree

Showing 5 changed files with 77 additions and 28 deletions.
diff --git a/blueprints/third-party-solutions/gitlab-runner/README.md b/blueprints/third-party-solutions/gitlab-runner/README.md
@@ -128,11 +128,12 @@ Please be aware of the following requirements for the Docker Autoscaler executor
 | name | description | type | required | default |
 |---|---|:---:|:---:|:---:|
 | [gitlab_config](variables.tf#L23) | Gitlab server configuration. | <code title="object&#40;&#123;&#10;  hostname    &#61; optional&#40;string, &#34;gitlab.gcp.example.com&#34;&#41;&#10;  ca_cert_pem &#61; optional&#40;string, null&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  |
-| [gitlab_runner_config](variables.tf#L31) | Gitlab Runner config. | <code title="object&#40;&#123;&#10;  authentication_token &#61; string&#10;  executors_config &#61; object&#40;&#123;&#10;    docker_autoscaler &#61; optional&#40;object&#40;&#123;&#10;      gcp_project_id &#61; string&#10;      zone           &#61; optional&#40;string, &#34;europe-west1-b&#34;&#41;&#10;      mig_name       &#61; optional&#40;string, &#34;gitlab-runner&#34;&#41;&#10;      machine_type   &#61; optional&#40;string, &#34;g1-small&#34;&#41;&#10;      machine_image  &#61; optional&#40;string, &#34;coreos-cloud&#47;global&#47;images&#47;family&#47;coreos-stable&#34;&#41;&#10;      network_tags   &#61; optional&#40;list&#40;string&#41;, &#91;&#34;gitlab-runner&#34;&#93;&#41;&#10;    &#125;&#41;, null&#41;&#10;    docker &#61; optional&#40;object&#40;&#123;&#10;      tls_verify &#61; optional&#40;bool, true&#41;&#10;    &#125;&#41;, null&#41;&#10;  &#125;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  |
+| [gitlab_runner_config](variables.tf#L31) | Gitlab Runner config. | <code title="object&#40;&#123;&#10;  authentication_token &#61; string&#10;  executors_config &#61; object&#40;&#123;&#10;    docker_autoscaler &#61; optional&#40;object&#40;&#123;&#10;      gcp_project_id &#61; string&#10;      zone           &#61; optional&#40;string&#41;&#10;      mig_name       &#61; optional&#40;string, &#34;gitlab-runner&#34;&#41;&#10;      machine_type   &#61; optional&#40;string, &#34;g1-small&#34;&#41;&#10;      machine_image  &#61; optional&#40;string, &#34;coreos-cloud&#47;global&#47;images&#47;family&#47;coreos-stable&#34;&#41;&#10;      network_tags   &#61; optional&#40;list&#40;string&#41;, &#91;&#34;gitlab-runner&#34;&#93;&#41;&#10;    &#125;&#41;, null&#41;&#10;    docker &#61; optional&#40;object&#40;&#123;&#10;      tls_verify &#61; optional&#40;bool, true&#41;&#10;    &#125;&#41;, null&#41;&#10;  &#125;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  |
 | [network_config](variables.tf#L58) | Shared VPC network configurations to use for Gitlab Runner VM. | <code title="object&#40;&#123;&#10;  host_project      &#61; optional&#40;string&#41;&#10;  network_self_link &#61; string&#10;  subnet_self_link  &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  |
 | [prefix](variables.tf#L67) | Prefix used for resource names. | <code>string</code> | ✓ |  |
 | [project_id](variables.tf#L86) | Project id, references existing project if `project_create` is null. | <code>string</code> | ✓ |  |
-| [vm_config](variables.tf#L91) | Gitlab runner GCE config. | <code title="object&#40;&#123;&#10;  boot_disk_size &#61; optional&#40;number, 100&#41;&#10;  name           &#61; optional&#40;string, &#34;gitlab-runner-0&#34;&#41;&#10;  instance_type  &#61; optional&#40;string, &#34;e2-standard-2&#34;&#41;&#10;  network_tags   &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10;  zone           &#61; optional&#40;string, &#34;europe-west1-b&#34;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  |
+| [region](variables.tf#L91) | Region for the created resources. | <code>string</code> | ✓ |  |
+| [vm_config](variables.tf#L96) | Gitlab runner GCE config. | <code title="object&#40;&#123;&#10;  boot_disk_size &#61; optional&#40;number, 100&#41;&#10;  name           &#61; optional&#40;string, &#34;gitlab-runner-0&#34;&#41;&#10;  instance_type  &#61; optional&#40;string, &#34;e2-standard-2&#34;&#41;&#10;  network_tags   &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10;  zone           &#61; optional&#40;string&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  |
 | [admin_principals](variables.tf#L17) | Users, groups and/or service accounts that are assigned roles, in IAM format (`group:[email protected]`). | <code>list&#40;string&#41;</code> |  | <code>&#91;&#93;</code> |
 | [project_create](variables.tf#L77) | Provide values if project creation is needed, uses existing project if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | <code title="object&#40;&#123;&#10;  billing_account_id &#61; string&#10;  parent             &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |
 <!-- END TFDOC -->
diff --git a/blueprints/third-party-solutions/gitlab-runner/assets/startup-script.sh.tpl b/blueprints/third-party-solutions/gitlab-runner/assets/startup-script.sh.tpl
@@ -1,3 +1,4 @@
+#!/bin/bash
 # Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,11 +13,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-#!/bin/bash
-
 GITLAB_URL=https://${gitlab_hostname}
 GITLAB_RUNNER_CONFIG=${gitlab_runner_config}
-
+GITLAB_TOKEN_SECRET_ID=${gitlab_token_secret_id}
+GITLAB_TOKEN_SECRET_VERSION="latest"
 GL_NAME=$(curl 169.254.169.254/computeMetadata/v1/instance/name --header "Metadata-Flavor:Google")
 GL_EXECUTOR=$(curl 169.254.169.254/computeMetadata/v1/instance/attributes/gl_executor --header "Metadata-Flavor:Google")
 
@@ -41,12 +41,12 @@ echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.
 apt-get update
 apt-get install -yq docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
 
-# setup new gitlab runner config
-echo $GITLAB_RUNNER_CONFIG | base64 -d > /etc/gitlab-runner/config.toml
-
 # Install Gitlab Runner
 apt install -y gitlab-runner
 
+# setup new gitlab runner config
+echo $GITLAB_RUNNER_CONFIG | base64 -d > /etc/gitlab-runner/config.toml
+
 %{ if gitlab_executor_type == "docker-autoscaler" }
 # Install GCP fleeting plugin for Docker Autoscale Runner
 # https://docs.gitlab.com/runner/executors/docker_autoscaler.html#install-a-fleeting-plugin
@@ -55,7 +55,10 @@ chmod +x fleeting-plugin-googlecompute-linux-386
 mv ./fleeting-plugin-googlecompute-linux-386 /usr/bin/fleeting-plugin-googlecompute
 %{ endif }
 
+# Fetch the gitlab auth token value from secret manager
+TOKEN=$(gcloud secrets versions access $GITLAB_TOKEN_SECRET_VERSION --secret $GITLAB_TOKEN_SECRET_ID)
+
 gitlab-runner register --non-interactive --name="$GL_NAME" \
-  --url="$GITLAB_URL" --token="${token}" --template-config="/etc/gitlab-runner/config.toml"
+  --url="$GITLAB_URL" --token="$TOKEN" --template-config="/etc/gitlab-runner/config.toml" \
   --executor="${gitlab_executor_type}"
 systemctl restart gitlab-runner
diff --git a/blueprints/third-party-solutions/gitlab-runner/main.tf b/blueprints/third-party-solutions/gitlab-runner/main.tf
@@ -15,25 +15,18 @@
  */
 
 locals {
-  role_id            = "projects/${module.project.project_id}/roles/${local.role_name}"
-  role_name          = "gitlab_runner_manager_role"
+  role_id            = module.project.custom_role_ids[local.role_name]
+  role_name          = "gitlabRunnerManagerRole"
   runner_config_type = [for key, value in var.gitlab_runner_config.executors_config : key if value != null][0]
   runner_startup_script_config = {
-    gitlab_hostname      = var.gitlab_config.hostname
-    gitlab_ca_cert       = base64encode(var.gitlab_config.ca_cert_pem)
-    token                = var.gitlab_runner_config.authentication_token
-    gitlab_runner_config = base64encode(templatefile("${path.module}/assets/config/${local.runner_config_type}_config.toml.tpl", var.gitlab_runner_config.executors_config[local.runner_config_type]))
-    gitlab_executor_type = replace(local.runner_config_type, "_", "-")
+    gitlab_hostname        = var.gitlab_config.hostname
+    gitlab_ca_cert         = base64encode(var.gitlab_config.ca_cert_pem)
+    gitlab_token_secret_id = local.gitlab_runner_auth_token_secret_id
+    gitlab_runner_config   = base64encode(templatefile("${path.module}/assets/config/${local.runner_config_type}_config.toml.tpl", var.gitlab_runner_config.executors_config[local.runner_config_type]))
+    gitlab_executor_type   = replace(local.runner_config_type, "_", "-")
   }
 }
 
-resource "google_service_account_iam_member" "admin-account-iam" {
-  count              = local.runner_config_type == "docker_autoscaler" ? 1 : 0
-  service_account_id = module.gitlab-runner-template.0.service_account.name
-  role               = "roles/iam.serviceAccountUser"
-  member             = "serviceAccount:${module.gitlab-runner.service_account.email}"
-}
-
 module "project" {
   source          = "../../../modules/project"
   parent          = try(var.project_create.parent, null)
@@ -50,14 +43,15 @@ module "project" {
     ]
   }
   iam = {
-    (local.role_id) = ["serviceAccount:${module.gitlab-runner.service_account.email}"]
+    (local.role_id) = ["serviceAccount:${module.runner-sa.email}"]
   }
   services = [
     "compute.googleapis.com",
     "storage.googleapis.com",
     "stackdriver.googleapis.com",
     "dns.googleapis.com",
     "iam.googleapis.com",
+    "secretmanager.googleapis.com"
   ]
   shared_vpc_service_config = {
     attach       = true

diff --git a/blueprints/third-party-solutions/gitlab-runner/runner.tf b/blueprints/third-party-solutions/gitlab-runner/runner.tf
@@ -14,6 +14,52 @@
  * limitations under the License.
  */
 
+locals {
+  gitlab_runner_auth_token_secret_id = "gitlab_runner_auth_token"
+}
+
+module "runner-sa" {
+  source     = "../../../modules/iam-service-account"
+  project_id = module.project.project_id
+  name       = "gitlab-runner-sa"
+}
+
+module "runner-mig-sa" {
+  count      = local.runner_config_type == "docker_autoscaler" ? 1 : 0
+  source     = "../../../modules/iam-service-account"
+  project_id = module.project.project_id
+  name       = "gitlab-runner-sa"
+  iam = {
+    "roles/iam.serviceAccountUser" = [
+      "serviceAccount:${module.runner-sa.email}"
+    ]
+  }
+}
+
+module "runner-secrets" {
+  source     = "../../../modules/secret-manager"
+  project_id = module.project.project_id
+  secrets = {
+    (local.gitlab_runner_auth_token_secret_id) = {
+      locations = [var.region]
+    }
+  }
+  versions = {
+    (local.gitlab_runner_auth_token_secret_id) = {
+      latest = {
+        enabled = true, data = var.gitlab_runner_config.authentication_token
+      }
+    }
+  }
+  iam = {
+    (local.gitlab_runner_auth_token_secret_id) = {
+      "roles/secretmanager.secretAccessor" = [
+        "serviceAccount:${module.runner-sa.email}"
+      ]
+    }
+  }
+}
+
 module "gitlab-runner" {
   source     = "../../../modules/compute-vm"
   project_id = module.project.project_id
@@ -36,7 +82,7 @@ module "gitlab-runner" {
     startup-script = templatefile("${path.module}/assets/startup-script.sh.tpl", local.runner_startup_script_config)
   }
   service_account = {
-    auto_create = true
+    email = module.runner-sa.email
   }
 }
 
@@ -59,7 +105,7 @@ module "gitlab-runner-template" {
     }
   }
   service_account = {
-    auto_create = true
+    email = module.runner-mig-sa.0.email
   }
   create_template = true
 }

diff --git a/blueprints/third-party-solutions/gitlab-runner/variables.tf b/blueprints/third-party-solutions/gitlab-runner/variables.tf
@@ -35,7 +35,7 @@ variable "gitlab_runner_config" {
     executors_config = object({
       docker_autoscaler = optional(object({
         gcp_project_id = string
-        zone           = optional(string, "europe-west1-b")
+        zone           = optional(string)
         mig_name       = optional(string, "gitlab-runner")
         machine_type   = optional(string, "g1-small")
         machine_image  = optional(string, "coreos-cloud/global/images/family/coreos-stable")
@@ -88,13 +88,18 @@ variable "project_id" {
   type        = string
 }
 
+variable "region" {
+  description = "Region for the created resources."
+  type        = string
+}
+
 variable "vm_config" {
   description = "Gitlab runner GCE config."
   type = object({
     boot_disk_size = optional(number, 100)
     name           = optional(string, "gitlab-runner-0")
     instance_type  = optional(string, "e2-standard-2")
     network_tags   = optional(list(string), [])
-    zone           = optional(string, "europe-west1-b")
+    zone           = optional(string)
   })
 }