From ce705319f1ae7e03b72263579fa0bfaa21f18ad2 Mon Sep 17 00:00:00 2001 From: Aaron Crickenberger Date: Wed, 18 Aug 2021 07:23:21 -0700 Subject: [PATCH 1/7] terraform/modules: add workload-identity-service-account --- .../README.md | 5 ++ .../workload-identity-service-account/main.tf | 45 +++++++++++++++++ .../outputs.tf | 25 ++++++++++ .../variables.tf | 48 +++++++++++++++++++ .../versions.tf | 28 +++++++++++ 5 files changed, 151 insertions(+) create mode 100644 infra/gcp/terraform/modules/workload-identity-service-account/README.md create mode 100644 infra/gcp/terraform/modules/workload-identity-service-account/main.tf create mode 100644 infra/gcp/terraform/modules/workload-identity-service-account/outputs.tf create mode 100644 infra/gcp/terraform/modules/workload-identity-service-account/variables.tf create mode 100644 infra/gcp/terraform/modules/workload-identity-service-account/versions.tf diff --git a/infra/gcp/terraform/modules/workload-identity-service-account/README.md b/infra/gcp/terraform/modules/workload-identity-service-account/README.md new file mode 100644 index 00000000000..090edd1f013 --- /dev/null +++ b/infra/gcp/terraform/modules/workload-identity-service-account/README.md @@ -0,0 +1,5 @@ +# `workload-identity-serviceaccount` terraform module + +This terraform module defines a GCP service account intended solely for use +by pods running in GKE clusters in a given project, running as a given K8s +service account in a given namespace. diff --git a/infra/gcp/terraform/modules/workload-identity-service-account/main.tf b/infra/gcp/terraform/modules/workload-identity-service-account/main.tf new file mode 100644 index 00000000000..1d9610e38b0 --- /dev/null +++ b/infra/gcp/terraform/modules/workload-identity-service-account/main.tf @@ -0,0 +1,45 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// creates a service account in project_id with name and description +// usable by pods in cluster_project_id +// running in namespace cluster_namespace +// running as cluster_serviceaccount_name + +locals { + description = var.description != "" ? var.description : var.name + cluster_project_id = var.cluster_project_id != "" ? var.cluster_project_id : var.project_id + cluster_serviceaccount_name = var.cluster_serviceaccount_name != "" ? var.cluster_serviceaccount_name : var.name +} + +resource "google_service_account" "serviceaccount" { + project = var.project_id + account_id = var.name + display_name = local.description +} +data "google_iam_policy" "workload_identity" { + binding { + members = [ + "serviceAccount:${local.cluster_project_id}.svc.id.goog[${var.cluster_namespace}/${local.cluster_serviceaccount_name}]", + ] + role = "roles/iam.workloadIdentityUser" + } +} +// authoritative binding, replaces any existing IAM policy on the service account +resource "google_service_account_iam_policy" "serviceaccount_iam" { + service_account_id = google_service_account.serviceaccount.name + policy_data = data.google_iam_policy.workload_identity.policy_data +} diff --git a/infra/gcp/terraform/modules/workload-identity-service-account/outputs.tf b/infra/gcp/terraform/modules/workload-identity-service-account/outputs.tf new file mode 100644 index 00000000000..008c15d396b --- /dev/null +++ b/infra/gcp/terraform/modules/workload-identity-service-account/outputs.tf @@ -0,0 +1,25 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +output "service_account" { + description = "The serviceaccount that was created" + value = google_service_account.serviceaccount +} + +output "iam_policy" { + description = "The serviceaccount iam_policy" + value = google_service_account_iam_policy.serviceaccount_iam +} diff --git a/infra/gcp/terraform/modules/workload-identity-service-account/variables.tf b/infra/gcp/terraform/modules/workload-identity-service-account/variables.tf new file mode 100644 index 00000000000..55e2af6d7cb --- /dev/null +++ b/infra/gcp/terraform/modules/workload-identity-service-account/variables.tf @@ -0,0 +1,48 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +variable "project_id" { + description = "The id of the project hosting the serviceaccount, eg: my-awesome-project" + type = string +} + +variable "name" { + description = "The name of the serviceaccount, eg: my-awesome-sa" + type = string +} + +variable "description" { + description = "The description of the service account, eg: My Awesome Service Account (default: name)" + type = string + default = "" +} + +variable "cluster_project_id" { + description = "The id of the project hosting clusters that will use the serviceaccount, eg: my-awesome-cluster-project (default: project_id)" + type = string + default = "" +} + +variable "cluster_serviceaccount_name" { + description = "The name of the kubernetes service account that will bind to the service account, eg: my-cluster-sa (default: name)" + type = string + default = "" +} + +variable "cluster_namespace" { + description = "The namespace of the kubernetes service account that will bind to the service account, eg: my-namespace" + type = string +} diff --git a/infra/gcp/terraform/modules/workload-identity-service-account/versions.tf b/infra/gcp/terraform/modules/workload-identity-service-account/versions.tf new file mode 100644 index 00000000000..9b1a854f01c --- /dev/null +++ b/infra/gcp/terraform/modules/workload-identity-service-account/versions.tf @@ -0,0 +1,28 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +terraform { + required_version = "~> 1.0.0" + required_providers { + google = { + source = "hashicorp/google" + version = "~> 3.74.0" + } + google-beta = { + source = "hashicorp/google-beta" + version = "~> 3.74.0" + } + } +} From e7225f5825a089b4cc3e27beb4f430306d09103d Mon Sep 17 00:00:00 2001 From: Aaron Crickenberger Date: Wed, 18 Aug 2021 07:52:58 -0700 Subject: [PATCH 2/7] terraform/k8s-infra-prow-build-trusted: use workload-identity-service-account --- .../k8s-infra-prow-build-trusted/main.tf | 230 ++++++------------ 1 file changed, 76 insertions(+), 154 deletions(-) diff --git a/infra/gcp/terraform/k8s-infra-prow-build-trusted/main.tf b/infra/gcp/terraform/k8s-infra-prow-build-trusted/main.tf index 64ad7a68caa..bbec3abbce8 100644 --- a/infra/gcp/terraform/k8s-infra-prow-build-trusted/main.tf +++ b/infra/gcp/terraform/k8s-infra-prow-build-trusted/main.tf @@ -23,18 +23,19 @@ This file defines: */ locals { - project_id = "k8s-infra-prow-build-trusted" - cluster_name = "prow-build-trusted" // The name of the cluster defined in this file - cluster_location = "us-central1" // The GCP location (region or zone) where the cluster should be created - bigquery_location = "US" // The bigquery specific location where the dataset should be created - pod_namespace = "test-pods" // MUST match whatever prow is configured to use when it schedules to this cluster + project_id = "k8s-infra-prow-build-trusted" + cluster_name = "prow-build-trusted" // The name of the cluster defined in this file + cluster_location = "us-central1" // The GCP location (region or zone) where the cluster should be created + bigquery_location = "US" // The bigquery specific location where the dataset should be created + pod_namespace = "test-pods" // MUST match whatever prow is configured to use when it schedules to this cluster // Service Accounts in ${pod_namespace} (usable via Workload Identity) - cluster_sa_name = "prow-build-trusted" // Pods use this by default - gcb_builder_sa_name = "gcb-builder" // Allowed to run GCB builds and push to GCS buckets - prow_deployer_sa_name = "prow-deployer" // Allowed to deploy to prow build clusters - k8s_metrics_sa_name = "k8s-metrics" // Allowed to write to gs://k8s-metrics - k8s_triage_sa_name = "k8s-triage" // Allowed to write to gs://k8s-project-triage + cluster_sa_name = "prow-build-trusted" // Pods use this by default + gcb_builder_sa_name = "gcb-builder" // Allowed to run GCB builds and push to GCS buckets + prow_deployer_sa_name = "prow-deployer" // Allowed to deploy to prow build clusters + k8s_metrics_sa_name = "k8s-metrics" // Allowed to write to gs://k8s-metrics + k8s_triage_sa_name = "k8s-triage" // Allowed to write to gs://k8s-project-triage + kubernetes_external_secrets_sa_name = "kubernetes-external-secrets" // Allowed to read from GSM in this and other projects } data "google_organization" "org" { @@ -42,9 +43,9 @@ data "google_organization" "org" { } module "project" { - source = "../../../modules/gke-project" - project_id = local.project_id - project_name = local.project_id + source = "../modules/gke-project" + project_id = local.project_id + project_name = local.project_id } // Ensure k8s-infra-prow-oncall@kuberentes.io has owner access to this project @@ -54,82 +55,39 @@ resource "google_project_iam_member" "k8s_infra_prow_oncall" { member = "group:k8s-infra-prow-oncall@kubernetes.io" } -// TODO: consider making this a real module to reduce copy-pasta +// TODO: consider moving the project role binding resources into the +// workload-identity-service-account module // -// The "workload_identity_service_account" comment denotes a pseudo-module of -// copy-pasted resources, similar to "ensure_workload_identity_serviceaccount" -// in ensure-main-project.sh. -// -// It is a shorthand for making a Kubernetes Service Account bound to a GCP -// Service Account of the same name, and optionally assigning it IAM roles. // Some of the roles are assigned in bash or other terraform modules, so as // to keep the permissions necessary to run this terraform module scoped to // "roles/owner" for local.project_id -// workload_identity_service_account: prow-build-trusted -// description: intended as default service account for pods in this cluster -resource "google_service_account" "prow_build_cluster_sa" { - project = local.project_id - account_id = local.cluster_sa_name - display_name = "Used by pods in '${local.cluster_name}' GKE cluster" -} -data "google_iam_policy" "prow_build_cluster_sa_workload_identity" { - binding { - role = "roles/iam.workloadIdentityUser" - - members = [ - "serviceAccount:${local.project_id}.svc.id.goog[${local.pod_namespace}/${local.cluster_sa_name}]", - ] - } -} -resource "google_service_account_iam_policy" "prow_build_cluster_sa_iam" { - service_account_id = google_service_account.prow_build_cluster_sa.name - policy_data = data.google_iam_policy.prow_build_cluster_sa_workload_identity.policy_data -} - -// workload_identity_service_account: gcb-builder -// description: used to trigger GCB builds in all k8s-staging projects -resource "google_service_account" "gcb_builder_sa" { - project = local.project_id - account_id = local.gcb_builder_sa_name - display_name = local.gcb_builder_sa_name +module "prow_build_cluster_sa" { + source = "../modules/workload-identity-service-account" + project_id = local.project_id + name = local.cluster_sa_name + description = "default service account for pods in ${local.cluster_name}" + cluster_namespace = local.pod_namespace } -data "google_iam_policy" "gcb_builder_sa_workload_identity" { - binding { - role = "roles/iam.workloadIdentityUser" +// roles: none - members = [ - "serviceAccount:${local.project_id}.svc.id.goog[${local.pod_namespace}/${local.gcb_builder_sa_name}]", - ] - } -} -resource "google_service_account_iam_policy" "gcb_builder_sa_iam" { - service_account_id = google_service_account.gcb_builder_sa.name - policy_data = data.google_iam_policy.gcb_builder_sa_workload_identity.policy_data +module "gcb_builder_sa" { + source = "../modules/workload-identity-service-account" + project_id = local.project_id + name = local.gcb_builder_sa_name + description = "trigger GCB builds in all k8s-staging projects" + cluster_namespace = local.pod_namespace } // roles: come from ensure-staging-storage.sh -// workload_identity_service_account: prow-deployer -// description: used to deploy k8s resources to k8s clusters -resource "google_service_account" "prow_deployer_sa" { - project = local.project_id - account_id = local.prow_deployer_sa_name - display_name = local.prow_deployer_sa_name -} -data "google_iam_policy" "prow_deployer_sa_workload_identity" { - binding { - role = "roles/iam.workloadIdentityUser" - - members = [ - "serviceAccount:${local.project_id}.svc.id.goog[${local.pod_namespace}/${local.prow_deployer_sa_name}]", - ] - } -} -resource "google_service_account_iam_policy" "prow_deployer_sa_iam" { - service_account_id = google_service_account.prow_deployer_sa.name - policy_data = data.google_iam_policy.prow_deployer_sa_workload_identity.policy_data +module "prow_deployer_sa" { + source = "../modules/workload-identity-service-account" + project_id = local.project_id + name = local.prow_deployer_sa_name + description = "deploys k8s resources to k8s clusters" + cluster_namespace = local.pod_namespace } -// roles: prow-deployer (there are also some assigned in ensure-main-project.sh) +// roles: there are also some assigned in ensure-main-project.sh resource "google_project_iam_member" "prow_deployer_for_prow_build_trusted" { project = local.project_id role = "roles/container.admin" @@ -141,84 +99,48 @@ resource "google_project_iam_member" "prow_deployer_for_prow_build" { member = "serviceAccount:${local.prow_deployer_sa_name}@${local.project_id}.iam.gserviceaccount.com" } -// workload_identity_service_account: k8s-metrics -resource "google_service_account" "k8s_metrics_sa" { - project = local.project_id - account_id = local.k8s_metrics_sa_name - display_name = local.k8s_metrics_sa_name -} -data "google_iam_policy" "k8s_metrics_sa_workload_identity" { - binding { - role = "roles/iam.workloadIdentityUser" - members = [ - "serviceAccount:${local.project_id}.svc.id.goog[${local.pod_namespace}/${local.k8s_metrics_sa_name}]", - ] - } -} -resource "google_service_account_iam_policy" "k8s_metrics_sa_iam" { - service_account_id = google_service_account.k8s_metrics_sa.name - policy_data = data.google_iam_policy.k8s_metrics_sa_workload_identity.policy_data +module "k8s_metrics_sa" { + source = "../modules/workload-identity-service-account" + project_id = local.project_id + name = local.k8s_metrics_sa_name + description = "read bigquery and write to gs://k8s-metrics" + cluster_namespace = local.pod_namespace } -// roles: k8s-metrics +// roles resource "google_project_iam_member" "k8s_metrics_sa_bigquery_user" { project = local.project_id role = "roles/bigquery.user" - member = "serviceAccount:${google_service_account.k8s_metrics_sa.email}" + member = "serviceAccount:${module.k8s_metrics_sa.service_account.email}" } -// workload_identity_service_account: triage -resource "google_service_account" "k8s_triage_sa" { - project = local.project_id - account_id = local.k8s_triage_sa_name - display_name = local.k8s_triage_sa_name -} -data "google_iam_policy" "k8s_triage_sa_workload_identity" { - binding { - role = "roles/iam.workloadIdentityUser" - members = [ - "serviceAccount:${local.project_id}.svc.id.goog[${local.pod_namespace}/${local.k8s_triage_sa_name}]", - ] - } -} -resource "google_service_account_iam_policy" "k8s_triage_sa_iam" { - service_account_id = google_service_account.k8s_triage_sa.name - policy_data = data.google_iam_policy.k8s_triage_sa_workload_identity.policy_data +module "k8s_triage_sa" { + source = "../modules/workload-identity-service-account" + project_id = local.project_id + name = local.k8s_triage_sa_name + description = "read bigquery and write to gs://k8s-triage" + cluster_namespace = local.pod_namespace } -// roles: triage +// roles resource "google_project_iam_member" "k8s_triage_sa_bigquery_user" { project = local.project_id role = "roles/bigquery.user" - member = "serviceAccount:${google_service_account.k8s_triage_sa.email}" + member = "serviceAccount:${module.k8s_triage_sa.service_account.email}" } -// workload_identity_service_account: kubernetes-external-secrets -// description: used by kubernetes-external-secrets to read specific secrets in this and other projects -resource "google_service_account" "kubernetes_external_secrets_sa" { - project = local.project_id - account_id = "kubernetes-external-secrets" - display_name = "kubernetes-external-secrets" -} -data "google_iam_policy" "kubernetes_external_secrets_sa_workload_identity" { - binding { - role = "roles/iam.workloadIdentityUser" - - members = [ - "serviceAccount:${local.project_id}.svc.id.goog[kubernetes-external-secrets/kubernetes-external-secrets]", - ] - } +module "kubernetes_external_secrets_sa" { + source = "../modules/workload-identity-service-account" + project_id = local.project_id + name = local.kubernetes_external_secrets_sa_name + description = "sync K8s secrets from GSM in this and other projects" + cluster_namespace = "kubernetes-external-secrets" } -resource "google_service_account_iam_policy" "kubernetes_external_secrets_sa_iam" { - service_account_id = google_service_account.kubernetes_external_secrets_sa.name - policy_data = data.google_iam_policy.kubernetes_external_secrets_sa_workload_identity.policy_data -} -// roles: kubernetes-external-secrets +// roles resource "google_project_iam_member" "kubernetes_external_secrets_for_prow_build_trusted" { project = local.project_id role = "roles/secretmanager.secretAccessor" - member = "serviceAccount:${google_service_account.kubernetes_external_secrets_sa.email}" + member = "serviceAccount:${module.kubernetes_external_secrets_sa.service_account.email}" } - // external (regional) ip addresses resource "google_compute_address" "kubernetes_external_secrets_metrics_address" { name = "kubernetes-external-secrets-metrics" @@ -237,26 +159,26 @@ resource "google_compute_address" "ghproxy_metrics_address" { } module "prow_build_cluster" { - source = "../../../modules/gke-cluster" - project_name = local.project_id - cluster_name = local.cluster_name - cluster_location = local.cluster_location - bigquery_location = local.bigquery_location - is_prod_cluster = "true" - release_channel = "REGULAR" - dns_cache_enabled = "true" + source = "../modules/gke-cluster" + project_name = local.project_id + cluster_name = local.cluster_name + cluster_location = local.cluster_location + bigquery_location = local.bigquery_location + is_prod_cluster = "true" + release_channel = "REGULAR" + dns_cache_enabled = "true" cloud_shell_access = false } module "prow_build_nodepool" { - source = "../../../modules/gke-nodepool" - project_name = local.project_id - cluster_name = module.prow_build_cluster.cluster.name - location = module.prow_build_cluster.cluster.location - name = "trusted-pool1" - initial_count = 1 - min_count = 1 - max_count = 6 + source = "../modules/gke-nodepool" + project_name = local.project_id + cluster_name = module.prow_build_cluster.cluster.name + location = module.prow_build_cluster.cluster.location + name = "trusted-pool1" + initial_count = 1 + min_count = 1 + max_count = 6 # image/machine/disk match prow-build for consistency's sake image_type = "UBUNTU_CONTAINERD" machine_type = "n1-highmem-8" From 0358385d0d707110611f7049d09cdd999cf2b250 Mon Sep 17 00:00:00 2001 From: Aaron Crickenberger Date: Wed, 18 Aug 2021 08:14:21 -0700 Subject: [PATCH 3/7] terraform/k8s-infra-prow-build-trusted: terraform fmt --- infra/gcp/terraform/k8s-infra-prow-build-trusted/versions.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/gcp/terraform/k8s-infra-prow-build-trusted/versions.tf b/infra/gcp/terraform/k8s-infra-prow-build-trusted/versions.tf index 6f171af4a1e..1a7d1da6605 100644 --- a/infra/gcp/terraform/k8s-infra-prow-build-trusted/versions.tf +++ b/infra/gcp/terraform/k8s-infra-prow-build-trusted/versions.tf @@ -13,7 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - + /* This file defines: - Required Terraform version From 0d096b78132f1ffcb5eb4f442564386d0cb0f21e Mon Sep 17 00:00:00 2001 From: Aaron Crickenberger Date: Wed, 18 Aug 2021 07:57:41 -0700 Subject: [PATCH 4/7] terraform/k8s-infra-prow-build: use workload-identity-service-account --- .../terraform/k8s-infra-prow-build/main.tf | 72 ++++++------------- 1 file changed, 22 insertions(+), 50 deletions(-) diff --git a/infra/gcp/terraform/k8s-infra-prow-build/main.tf b/infra/gcp/terraform/k8s-infra-prow-build/main.tf index 7d681469aa2..643433cbf4b 100644 --- a/infra/gcp/terraform/k8s-infra-prow-build/main.tf +++ b/infra/gcp/terraform/k8s-infra-prow-build/main.tf @@ -25,12 +25,12 @@ This file defines: locals { project_id = "k8s-infra-prow-build" - cluster_name = "prow-build" // The name of the cluster defined in this file - cluster_location = "us-central1" // The GCP location (region or zone) where the cluster should be created - bigquery_location = "US" // The bigquery specific location where the dataset should be created - pod_namespace = "test-pods" // MUST match whatever prow is configured to use when it schedules to this cluster - cluster_sa_name = "prow-build" // Name of the GSA and KSA that pods use by default - boskos_janitor_sa_name = "boskos-janitor" // Name of the GSA and KSA used by boskos-janitor + cluster_name = "prow-build" // The name of the cluster defined in this file + cluster_location = "us-central1" // The GCP location (region or zone) where the cluster should be created + bigquery_location = "US" // The bigquery specific location where the dataset should be created + pod_namespace = "test-pods" // MUST match whatever prow is configured to use when it schedules to this cluster + cluster_sa_name = "prow-build" // Name of the GSA and KSA that pods use by default + boskos_janitor_sa_name = "boskos-janitor" // Name of the GSA and KSA used by boskos-janitor } data "google_organization" "org" { @@ -38,7 +38,7 @@ data "google_organization" "org" { } module "project" { - source = "../../../modules/gke-project" + source = "../modules/gke-project" project_id = local.project_id project_name = local.project_id } @@ -62,48 +62,20 @@ resource "google_project_iam_member" "k8s_infra_prow_viewers" { member = "group:k8s-infra-prow-viewers@kubernetes.io" } -// Create GCP SA for pods -resource "google_service_account" "prow_build_cluster_sa" { - project = local.project_id - account_id = local.cluster_sa_name - display_name = "Used by pods in '${local.cluster_name}' GKE cluster" -} -// Allow pods using the build cluster KSA to use the GCP SA via workload identity -data "google_iam_policy" "prow_build_cluster_sa_workload_identity" { - binding { - role = "roles/iam.workloadIdentityUser" - - members = [ - "serviceAccount:${local.project_id}.svc.id.goog[${local.pod_namespace}/${local.cluster_sa_name}]", - ] - } -} -// Authoritative iam-policy: replaces any existing policy attached to this service_account -resource "google_service_account_iam_policy" "prow_build_cluster_sa_iam" { - service_account_id = google_service_account.prow_build_cluster_sa.name - policy_data = data.google_iam_policy.prow_build_cluster_sa_workload_identity.policy_data +module "prow_build_cluster_sa" { + source = "../modules/workload-identity-service-account" + project_id = local.project_id + name = local.cluster_sa_name + description = "default service account for pods in ${local.cluster_name}" + cluster_namespace = local.pod_namespace } -// Create GCP SA for boskos-janitor -resource "google_service_account" "boskos_janitor_sa" { - project = local.project_id - account_id = local.boskos_janitor_sa_name - display_name = "Used by ${local.boskos_janitor_sa_name} in '${local.cluster_name}' GKE cluster" -} -// Allow pods using the build cluster KSA to use the GCP SA via workload identity -data "google_iam_policy" "boskos_janitor_sa_workload_identity" { - binding { - role = "roles/iam.workloadIdentityUser" - - members = [ - "serviceAccount:${local.project_id}.svc.id.goog[${local.pod_namespace}/${local.boskos_janitor_sa_name}]", - ] - } -} -// Authoritative iam-policy: replaces any existing policy attached to this service account -resource "google_service_account_iam_policy" "boskos_janitor_sa_iam" { - service_account_id = google_service_account.boskos_janitor_sa.name - policy_data = data.google_iam_policy.boskos_janitor_sa_workload_identity.policy_data +module "boskos_janitor_sa" { + source = "../modules/workload-identity-service-account" + project_id = local.project_id + name = local.boskos_janitor_sa_name + description = "used by boskos-janitor in ${local.cluster_name}" + cluster_namespace = local.pod_namespace } // external ip formerly managed by infra/gcp/prow/ensure-e2e-projects.sh @@ -125,7 +97,7 @@ resource "google_compute_address" "greenhouse_metrics" { } module "prow_build_cluster" { - source = "../../../modules/gke-cluster" + source = "../modules/gke-cluster" project_name = local.project_id cluster_name = local.cluster_name cluster_location = local.cluster_location @@ -137,7 +109,7 @@ module "prow_build_cluster" { } module "prow_build_nodepool_n1_highmem_8_maxiops" { - source = "../../../modules/gke-nodepool" + source = "../modules/gke-nodepool" project_name = local.project_id cluster_name = module.prow_build_cluster.cluster.name location = module.prow_build_cluster.cluster.location @@ -157,7 +129,7 @@ module "prow_build_nodepool_n1_highmem_8_maxiops" { } module "greenhouse_nodepool" { - source = "../../../modules/gke-nodepool" + source = "../modules/gke-nodepool" project_name = local.project_id cluster_name = module.prow_build_cluster.cluster.name location = module.prow_build_cluster.cluster.location From 75f06650a4a5b7163f88aaf14fe608e80ce60c13 Mon Sep 17 00:00:00 2001 From: Aaron Crickenberger Date: Wed, 18 Aug 2021 08:13:55 -0700 Subject: [PATCH 5/7] terraform/k8s-infra-prow-build: terraform fmt --- infra/gcp/terraform/k8s-infra-prow-build/00-provider.tf | 2 +- infra/gcp/terraform/k8s-infra-prow-build/versions.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/infra/gcp/terraform/k8s-infra-prow-build/00-provider.tf b/infra/gcp/terraform/k8s-infra-prow-build/00-provider.tf index 92de6aeff78..04967b33b23 100644 --- a/infra/gcp/terraform/k8s-infra-prow-build/00-provider.tf +++ b/infra/gcp/terraform/k8s-infra-prow-build/00-provider.tf @@ -13,7 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - + /* This file defines: - Required provider versions diff --git a/infra/gcp/terraform/k8s-infra-prow-build/versions.tf b/infra/gcp/terraform/k8s-infra-prow-build/versions.tf index 6f171af4a1e..1a7d1da6605 100644 --- a/infra/gcp/terraform/k8s-infra-prow-build/versions.tf +++ b/infra/gcp/terraform/k8s-infra-prow-build/versions.tf @@ -13,7 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - + /* This file defines: - Required Terraform version From d014db364d1cea1f31c9c6fe5802813d5b5f633b Mon Sep 17 00:00:00 2001 From: Aaron Crickenberger Date: Mon, 30 Aug 2021 11:08:36 -0400 Subject: [PATCH 6/7] terraform: output email for workload-identity-service-account --- infra/gcp/terraform/k8s-infra-prow-build-trusted/main.tf | 6 +++--- .../modules/workload-identity-service-account/outputs.tf | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/infra/gcp/terraform/k8s-infra-prow-build-trusted/main.tf b/infra/gcp/terraform/k8s-infra-prow-build-trusted/main.tf index bbec3abbce8..5bea4de59b5 100644 --- a/infra/gcp/terraform/k8s-infra-prow-build-trusted/main.tf +++ b/infra/gcp/terraform/k8s-infra-prow-build-trusted/main.tf @@ -110,7 +110,7 @@ module "k8s_metrics_sa" { resource "google_project_iam_member" "k8s_metrics_sa_bigquery_user" { project = local.project_id role = "roles/bigquery.user" - member = "serviceAccount:${module.k8s_metrics_sa.service_account.email}" + member = "serviceAccount:${module.k8s_metrics_sa.email}" } module "k8s_triage_sa" { @@ -124,7 +124,7 @@ module "k8s_triage_sa" { resource "google_project_iam_member" "k8s_triage_sa_bigquery_user" { project = local.project_id role = "roles/bigquery.user" - member = "serviceAccount:${module.k8s_triage_sa.service_account.email}" + member = "serviceAccount:${module.k8s_triage_sa.email}" } module "kubernetes_external_secrets_sa" { @@ -138,7 +138,7 @@ module "kubernetes_external_secrets_sa" { resource "google_project_iam_member" "kubernetes_external_secrets_for_prow_build_trusted" { project = local.project_id role = "roles/secretmanager.secretAccessor" - member = "serviceAccount:${module.kubernetes_external_secrets_sa.service_account.email}" + member = "serviceAccount:${module.kubernetes_external_secrets_sa.email}" } // external (regional) ip addresses diff --git a/infra/gcp/terraform/modules/workload-identity-service-account/outputs.tf b/infra/gcp/terraform/modules/workload-identity-service-account/outputs.tf index 008c15d396b..d868edc45b5 100644 --- a/infra/gcp/terraform/modules/workload-identity-service-account/outputs.tf +++ b/infra/gcp/terraform/modules/workload-identity-service-account/outputs.tf @@ -14,9 +14,9 @@ See the License for the specific language governing permissions and limitations under the License. */ -output "service_account" { - description = "The serviceaccount that was created" - value = google_service_account.serviceaccount +output "email" { + description = "The email of the serviceaccount that was created" + value = google_service_account.serviceaccount.email } output "iam_policy" { From 2b8f691f5cbe90d3138ee7fd05a744c68c45ca20 Mon Sep 17 00:00:00 2001 From: Aaron Crickenberger Date: Mon, 30 Aug 2021 11:10:58 -0400 Subject: [PATCH 7/7] terraform/modules: output policy data for workload-identity-service-account --- .../modules/workload-identity-service-account/outputs.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/gcp/terraform/modules/workload-identity-service-account/outputs.tf b/infra/gcp/terraform/modules/workload-identity-service-account/outputs.tf index d868edc45b5..61dc55aead8 100644 --- a/infra/gcp/terraform/modules/workload-identity-service-account/outputs.tf +++ b/infra/gcp/terraform/modules/workload-identity-service-account/outputs.tf @@ -21,5 +21,5 @@ output "email" { output "iam_policy" { description = "The serviceaccount iam_policy" - value = google_service_account_iam_policy.serviceaccount_iam + value = google_service_account_iam_policy.serviceaccount_iam.policy_data }