Merge pull request #2574 from spiffxp/workload-identity-service-account

terraform: add and use workload-identity-service-account module
kubernetes · Aug 30, 2021 · 40b89dd · 40b89dd
2 parents 15f7c4d + 2b8f691
commit 40b89dd
Show file tree

Hide file tree

Showing 10 changed files with 252 additions and 207 deletions.
diff --git a/infra/gcp/terraform/k8s-infra-prow-build-trusted/main.tf b/infra/gcp/terraform/k8s-infra-prow-build-trusted/main.tf
@@ -23,28 +23,29 @@ This file defines:
 */
 
 locals {
-  project_id                   = "k8s-infra-prow-build-trusted"
-  cluster_name                 = "prow-build-trusted"   // The name of the cluster defined in this file
-  cluster_location             = "us-central1"          // The GCP location (region or zone) where the cluster should be created
-  bigquery_location            = "US"                   // The bigquery specific location where the dataset should be created
-  pod_namespace                = "test-pods"            // MUST match whatever prow is configured to use when it schedules to this cluster
+  project_id        = "k8s-infra-prow-build-trusted"
+  cluster_name      = "prow-build-trusted" // The name of the cluster defined in this file
+  cluster_location  = "us-central1"        // The GCP location (region or zone) where the cluster should be created
+  bigquery_location = "US"                 // The bigquery specific location where the dataset should be created
+  pod_namespace     = "test-pods"          // MUST match whatever prow is configured to use when it schedules to this cluster
 
   // Service Accounts in ${pod_namespace} (usable via Workload Identity)
-  cluster_sa_name              = "prow-build-trusted"   // Pods use this by default
-  gcb_builder_sa_name          = "gcb-builder"          // Allowed to run GCB builds and push to GCS buckets
-  prow_deployer_sa_name        = "prow-deployer"        // Allowed to deploy to prow build clusters
-  k8s_metrics_sa_name          = "k8s-metrics"          // Allowed to write to gs://k8s-metrics
-  k8s_triage_sa_name           = "k8s-triage"           // Allowed to write to gs://k8s-project-triage
+  cluster_sa_name                     = "prow-build-trusted"          // Pods use this by default
+  gcb_builder_sa_name                 = "gcb-builder"                 // Allowed to run GCB builds and push to GCS buckets
+  prow_deployer_sa_name               = "prow-deployer"               // Allowed to deploy to prow build clusters
+  k8s_metrics_sa_name                 = "k8s-metrics"                 // Allowed to write to gs://k8s-metrics
+  k8s_triage_sa_name                  = "k8s-triage"                  // Allowed to write to gs://k8s-project-triage
+  kubernetes_external_secrets_sa_name = "kubernetes-external-secrets" // Allowed to read from GSM in this and other projects
 }
 
 data "google_organization" "org" {
   domain = "kubernetes.io"
 }
 
 module "project" {
-  source = "../../../modules/gke-project"
-  project_id            = local.project_id
-  project_name          = local.project_id
+  source       = "../modules/gke-project"
+  project_id   = local.project_id
+  project_name = local.project_id
 }
 
 // Ensure [email protected] has owner access to this project
@@ -54,82 +55,39 @@ resource "google_project_iam_member" "k8s_infra_prow_oncall" {
   member  = "group:[email protected]"
 }
 
-// TODO: consider making this a real module to reduce copy-pasta
+// TODO: consider moving the project role binding resources into the
+//       workload-identity-service-account module
 // 
-// The "workload_identity_service_account" comment denotes a pseudo-module of
-// copy-pasted resources, similar to "ensure_workload_identity_serviceaccount"
-// in ensure-main-project.sh.
-//
-// It is a shorthand for making a Kubernetes Service Account bound to a GCP
-// Service Account of the same name, and optionally assigning it IAM roles.
 // Some of the roles are assigned in bash or other terraform modules, so as
 // to keep the permissions necessary to run this terraform module scoped to
 // "roles/owner" for local.project_id
 
-// workload_identity_service_account: prow-build-trusted
-// description: intended as default service account for pods in this cluster
-resource "google_service_account" "prow_build_cluster_sa" {
-  project      = local.project_id
-  account_id   = local.cluster_sa_name
-  display_name = "Used by pods in '${local.cluster_name}' GKE cluster"
-}
-data "google_iam_policy" "prow_build_cluster_sa_workload_identity" {
-  binding {
-    role = "roles/iam.workloadIdentityUser"
-
-    members = [
-      "serviceAccount:${local.project_id}.svc.id.goog[${local.pod_namespace}/${local.cluster_sa_name}]",
-    ]
-  }
-}
-resource "google_service_account_iam_policy" "prow_build_cluster_sa_iam" {
-  service_account_id = google_service_account.prow_build_cluster_sa.name
-  policy_data        = data.google_iam_policy.prow_build_cluster_sa_workload_identity.policy_data
-}
-
-// workload_identity_service_account: gcb-builder
-// description: used to trigger GCB builds in all k8s-staging projects
-resource "google_service_account" "gcb_builder_sa" {
-  project      = local.project_id
-  account_id   = local.gcb_builder_sa_name
-  display_name = local.gcb_builder_sa_name
+module "prow_build_cluster_sa" {
+  source            = "../modules/workload-identity-service-account"
+  project_id        = local.project_id
+  name              = local.cluster_sa_name
+  description       = "default service account for pods in ${local.cluster_name}"
+  cluster_namespace = local.pod_namespace
 }
-data "google_iam_policy" "gcb_builder_sa_workload_identity" {
-  binding {
-    role = "roles/iam.workloadIdentityUser"
+// roles: none
 
-    members = [
-      "serviceAccount:${local.project_id}.svc.id.goog[${local.pod_namespace}/${local.gcb_builder_sa_name}]",
-    ]
-  }
-}
-resource "google_service_account_iam_policy" "gcb_builder_sa_iam" {
-  service_account_id = google_service_account.gcb_builder_sa.name
-  policy_data        = data.google_iam_policy.gcb_builder_sa_workload_identity.policy_data
+module "gcb_builder_sa" {
+  source            = "../modules/workload-identity-service-account"
+  project_id        = local.project_id
+  name              = local.gcb_builder_sa_name
+  description       = "trigger GCB builds in all k8s-staging projects"
+  cluster_namespace = local.pod_namespace
 }
 // roles: come from ensure-staging-storage.sh
 
-// workload_identity_service_account: prow-deployer
-// description: used to deploy k8s resources to k8s clusters
-resource "google_service_account" "prow_deployer_sa" {
-  project      = local.project_id
-  account_id   = local.prow_deployer_sa_name
-  display_name = local.prow_deployer_sa_name
-}
-data "google_iam_policy" "prow_deployer_sa_workload_identity" {
-  binding {
-    role = "roles/iam.workloadIdentityUser"
-
-    members = [
-      "serviceAccount:${local.project_id}.svc.id.goog[${local.pod_namespace}/${local.prow_deployer_sa_name}]",
-    ]
-  }
-}
-resource "google_service_account_iam_policy" "prow_deployer_sa_iam" {
-  service_account_id = google_service_account.prow_deployer_sa.name
-  policy_data        = data.google_iam_policy.prow_deployer_sa_workload_identity.policy_data
+module "prow_deployer_sa" {
+  source            = "../modules/workload-identity-service-account"
+  project_id        = local.project_id
+  name              = local.prow_deployer_sa_name
+  description       = "deploys k8s resources to k8s clusters"
+  cluster_namespace = local.pod_namespace
 }
-// roles: prow-deployer (there are also some assigned in ensure-main-project.sh)
+// roles: there are also some assigned in ensure-main-project.sh
 resource "google_project_iam_member" "prow_deployer_for_prow_build_trusted" {
   project = local.project_id
   role    = "roles/container.admin"
@@ -141,84 +99,48 @@ resource "google_project_iam_member" "prow_deployer_for_prow_build" {
   member  = "serviceAccount:${local.prow_deployer_sa_name}@${local.project_id}.iam.gserviceaccount.com"
 }
 
-// workload_identity_service_account: k8s-metrics
-resource "google_service_account" "k8s_metrics_sa" {
-  project      = local.project_id
-  account_id   = local.k8s_metrics_sa_name
-  display_name = local.k8s_metrics_sa_name
-}
-data "google_iam_policy" "k8s_metrics_sa_workload_identity" {
-  binding {
-    role = "roles/iam.workloadIdentityUser"
-    members = [
-      "serviceAccount:${local.project_id}.svc.id.goog[${local.pod_namespace}/${local.k8s_metrics_sa_name}]",
-    ]
-  }
-}
-resource "google_service_account_iam_policy" "k8s_metrics_sa_iam" {
-  service_account_id = google_service_account.k8s_metrics_sa.name
-  policy_data        = data.google_iam_policy.k8s_metrics_sa_workload_identity.policy_data
+module "k8s_metrics_sa" {
+  source            = "../modules/workload-identity-service-account"
+  project_id        = local.project_id
+  name              = local.k8s_metrics_sa_name
+  description       = "read bigquery and write to gs://k8s-metrics"
+  cluster_namespace = local.pod_namespace
 }
-// roles: k8s-metrics
+// roles
 resource "google_project_iam_member" "k8s_metrics_sa_bigquery_user" {
   project = local.project_id
   role    = "roles/bigquery.user"
-  member  = "serviceAccount:${google_service_account.k8s_metrics_sa.email}"
+  member  = "serviceAccount:${module.k8s_metrics_sa.email}"
 }
 
-// workload_identity_service_account: triage
-resource "google_service_account" "k8s_triage_sa" {
-  project      = local.project_id
-  account_id   = local.k8s_triage_sa_name
-  display_name = local.k8s_triage_sa_name
-}
-data "google_iam_policy" "k8s_triage_sa_workload_identity" {
-  binding {
-    role = "roles/iam.workloadIdentityUser"
-    members = [
-      "serviceAccount:${local.project_id}.svc.id.goog[${local.pod_namespace}/${local.k8s_triage_sa_name}]",
-    ]
-  }
-}
-resource "google_service_account_iam_policy" "k8s_triage_sa_iam" {
-  service_account_id = google_service_account.k8s_triage_sa.name
-  policy_data        = data.google_iam_policy.k8s_triage_sa_workload_identity.policy_data
+module "k8s_triage_sa" {
+  source            = "../modules/workload-identity-service-account"
+  project_id        = local.project_id
+  name              = local.k8s_triage_sa_name
+  description       = "read bigquery and write to gs://k8s-triage"
+  cluster_namespace = local.pod_namespace
 }
-// roles: triage
+// roles
 resource "google_project_iam_member" "k8s_triage_sa_bigquery_user" {
   project = local.project_id
   role    = "roles/bigquery.user"
-  member  = "serviceAccount:${google_service_account.k8s_triage_sa.email}"
+  member  = "serviceAccount:${module.k8s_triage_sa.email}"
 }
 
-// workload_identity_service_account: kubernetes-external-secrets
-// description: used by kubernetes-external-secrets to read specific secrets in this and other projects
-resource "google_service_account" "kubernetes_external_secrets_sa" {
-  project      = local.project_id
-  account_id   = "kubernetes-external-secrets"
-  display_name = "kubernetes-external-secrets"
-}
-data "google_iam_policy" "kubernetes_external_secrets_sa_workload_identity" {
-  binding {
-    role = "roles/iam.workloadIdentityUser"
-
-    members = [
-      "serviceAccount:${local.project_id}.svc.id.goog[kubernetes-external-secrets/kubernetes-external-secrets]",
-    ]
-  }
+module "kubernetes_external_secrets_sa" {
+  source            = "../modules/workload-identity-service-account"
+  project_id        = local.project_id
+  name              = local.kubernetes_external_secrets_sa_name
+  description       = "sync K8s secrets from GSM in this and other projects"
+  cluster_namespace = "kubernetes-external-secrets"
 }
-resource "google_service_account_iam_policy" "kubernetes_external_secrets_sa_iam" {
-  service_account_id = google_service_account.kubernetes_external_secrets_sa.name
-  policy_data        = data.google_iam_policy.kubernetes_external_secrets_sa_workload_identity.policy_data
-}
-// roles: kubernetes-external-secrets
+// roles
 resource "google_project_iam_member" "kubernetes_external_secrets_for_prow_build_trusted" {
   project = local.project_id
   role    = "roles/secretmanager.secretAccessor"
-  member  = "serviceAccount:${google_service_account.kubernetes_external_secrets_sa.email}"
+  member  = "serviceAccount:${module.kubernetes_external_secrets_sa.email}"
 }
 
-
 // external (regional) ip addresses
 resource "google_compute_address" "kubernetes_external_secrets_metrics_address" {
   name         = "kubernetes-external-secrets-metrics"
@@ -237,26 +159,26 @@ resource "google_compute_address" "ghproxy_metrics_address" {
 }
 
 module "prow_build_cluster" {
-  source = "../../../modules/gke-cluster"
-  project_name      = local.project_id
-  cluster_name      = local.cluster_name
-  cluster_location  = local.cluster_location
-  bigquery_location = local.bigquery_location
-  is_prod_cluster   = "true"
-  release_channel   = "REGULAR"
-  dns_cache_enabled = "true"
+  source             = "../modules/gke-cluster"
+  project_name       = local.project_id
+  cluster_name       = local.cluster_name
+  cluster_location   = local.cluster_location
+  bigquery_location  = local.bigquery_location
+  is_prod_cluster    = "true"
+  release_channel    = "REGULAR"
+  dns_cache_enabled  = "true"
   cloud_shell_access = false
 }
 
 module "prow_build_nodepool" {
-  source = "../../../modules/gke-nodepool"
-  project_name    = local.project_id
-  cluster_name    = module.prow_build_cluster.cluster.name
-  location        = module.prow_build_cluster.cluster.location
-  name            = "trusted-pool1"
-  initial_count   = 1
-  min_count       = 1
-  max_count       = 6
+  source        = "../modules/gke-nodepool"
+  project_name  = local.project_id
+  cluster_name  = module.prow_build_cluster.cluster.name
+  location      = module.prow_build_cluster.cluster.location
+  name          = "trusted-pool1"
+  initial_count = 1
+  min_count     = 1
+  max_count     = 6
   # image/machine/disk match prow-build for consistency's sake
   image_type      = "UBUNTU_CONTAINERD"
   machine_type    = "n1-highmem-8"

diff --git a/infra/gcp/terraform/k8s-infra-prow-build-trusted/versions.tf b/infra/gcp/terraform/k8s-infra-prow-build-trusted/versions.tf
@@ -13,7 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
- 
+
 /*
 This file defines:
 - Required Terraform version

diff --git a/infra/gcp/terraform/k8s-infra-prow-build/00-provider.tf b/infra/gcp/terraform/k8s-infra-prow-build/00-provider.tf
@@ -13,7 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
- 
+
 /*
 This file defines:
 - Required provider versions