Merge pull request #2451 from spiffxp/k8s-metrics-cleanup

infra/gcp: k8s-metrics move cleanup

infra/gcp/clusters/projects/k8s-infra-prow-build-trusted/prow-build-trusted/main.tf

@@ -53,13 +53,25 @@ resource "google_project_iam_member" "k8s_infra_prow_oncall" {
   member  = "group:[email protected]"
 }
 
-// Create GCP SA for pods
+// TODO: consider making this a real module to reduce copy-pasta
+// 
+// The "workload_identity_service_account" comment denotes a pseudo-module of
+// copy-pasted resources, similar to "ensure_workload_identity_serviceaccount"
+// in ensure-main-project.sh.
+//
+// It is a shorthand for making a Kubernetes Service Account bound to a GCP
+// Service Account of the same name, and optionally assigning it IAM roles.
+// Some of the roles are assigned in bash or other terraform modules, so as
+// to keep the permissions necessary to run this terraform module scoped to
+// "roles/owner" for local.project_id
+
+// workload_identity_service_account: prow-build-trusted
+// description: intended as default service account for pods in this cluster
 resource "google_service_account" "prow_build_cluster_sa" {
   project      = local.project_id
   account_id   = local.cluster_sa_name
   display_name = "Used by pods in '${local.cluster_name}' GKE cluster"
 }
-// Allow pods using the build cluster KSA to use the GCP SA via workload identity
 data "google_iam_policy" "prow_build_cluster_sa_workload_identity" {
   binding {
     role = "roles/iam.workloadIdentityUser"
@@ -69,19 +81,18 @@ data "google_iam_policy" "prow_build_cluster_sa_workload_identity" {
     ]
   }
 }
-// Authoritative iam-policy: replaces any existing policy attached to this service_account
 resource "google_service_account_iam_policy" "prow_build_cluster_sa_iam" {
   service_account_id = google_service_account.prow_build_cluster_sa.name
   policy_data        = data.google_iam_policy.prow_build_cluster_sa_workload_identity.policy_data
 }
 
-// Create GCP SA for jobs that use GCB and push results to GCS
+// workload_identity_service_account: gcb-builder
+// description: used to trigger GCB builds in all k8s-staging projects
 resource "google_service_account" "gcb_builder_sa" {
   project      = local.project_id
   account_id   = local.gcb_builder_sa_name
   display_name = local.gcb_builder_sa_name
 }
-// Allow pods using the build cluster KSA to use the GCP SA via workload identity
 data "google_iam_policy" "gcb_builder_sa_workload_identity" {
   binding {
     role = "roles/iam.workloadIdentityUser"
@@ -91,19 +102,19 @@ data "google_iam_policy" "gcb_builder_sa_workload_identity" {
     ]
   }
 }
-// Authoritative iam-policy: replaces any existing policy attached to this service_account
 resource "google_service_account_iam_policy" "gcb_builder_sa_iam" {
   service_account_id = google_service_account.gcb_builder_sa.name
   policy_data        = data.google_iam_policy.gcb_builder_sa_workload_identity.policy_data
 }
+// roles: come from ensure-staging-storage.sh
 
-// Create GCP SA for jobs that deploy to k8s-infra prow clusters
+// workload_identity_service_account: prow-deployer
+// description: used to deploy k8s resources to k8s clusters
 resource "google_service_account" "prow_deployer_sa" {
   project      = local.project_id
   account_id   = local.prow_deployer_sa_name
   display_name = local.prow_deployer_sa_name
 }
-// Allow pods using the build cluster KSA to use the GCP SA via workload identity
 data "google_iam_policy" "prow_deployer_sa_workload_identity" {
   binding {
     role = "roles/iam.workloadIdentityUser"
@@ -113,51 +124,54 @@ data "google_iam_policy" "prow_deployer_sa_workload_identity" {
     ]
   }
 }
-// Authoritative iam-policy: replaces any existing policy attached to this service_account
 resource "google_service_account_iam_policy" "prow_deployer_sa_iam" {
   service_account_id = google_service_account.prow_deployer_sa.name
   policy_data        = data.google_iam_policy.prow_deployer_sa_workload_identity.policy_data
 }
-// Create GCP SA for jobs that write to gs://k8s-metrics and gs://k8s-project-metrics
+// roles: prow-deployer (there are also some assigned in ensure-main-project.sh)
+resource "google_project_iam_member" "prow_deployer_for_prow_build_trusted" {
+  project = local.project_id
+  role    = "${data.google_organization.org.name}/roles/container.deployer"
+  member  = "serviceAccount:${local.prow_deployer_sa_name}@${local.project_id}.iam.gserviceaccount.com"
+}
+resource "google_project_iam_member" "prow_deployer_for_prow_build" {
+  project = "k8s-infra-prow-build"
+  role    = "${data.google_organization.org.name}/roles/container.deployer"
+  member  = "serviceAccount:${local.prow_deployer_sa_name}@${local.project_id}.iam.gserviceaccount.com"
+}
+
+// workload_identity_service_account: k8s-metrics
 resource "google_service_account" "k8s_metrics_sa" {
   project      = local.project_id
   account_id   = local.k8s_metrics_sa_name
   display_name = local.k8s_metrics_sa_name
 }
-// Allow pods using the build cluster KSA to use the GCP SA via workload identity
 data "google_iam_policy" "k8s_metrics_sa_workload_identity" {
   binding {
     role = "roles/iam.workloadIdentityUser"
-
     members = [
       "serviceAccount:${local.project_id}.svc.id.goog[${local.pod_namespace}/${local.k8s_metrics_sa_name}]",
     ]
   }
 }
-// Authoritative iam-policy: replaces any existing policy attached to this service_account
 resource "google_service_account_iam_policy" "k8s_metrics_sa_iam" {
   service_account_id = google_service_account.k8s_metrics_sa.name
   policy_data        = data.google_iam_policy.k8s_metrics_sa_workload_identity.policy_data
 }
-
-resource "google_project_iam_member" "prow_deployer_for_prow_build_trusted" {
+// roles: k8s-metrics
+resource "google_project_iam_member" "k8s_metrics_sa_bigquery_user" {
   project = local.project_id
-  role    = "${data.google_organization.org.name}/roles/container.deployer"
-  member  = "serviceAccount:${local.prow_deployer_sa_name}@${local.project_id}.iam.gserviceaccount.com"
-}
-resource "google_project_iam_member" "prow_deployer_for_prow_build" {
-  project = "k8s-infra-prow-build"
-  role    = "${data.google_organization.org.name}/roles/container.deployer"
-  member  = "serviceAccount:${local.prow_deployer_sa_name}@${local.project_id}.iam.gserviceaccount.com"
+  role    = "roles/bigquery.user"
+  member  = "serviceAccount:${google_service_account.k8s_metrics_sa.email}"
 }
 
-// Create GCP SA for kubernetes-external-secrets
+// workload_identity_service_account: kubernetes-external-secrets
+// description: used by kubernetes-external-secrets to read specific secrets in this and other projects
 resource "google_service_account" "kubernetes_external_secrets_sa" {
   project      = local.project_id
   account_id   = "kubernetes-external-secrets"
   display_name = "kubernetes-external-secrets"
 }
-// Allow kubernetes-external-secrets pods to use the GCP SA
 data "google_iam_policy" "kubernetes_external_secrets_sa_workload_identity" {
   binding {
     role = "roles/iam.workloadIdentityUser"
@@ -167,18 +181,19 @@ data "google_iam_policy" "kubernetes_external_secrets_sa_workload_identity" {
     ]
   }
 }
-// Authoritative iam-policy: replaces any existing policy attached to this service_account
 resource "google_service_account_iam_policy" "kubernetes_external_secrets_sa_iam" {
   service_account_id = google_service_account.kubernetes_external_secrets_sa.name
   policy_data        = data.google_iam_policy.kubernetes_external_secrets_sa_workload_identity.policy_data
 }
+// roles: kubernetes-external-secrets
 resource "google_project_iam_member" "kubernetes_external_secrets_for_prow_build_trusted" {
   project = local.project_id
   role    = "roles/secretmanager.secretAccessor"
   member  = "serviceAccount:${google_service_account.kubernetes_external_secrets_sa.email}"
 }
 
 
+// external (regional) ip addresses
 resource "google_compute_address" "kubernetes_external_secrets_metrics_address" {
   name         = "kubernetes-external-secrets-metrics"
   description  = "to allow monitoring.k8s.prow.io to scrape kubernetes-external-secrets metrics"

infra/gcp/ensure-main-project.sh

@@ -397,17 +397,17 @@ function ensure_prow_special_cases {
     fi
     local project="${1}"
 
-    local bucket principal secret
+    local principal secret
 
     color 6 "Special case: ensuring k8s-infra-ci-robot-github-token accessible by k8s-infra-prow-build-trusted"
     principal="serviceAccount:$(svc_acct_email "k8s-infra-prow-build-trusted" "kubernetes-external-secrets")"
     secret=$(secret_full_name "${project}" "k8s-infra-ci-robot-github-token")
     ensure_secret_role_binding "${secret}" "${principal}" "roles/secretmanager.secretAccessor" 2>&1 | indent
 
-    color 6 "Special case: ensuring gs://k8s-project-metrics exists for gs://k8s-metrics migration"
+    color 6 "Special case: ensuring gs://k8s-metrics exists"
     (
-      bucket="gs://k8s-project-metrics"
-      owners="[email protected]"
+      local bucket="gs://k8s-metrics"
+      local owners="[email protected]"
       local old_service_account="[email protected]"
 
       ensure_public_gcs_bucket "${project}" "${bucket}"
@@ -416,15 +416,20 @@ function ensure_prow_special_cases {
       empower_gcs_admins "${project}" "${bucket}"
       # bucket owners can admin this bucket
       empower_group_to_admin_gcs_bucket "${owners}" "${bucket}"
-      # TODO(spiffxp): copy pasted to flip to ensure_removed when migrated
-      # k8s-prow-builds can write to this bucket
+      # TODO(spiffxp): remove once bindings have been removed
+      # k8s-prow-builds can no longer write to this bucket
       principal="serviceAccount:${old_service_account}"
-      ensure_gcs_role_binding "${bucket}" "${principal}" "objectAdmin"
-      ensure_gcs_role_binding "${bucket}" "${principal}" "legacyBucketWriter"
+      ensure_removed_gcs_role_binding "${bucket}" "${principal}" "objectAdmin"
+      ensure_removed_gcs_role_binding "${bucket}" "${principal}" "legacyBucketWriter"
       # k8s-infra-prow-build-trusted can write to this bucket
       principal="serviceAccount:$(svc_acct_email "k8s-infra-prow-build-trusted" "k8s-metrics")"
       ensure_gcs_role_binding "${bucket}" "${principal}" "objectAdmin"
       ensure_gcs_role_binding "${bucket}" "${principal}" "legacyBucketWriter"
+      # TODO(spiffxp): this is a test to confirm we _can_ charge bigquery usage elsewhere
+      #                and might prove convenient since there are datasets in this project,
+      #                but this should probably not be the long-term home of usage billing
+      # k8s-infra-prow-build-trusted can charge bigquery usage to this project
+      ensure_project_role_binding "${project}" "${principal}" "roles/bigquery.user"
     ) 2>&1 | indent
 }