From 306fd98947d8148f54ae8200e06c27779ad70c10 Mon Sep 17 00:00:00 2001
From: Aaron Crickenberger <spiffxp@gmail.com>
Date: Sat, 24 Apr 2021 12:44:21 -0400
Subject: [PATCH] infra/gcp/main: refactor into functions

specifically:

- refactored by:
  - grouping blocks of code into functions
  - parameterizing those functions on PROJECT
  - adding ensure_workload_identity_serviceaccount to handle the sundry
    single-role service accounts intended for use by GKE clusters
- prune main_project_services down to just what we directly use
  - drop compute: direct dep of container
  - drop oslogin: direct dep of compute
  - swap bigquery-json for biquery: the former doesn't exist
  - add comments explaning each of the services we do use
- fix a bunch of shellcheck nits
- add some TODO(spiffxp) lines for role or binding removal as followup
---
 infra/gcp/ensure-main-project.sh | 518 ++++++++++++++++++-------------
 1 file changed, 306 insertions(+), 212 deletions(-)

diff --git a/infra/gcp/ensure-main-project.sh b/infra/gcp/ensure-main-project.sh
index 721cccdde9b..5c27e314d87 100755
--- a/infra/gcp/ensure-main-project.sh
+++ b/infra/gcp/ensure-main-project.sh
@@ -34,249 +34,343 @@ if [ $# != 0 ]; then
 fi
 
 # The GCP project name.
-PROJECT="kubernetes-public"
+readonly PROJECT="kubernetes-public"
 
 # The BigQuery dataset for billing data.
-BQ_BILLING_DATASET="kubernetes_public_billing"
+readonly BQ_BILLING_DATASET="kubernetes_public_billing"
 
 # The BigQuery admins group.
-BQ_ADMINS_GROUP="k8s-infra-bigquery-admins@kubernetes.io"
+readonly BQ_ADMINS_GROUP="k8s-infra-bigquery-admins@kubernetes.io"
 
 # The cluster admins group.
-CLUSTER_ADMINS_GROUP="k8s-infra-cluster-admins@kubernetes.io"
+readonly CLUSTER_ADMINS_GROUP="k8s-infra-cluster-admins@kubernetes.io"
 
 # The accounting group.
-ACCOUNTING_GROUP="k8s-infra-gcp-accounting@kubernetes.io"
+readonly ACCOUNTING_GROUP="k8s-infra-gcp-accounting@kubernetes.io"
 
 # The GCS bucket which hold terraform state for clusters
-LEGACY_CLUSTER_TERRAFORM_BUCKET="k8s-infra-clusters-terraform"
+readonly LEGACY_CLUSTER_TERRAFORM_BUCKET="k8s-infra-clusters-terraform"
 
 # The GKE security groups group
-CLUSTER_USERS_GROUP="gke-security-groups@kubernetes.io"
+readonly CLUSTER_USERS_GROUP="gke-security-groups@kubernetes.io"
 
 # The DNS admins group.
-DNS_GROUP="k8s-infra-dns-admins@kubernetes.io"
+readonly DNS_GROUP="k8s-infra-dns-admins@kubernetes.io"
 
-color 6 "Ensuring project exists: ${PROJECT}"
-ensure_project "${PROJECT}"
-
-# Enable APIs we know we need
-apis=(
-    bigquery-json.googleapis.com
-    compute.googleapis.com
-    container.googleapis.com
-    dns.googleapis.com
-    logging.googleapis.com
-    monitoring.googleapis.com
-    oslogin.googleapis.com
-    secretmanager.googleapis.com
-    storage-component.googleapis.com
-)
-ensure_only_services "${PROJECT}" "${apis[@]}"
-
-# buckets to hold terraform state
+# GCS buckets to hold terraform state
+#
 # - since we are using uniform bucket level access (ubla), each bucket should
 #   represent a logical group of access, with org admins given storage.admin
 #   for break-glass purposes
 # - the legacy bucket (k8s-infra-clusters-terraform) assumed the same set of
-#   users should have access to all gke clusters
+#   users should have access to all gke clusters (~all terraform-based infra)
 # - new bucket schema is "k8s-infra-tf-{folder}[-{suffix}]" where:
 #   - folder: intended GCP folder for GCP projects managed by this terraform,
 #             access level is ~owners of folder
 #   - suffix: some subset of resources contained somewhere underneath folder,
 #             access level is ~editors of those resources
 # - entry syntax is "bucket_name:owners_group" (: is invalid bucket name char)
-terraform_state_bucket_entries=(
-  "${LEGACY_CLUSTER_TERRAFORM_BUCKET}:${CLUSTER_ADMINS_GROUP}"
-  k8s-infra-tf-aws:k8s-infra-aws-admins@kubernetes.io
-  k8s-infra-tf-prow-clusters:k8s-infra-prow-oncall@kubernetes.io
-  k8s-infra-tf-public-clusters:"${CLUSTER_ADMINS_GROUP}"
-  k8s-infra-tf-sandbox-ii:k8s-infra-ii-coop@kubernetes.io
+readonly TERRAFORM_STATE_BUCKET_ENTRIES=(
+    "${LEGACY_CLUSTER_TERRAFORM_BUCKET}:${CLUSTER_ADMINS_GROUP}"
+    k8s-infra-tf-aws:k8s-infra-aws-admins@kubernetes.io
+    k8s-infra-tf-prow-clusters:k8s-infra-prow-oncall@kubernetes.io
+    k8s-infra-tf-public-clusters:"${CLUSTER_ADMINS_GROUP}"
+    k8s-infra-tf-sandbox-ii:k8s-infra-ii-coop@kubernetes.io
 )
-color 6 "Ensuring terraform state buckets exist with correct permissions"
-for entry in "${terraform_state_bucket_entries[@]}"; do
-    bucket="gs://$(echo "${entry}" | cut -d: -f1)"
-    owners="$(echo "${entry}" | cut -d: -f2-)"
-    color 6 "Ensuring '${bucket}' exists as private with owners '${owners}'"; (
-        ensure_private_gcs_bucket "${PROJECT}" "${bucket}"
-        empower_group_to_admin_gcs_bucket "${owners}" "${bucket}"
-        ensure_gcs_role_binding "${bucket}" "group:k8s-infra-gcp-org-admins@kubernetes.io" "admin"
-    ) 2>&1 | indent
-done 2>&1 | indent
-
-color 6 "Empowering BigQuery admins"
-ensure_project_role_binding \
-    "${PROJECT}" \
-    "group:${BQ_ADMINS_GROUP}" \
-    "roles/bigquery.admin"
-
-color 6 "Empowering cluster admins"
-# TODO: this can also be a custom role
-cluster_admin_roles=(
-    roles/compute.viewer
-    roles/container.admin
-    roles/compute.loadBalancerAdmin
-    $(custom_org_role_name iam.serviceAccountLister)
+
+# The services we explicitly want enabled for the main project
+#
+# NOTE: Expected services include dependencies of these services, which may be
+#       more than direct dependencies. Do we care to statically encode the
+#       graph here? ensure_only_services dynamically computes the set of
+#       expected services
+readonly MAIN_PROJECT_SERVICES=(
+    # billing data gets exported to bigquery
+    bigquery.googleapis.com
+    # GKE clusters are hosted in this project
+    container.googleapis.com
+    # DNS zones are managed in this project
+    dns.googleapis.com
+    # We look at logs in this project (e.g. from GKE)
+    logging.googleapis.com
+    # We look at monitoring dashboards in this project
+    monitoring.googleapis.com
+    # Secrets are hosted in this project
+    secretmanager.googleapis.com
+    # GCS buckets are hosted in this project
+    storage-component.googleapis.com
 )
-for role in "${cluster_admin_roles[@]}"; do
-    ensure_project_role_binding "${PROJECT}" "group:${CLUSTER_ADMINS_GROUP}" "${role}"
-done
-# TODO(spiffxp): remove when bindings for custom project role are gone
-ensure_removed_project_role_binding "${PROJECT}" "group:${CLUSTER_ADMINS_GROUP}" "$(custom_project_role_name "${PROJECT}" ServiceAccountLister)"
-ensure_removed_custom_project_iam_role "${PROJECT}" "ServiceAccountLister"
-
-
-color 6 "Empowering cluster users"
-ensure_project_role_binding \
-    "${PROJECT}" \
-    "group:${CLUSTER_USERS_GROUP}" \
-    "roles/container.clusterViewer"
-
-color 6 "Empowering GCP accounting"
-ensure_project_role_binding \
-  "${PROJECT}" \
-  "group:${ACCOUNTING_GROUP}" \
-  "roles/bigquery.jobUser"
-
-color 6 "Ensuring the k8s-infra-gcp-auditor serviceaccount exists"
-ensure_service_account \
-  "${PROJECT}" \
-  "k8s-infra-gcp-auditor" \
-  "Grants readonly access to org resources"
-
-color 6 "Empowering k8s-infra-gcp-auditor serviceaccount to be used on trusted build cluster"
-empower_ksa_to_svcacct \
-  "k8s-infra-prow-build-trusted.svc.id.goog[test-pods/k8s-infra-gcp-auditor]" \
-  "${PROJECT}" \
-  $(svc_acct_email "${PROJECT}" "k8s-infra-gcp-auditor")
-# TODO(spiffxp): delete this binding
-empower_ksa_to_svcacct \
-  "kubernetes-public.svc.id.goog[test-pods/k8s-infra-gcp-auditor]" \
-  "${PROJECT}" \
-  $(svc_acct_email "${PROJECT}" "k8s-infra-gcp-auditor")
-
-color 6 "Ensuring the k8s-infra-dns-updater serviceaccount exists"
-ensure_service_account \
-    "${PROJECT}" \
-    "k8s-infra-dns-updater" \
-    "k8s-infra dns updater"
-
-color 6 "Empowering k8s-infra-dns-updater serviceaccount to be used on build cluster"
-empower_ksa_to_svcacct \
-    "k8s-infra-prow-build-trusted.svc.id.goog[test-pods/k8s-infra-dns-updater]" \
-    "${PROJECT}" \
-    "$(svc_acct_email "${PROJECT}" "k8s-infra-dns-updater")"
-
-color 6 "Empowering ${DNS_GROUP}"
-color 6 "Empowering BigQuery admins"
-ensure_project_role_binding \
-    "${PROJECT}" \
-    "group:${DNS_GROUP}" \
-    "roles/dns.admin"
-
-# Monitoring
-MONITORING_SVCACCT_NAME="$(svc_acct_email "${PROJECT}" \
-    "k8s-infra-monitoring-viewer")"
-
-color 6 "Ensuring the k8s-infra-monitoring-viewer serviceaccount exists"
-ensure_service_account \
-    "${PROJECT}" \
-    "k8s-infra-monitoring-viewer" \
-    "k8s-infra monitoring viewer"
-
-color 6 "Empowering k8s-infra-monitoring-viewer serviceaccount to be used on the 'aaa' cluster inside the 'monitoring' namespace"
-empower_ksa_to_svcacct \
-    "kubernetes-public.svc.id.goog[monitoring/k8s-infra-monitoring-viewer]" \
-    "${PROJECT}" \
-    "${MONITORING_SVCACCT_NAME}"
-
-color 6 "Empowering service account ${MONITORING_SVCACCT_NAME}"
-ensure_project_role_binding \
-    "${PROJECT}" \
-    "serviceAccount:${MONITORING_SVCACCT_NAME}" \
-    "roles/monitoring.viewer"
-
-# Kubernetes External Secrets
-color 6 -n "Ensure the kubernetes-external-secrets serviceaccount exists"
-ensure_service_account \
-    "${PROJECT}" \
-    "kubernetes-external-secrets" \
-    "Kubernetes External Secrets Service Account"
-
-color 6 -n "Empowering kubernetes-external-secrets serviceaccount to be used on aaa cluster"
-empower_ksa_to_svcacct \
-    "kubernetes-public.svc.id.goog[kubernetes-external-secrets/kubernetes-external-secrets]" \
-    "${PROJECT}" \
-    "$(svc_acct_email "${PROJECT}" "kubernetes-external-secrets")"
-
-color 6 "Empowering service account kubernetes-external-services to access payloads of the secrets"
-ensure_project_role_binding "${PROJECT}" \
-    "serviceAccount:$(svc_acct_email "${PROJECT}" "kubernetes-external-secrets")" \
-    roles/secretmanager.secretAccessor
-
-# Bootstrap DNS zones
-ensure_dns_zone "${PROJECT}" "k8s-io" "k8s.io"
-ensure_dns_zone "${PROJECT}" "kubernetes-io" "kubernetes.io"
-ensure_dns_zone "${PROJECT}" "x-k8s-io" "x-k8s.io"
-ensure_dns_zone "${PROJECT}" "k8s-e2e-com" "k8s-e2e.com"
-ensure_dns_zone "${PROJECT}" "canary-k8s-io" "canary.k8s.io"
-ensure_dns_zone "${PROJECT}" "canary-kubernetes-io" "canary.kubernetes.io"
-ensure_dns_zone "${PROJECT}" "canary-x-k8s-io" "canary.x-k8s.io"
-ensure_dns_zone "${PROJECT}" "canary-k8s-e2e-com" "canary.k8s-e2e.com"
-
-color 6 "Creating the BigQuery dataset for billing data"
-if ! bq --project_id "${PROJECT}" ls "${BQ_BILLING_DATASET}" >/dev/null 2>&1; then
-    bq --project_id "${PROJECT}" mk "${BQ_BILLING_DATASET}"
-fi
 
-color 6 "Setting BigQuery permissions"
+# Create a GCP service account intended for use by GKE cluster workloads
+# $1: The GCP project hosting the service account (e.g. k8s-infra-foo)
+# $2: The name of the GCP service account (e.g. k8s-infra-doer)
+# $3: The IAM role the service account will have on the project (e.g. roles/foo.bar)
+# $4: The GCP project hosting GKE cluster(s) that will use this service account (e.g. k8s-infra-foo-clusters)
+# $5: The K8s namespace hosting the service account that will use this GCP SA (e.g. my-app-ns)
+#
+# e.g. the above allows pods running as my-app-ns/k8s-infra-doer in clusters in
+#      k8s-infra-foo-clusters to act as k8s-infra-doer@k8s-infra-foo.iam.gserviceaccount.com,
+#      using the permissions granted by roles/foo.bar on k8s-infra-foo
+function ensure_workload_identity_serviceaccount() {
+    if [ $# -lt 5 ] || [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ] || [ -z "$4" ] || [ -z "$5" ]; then
+        echo "${FUNCNAME[0]}(project, name, role, cluster_project, cluster_namespace) requires 5 arguments" >&2
+        return 1
+    fi
+    local serviceaccount_project="${1}"
+    local serviceaccount_name="${2}"
+    local serviceaccount_email
+    serviceaccount_email="$(svc_acct_email "${serviceaccount_project}" "${serviceaccount_name}")"
+    local serviceaccount_description="${serviceaccount_name}"
+    local serviceaccount_role="${3}"
+    local cluster_project="${4}"
+    local cluster_namespace="${5}"
+
+    color 6 "Ensuring serviceaccount ${serviceaccount_email} exists"
+    ensure_service_account "${serviceaccount_project}" "${serviceaccount_name}" "${serviceaccount_description}"
+
+    color 6 "Empowering ${serviceaccount_email} with role ${serviceaccount_role} in project ${serviceaccount_project}"
+    ensure_project_role_binding \
+        "${serviceaccount_project}" \
+        "serviceAccount:${serviceaccount_email}" \
+        "${serviceaccount_role}"
+
+    local clusters="clusters in project ${cluster_project}"
+    local k8s_sa="KSA ${cluster_namespace}/${serviceaccount_name}"
+    local gcp_sa="GCP SA ${serviceaccount_email}"
+
+    color 6 "Empowering ${clusters} to run workloads as ${k8s_sa} to act as ${gcp_sa}"
+    empower_gke_for_serviceaccount "${cluster_project}" "${cluster_namespace}" "${serviceaccount_email}"
+}
+
+#
+# main functions
+#
 
-# Merge existing permissions with the ones we need to exist.  We merge
-# permissions because:
-#   * The full list is large and has stuff that is inherited listed in it
-#   * All of our other IAM binding logic calls are additive
+function ensure_terraform_state_buckets() {
+    if [ $# -ne 1 ] || [ -z "$1" ]; then
+        echo "${FUNCNAME[0]}(gcp_project) requires 1 argument" >&2
+        return 1
+    fi
+    local project="${1}"
+
+    for entry in "${TERRAFORM_STATE_BUCKET_ENTRIES[@]}"; do
+        bucket="gs://$(echo "${entry}" | cut -d: -f1)"
+        owners="$(echo "${entry}" | cut -d: -f2-)"
+        color 6 "Ensuring '${bucket}' exists as private with owners '${owners}'"; (
+            ensure_private_gcs_bucket "${project}" "${bucket}"
+            empower_group_to_admin_gcs_bucket "${owners}" "${bucket}"
+            ensure_gcs_role_binding "${bucket}" "group:k8s-infra-gcp-org-admins@kubernetes.io" "admin"
+        ) 2>&1 | indent
+    done
+}
 
-CUR=${TMPDIR}/k8s-infra-bq-access.before.json
-bq show --format=prettyjson "${PROJECT}":"${BQ_BILLING_DATASET}"  > "${CUR}"
+function empower_cluster_admins_and_users() {
+    if [ $# -ne 1 ] || [ -z "$1" ]; then
+        echo "${FUNCNAME[0]}(gcp_project) requires 1 argument" >&2
+        return 1
+    fi
+    local project="${1}"
+
+    color 6 "Empowering cluster admins"
+    # TODO(spiffxp): make this a custom role
+    cluster_admin_roles=(
+        roles/compute.viewer
+        roles/container.admin
+        roles/compute.loadBalancerAdmin
+        "$(custom_org_role_name iam.serviceAccountLister)"
+    )
+    for role in "${cluster_admin_roles[@]}"; do
+        ensure_project_role_binding "${project}" "group:${CLUSTER_ADMINS_GROUP}" "${role}"
+    done
+    # TODO(spiffxp): remove when these bindings have been removed
+    removed_cluster_admin_roles=(
+        "$(custom_project_role_name "${project}" ServiceAccountLister)"
+    )
+    for role in "${removed_cluster_admin_roles[@]}"; do
+        ensure_removed_project_role_binding "${project}" "group:${CLUSTER_ADMINS_GROUP}" "${role}"
+    done
+    # TODO(spiffxp): ensure this is removed when the binding(s) using it have been removed
+    ensure_removed_custom_project_iam_role "${project}" "ServiceAccountLister"
+
+    color 6 "Empowering cluster users"
+    ensure_project_role_binding \
+        "${project}" \
+        "group:${CLUSTER_USERS_GROUP}" \
+        "roles/container.clusterViewer"
+}
 
-ENSURE=${TMPDIR}/k8s-infra-bq-access.ensure.json
-cat > "${ENSURE}" << __EOF__
-{
-  "access": [
-    {
-      "groupByEmail": "${ACCOUNTING_GROUP}",
-      "role": "READER"
-    },
-    {
-      "groupByEmail": "${ACCOUNTING_GROUP}",
-      "role": "roles/bigquery.metadataViewer"
-    },
+
+function ensure_dns() {
+    if [ $# -ne 1 ] || [ -z "$1" ]; then
+        echo "${FUNCNAME[0]}(gcp_project) requires 1 argument" >&2
+        return 1
+    fi
+    local project="${1}"
+
+    local domains=(
+        k8s.io
+        kubernetes.io
+        x-k8s.io
+        k8s-e2e.com
+    )
+
+    color 6 "Empowering ${DNS_GROUP}"
+    ensure_project_role_binding \
+        "${project}" \
+        "group:${DNS_GROUP}" \
+        "roles/dns.admin"
+
+    # Bootstrap DNS zones
+    for domain in "${domains[@]}"; do
+        # canary domain for each domain, e.g. k8s.io and canary.k8s.io
+        for prefix in "" "canary."; do
+            zone="${prefix}${domain}"
+            name="${zone//./-}"
+            color 6 "Ensuring DNS zone ${zone}"
+            ensure_dns_zone "${project}" "${name}" "${zone}" 2>&1 | indent
+        done
+    done
+}
+
+function ensure_billing_bigquery() {
+    if [ $# -ne 1 ] || [ -z "$1" ]; then
+        echo "${FUNCNAME[0]}(gcp_project) requires 1 argument" >&2
+        return 1
+    fi
+    local project="${1}"
+
+    color 6 "Empowering BigQuery admins"
+    ensure_project_role_binding \
+        "${project}" \
+        "group:${BQ_ADMINS_GROUP}" \
+        "roles/bigquery.admin"
+
+    color 6 "Empowering GCP accounting"
+    ensure_project_role_binding \
+        "${project}" \
+        "group:${ACCOUNTING_GROUP}" \
+        "roles/bigquery.jobUser"
+
+    color 6 "Creating the BigQuery dataset for billing data"
+    if ! bq --project_id "${project}" ls "${BQ_BILLING_DATASET}" >/dev/null 2>&1; then
+        bq --project_id "${project}" mk "${BQ_BILLING_DATASET}"
+    fi
+
+    color 6 "Setting BigQuery permissions"
+    # Merge existing permissions with the ones we need to exist.  We merge
+    # permissions because:
+    #   * The full list is large and has stuff that is inherited listed in it
+    #   * All of our other IAM binding logic calls are additive
+    local before="${TMPDIR}/k8s-infra-bq-access.before.json"
+    local ensure="${TMPDIR}/k8s-infra-bq-access.ensure.json"
+    local final="${TMPDIR}/k8s-infra-bq-access.final.json"
+
+    bq show --format=prettyjson "${project}":"${BQ_BILLING_DATASET}"  > "${before}"
+
+    cat > "${ensure}" <<EOF
     {
-      "groupByEmail": "${ACCOUNTING_GROUP}",
-      "role": "roles/bigquery.user"
+      "access": [
+        {
+          "groupByEmail": "${ACCOUNTING_GROUP}",
+          "role": "READER"
+        },
+        {
+          "groupByEmail": "${ACCOUNTING_GROUP}",
+          "role": "roles/bigquery.metadataViewer"
+        },
+        {
+          "groupByEmail": "${ACCOUNTING_GROUP}",
+          "role": "roles/bigquery.user"
+        }
+      ]
     }
-  ]
+EOF
+
+    jq -s '.[0].access + .[1].access | { access: . }' "${before}" "${ensure}" > "${final}"
+
+    bq update --source "${final}" "${project}":"${BQ_BILLING_DATASET}"
+
+    color 4 "To enable billing export, a human must log in to the cloud"
+    color 4 -n "console.  Go to "
+    color 6 -n "Billing"
+    color 4 -n "; "
+    color 6 -n "Billing export"
+    color 4 " and export to BigQuery"
+    color 4 -n "in project "
+    color 6 -n "${project}"
+    color 4 -n " dataset "
+    color 6 -n "${BQ_BILLING_DATASET}"
+    color 4 " ."
+    echo
+    color 4 "Press enter to acknowledge"
+    read -rs
 }
-__EOF__
-
-FINAL=${TMPDIR}/k8s-infra-bq-access.final.json
-jq -s '.[0].access + .[1].access | { access: . }' "${CUR}" "${ENSURE}" > "${FINAL}"
-
-bq update --source "${FINAL}" "${PROJECT}":"${BQ_BILLING_DATASET}"
-
-color 4 "To enable billing export, a human must log in to the cloud"
-color 4 -n "console.  Go to "
-color 6 -n "Billing"
-color 4 -n "; "
-color 6 -n "Billing export"
-color 4 " and export to BigQuery"
-color 4 -n "in project "
-color 6 -n "${PROJECT}"
-color 4 -n " dataset "
-color 6 -n "${BQ_BILLING_DATASET}"
-color 4 " ."
-echo
-color 4 "Press enter to acknowledge"
-read -s
-
-color 6 "Done"
+
+function ensure_main_project() {
+    if [ $# -ne 1 ] || [ -z "$1" ]; then
+        echo "${FUNCNAME[0]}(gcp_project) requires 1 argument" >&2
+        return 1
+    fi
+    local project="${1}"
+
+    color 6 "Ensuring project exists: ${project}"
+    ensure_project "${project}" 2>&1 | indent
+
+    color 6 "Ensuring main project services enabled for: ${project}"
+    # TODO: there are many services that appear to have been enabled piecemeal
+    #       and it's not yet clear which ones we actually rely upon, so this is
+    #       not being run with K8S_INFRA_ENSURE_ONLY_SERVICES_WILL_FORCE_DISABLE
+    ensure_only_services "${project}" "${MAIN_PROJECT_SERVICES[@]}" 2>&1 | indent
+
+    color 6 "Ensuring terraform state buckets exist with correct permissions in: ${project}"
+    ensure_terraform_state_buckets "${project}" 2>&1 | indent
+
+    color 6 "Empowering cluster users and admins for clusters in: ${project}"
+    empower_cluster_admins_and_users "${project}" 2>&1 | indent
+
+    color 6 "Ensuring specific workload identity serviceaccounts exist in: ${project}"; (
+        local svcacct_args cluster_args
+
+        color 6 "Ensuring GCP Auditor serviceaccount"
+        # roles/viewer on kubernetes-public is a bootstrap; the true purpose
+        # is custom role audit.viewer on the kubernetes.io org, but that is
+        # handled by ensure-organization.sh
+        svcacct_args=("${project}" "k8s-infra-gcp-auditor" "roles/viewer")
+        cluster_args=("k8s-infra-prow-build-trusted" "test-pods")
+        ensure_workload_identity_serviceaccount "${svcacct_args[@]}" "${cluster_args[@]}" 2>&1 | indent
+
+        # TODO(spiffxp): remove once this binding has been deleted
+        local gcp_auditor_email
+        gcp_auditor_email=$(svc_acct_email "${project}" "k8s-infra-gcp-auditor")
+        color 6 "Ensuring removed workload identity on kubernetes-public/test-pods for ${gcp_auditor_email}"
+        unempower_gke_for_serviceaccount \
+            "kubernetes-public" \
+            "test-pods" \
+            "${gcp_auditor_email}" 2>&1 | indent
+
+        color 6 "Ensuring DNS Updater serviceaccount"
+        svcacct_args=("${project}" "k8s-infra-dns-updater" "roles/dns.admin")
+        cluster_args=("k8s-infra-prow-build-trusted" "test-pods")
+        ensure_workload_identity_serviceaccount "${svcacct_args[@]}" "${cluster_args[@]}" 2>&1 | indent
+
+        color 6 "Ensuring Monitoring Viewer serviceaccount"
+        svcacct_args=("${project}" "k8s-infra-monitoring-viewer" "roles/monitoring.viewer")
+        cluster_args=("${project}" "monitoring")
+        ensure_workload_identity_serviceaccount "${svcacct_args[@]}" "${cluster_args[@]}" 2>&1 | indent
+
+        color 6 "Ensuring Kubernetes External Secrets serviceaccount"
+        svcacct_args=("${project}" "kubernetes-external-secrets" "roles/secretmanager.secretAccessor")
+        cluster_args=("${project}" "kubernetes-external-secrets")
+        ensure_workload_identity_serviceaccount "${svcacct_args[@]}" "${cluster_args[@]}" 2>&1 | indent
+    ) 2>&1 | indent
+
+    color 6 "Ensuring DNS is configured in: ${project}"
+    ensure_dns "${project}" 2>&1 | indent
+
+    color 6 "Ensuring biquery configured for billing and access by appropriate groups in: ${project}"
+    ensure_billing_bigquery "${project}" 2>&1 | indent
+
+    color 6 "Done"
+}
+
+ensure_main_project "${PROJECT}"