diff --git a/infra/gcp/clusters/projects/k8s-infra-prow-build-trusted/prow-build-trusted/main.tf b/infra/gcp/clusters/projects/k8s-infra-prow-build-trusted/prow-build-trusted/main.tf index e86716046d9..dbaaec65937 100644 --- a/infra/gcp/clusters/projects/k8s-infra-prow-build-trusted/prow-build-trusted/main.tf +++ b/infra/gcp/clusters/projects/k8s-infra-prow-build-trusted/prow-build-trusted/main.tf @@ -53,13 +53,25 @@ resource "google_project_iam_member" "k8s_infra_prow_oncall" { member = "group:k8s-infra-prow-oncall@kubernetes.io" } -// Create GCP SA for pods +// TODO: consider making this a real module to reduce copy-pasta +// +// The "workload_identity_service_account" comment denotes a pseudo-module of +// copy-pasted resources, similar to "ensure_workload_identity_serviceaccount" +// in ensure-main-project.sh. +// +// It is a shorthand for making a Kubernetes Service Account bound to a GCP +// Service Account of the same name, and optionally assigning it IAM roles. +// Some of the roles are assigned in bash or other terraform modules, so as +// to keep the permissions necessary to run this terraform module scoped to +// "roles/owner" for local.project_id + +// workload_identity_service_account: prow-build-trusted +// description: intended as default service account for pods in this cluster resource "google_service_account" "prow_build_cluster_sa" { project = local.project_id account_id = local.cluster_sa_name display_name = "Used by pods in '${local.cluster_name}' GKE cluster" } -// Allow pods using the build cluster KSA to use the GCP SA via workload identity data "google_iam_policy" "prow_build_cluster_sa_workload_identity" { binding { role = "roles/iam.workloadIdentityUser" @@ -69,19 +81,18 @@ data "google_iam_policy" "prow_build_cluster_sa_workload_identity" { ] } } -// Authoritative iam-policy: replaces any existing policy attached to this service_account resource "google_service_account_iam_policy" "prow_build_cluster_sa_iam" { service_account_id = google_service_account.prow_build_cluster_sa.name policy_data = data.google_iam_policy.prow_build_cluster_sa_workload_identity.policy_data } -// Create GCP SA for jobs that use GCB and push results to GCS +// workload_identity_service_account: gcb-builder +// description: used to trigger GCB builds in all k8s-staging projects resource "google_service_account" "gcb_builder_sa" { project = local.project_id account_id = local.gcb_builder_sa_name display_name = local.gcb_builder_sa_name } -// Allow pods using the build cluster KSA to use the GCP SA via workload identity data "google_iam_policy" "gcb_builder_sa_workload_identity" { binding { role = "roles/iam.workloadIdentityUser" @@ -91,19 +102,19 @@ data "google_iam_policy" "gcb_builder_sa_workload_identity" { ] } } -// Authoritative iam-policy: replaces any existing policy attached to this service_account resource "google_service_account_iam_policy" "gcb_builder_sa_iam" { service_account_id = google_service_account.gcb_builder_sa.name policy_data = data.google_iam_policy.gcb_builder_sa_workload_identity.policy_data } +// roles: come from ensure-staging-storage.sh -// Create GCP SA for jobs that deploy to k8s-infra prow clusters +// workload_identity_service_account: prow-deployer +// description: used to deploy k8s resources to k8s clusters resource "google_service_account" "prow_deployer_sa" { project = local.project_id account_id = local.prow_deployer_sa_name display_name = local.prow_deployer_sa_name } -// Allow pods using the build cluster KSA to use the GCP SA via workload identity data "google_iam_policy" "prow_deployer_sa_workload_identity" { binding { role = "roles/iam.workloadIdentityUser" @@ -113,51 +124,54 @@ data "google_iam_policy" "prow_deployer_sa_workload_identity" { ] } } -// Authoritative iam-policy: replaces any existing policy attached to this service_account resource "google_service_account_iam_policy" "prow_deployer_sa_iam" { service_account_id = google_service_account.prow_deployer_sa.name policy_data = data.google_iam_policy.prow_deployer_sa_workload_identity.policy_data } -// Create GCP SA for jobs that write to gs://k8s-metrics and gs://k8s-project-metrics +// roles: prow-deployer (there are also some assigned in ensure-main-project.sh) +resource "google_project_iam_member" "prow_deployer_for_prow_build_trusted" { + project = local.project_id + role = "${data.google_organization.org.name}/roles/container.deployer" + member = "serviceAccount:${local.prow_deployer_sa_name}@${local.project_id}.iam.gserviceaccount.com" +} +resource "google_project_iam_member" "prow_deployer_for_prow_build" { + project = "k8s-infra-prow-build" + role = "${data.google_organization.org.name}/roles/container.deployer" + member = "serviceAccount:${local.prow_deployer_sa_name}@${local.project_id}.iam.gserviceaccount.com" +} + +// workload_identity_service_account: k8s-metrics resource "google_service_account" "k8s_metrics_sa" { project = local.project_id account_id = local.k8s_metrics_sa_name display_name = local.k8s_metrics_sa_name } -// Allow pods using the build cluster KSA to use the GCP SA via workload identity data "google_iam_policy" "k8s_metrics_sa_workload_identity" { binding { role = "roles/iam.workloadIdentityUser" - members = [ "serviceAccount:${local.project_id}.svc.id.goog[${local.pod_namespace}/${local.k8s_metrics_sa_name}]", ] } } -// Authoritative iam-policy: replaces any existing policy attached to this service_account resource "google_service_account_iam_policy" "k8s_metrics_sa_iam" { service_account_id = google_service_account.k8s_metrics_sa.name policy_data = data.google_iam_policy.k8s_metrics_sa_workload_identity.policy_data } - -resource "google_project_iam_member" "prow_deployer_for_prow_build_trusted" { +// roles: k8s-metrics +resource "google_project_iam_member" "k8s_metrics_sa_bigquery_user" { project = local.project_id - role = "${data.google_organization.org.name}/roles/container.deployer" - member = "serviceAccount:${local.prow_deployer_sa_name}@${local.project_id}.iam.gserviceaccount.com" -} -resource "google_project_iam_member" "prow_deployer_for_prow_build" { - project = "k8s-infra-prow-build" - role = "${data.google_organization.org.name}/roles/container.deployer" - member = "serviceAccount:${local.prow_deployer_sa_name}@${local.project_id}.iam.gserviceaccount.com" + role = "roles/bigquery.user" + member = "serviceAccount:${google_service_account.k8s_metrics_sa.email}" } -// Create GCP SA for kubernetes-external-secrets +// workload_identity_service_account: kubernetes-external-secrets +// description: used by kubernetes-external-secrets to read specific secrets in this and other projects resource "google_service_account" "kubernetes_external_secrets_sa" { project = local.project_id account_id = "kubernetes-external-secrets" display_name = "kubernetes-external-secrets" } -// Allow kubernetes-external-secrets pods to use the GCP SA data "google_iam_policy" "kubernetes_external_secrets_sa_workload_identity" { binding { role = "roles/iam.workloadIdentityUser" @@ -167,11 +181,11 @@ data "google_iam_policy" "kubernetes_external_secrets_sa_workload_identity" { ] } } -// Authoritative iam-policy: replaces any existing policy attached to this service_account resource "google_service_account_iam_policy" "kubernetes_external_secrets_sa_iam" { service_account_id = google_service_account.kubernetes_external_secrets_sa.name policy_data = data.google_iam_policy.kubernetes_external_secrets_sa_workload_identity.policy_data } +// roles: kubernetes-external-secrets resource "google_project_iam_member" "kubernetes_external_secrets_for_prow_build_trusted" { project = local.project_id role = "roles/secretmanager.secretAccessor" @@ -179,6 +193,7 @@ resource "google_project_iam_member" "kubernetes_external_secrets_for_prow_build } +// external (regional) ip addresses resource "google_compute_address" "kubernetes_external_secrets_metrics_address" { name = "kubernetes-external-secrets-metrics" description = "to allow monitoring.k8s.prow.io to scrape kubernetes-external-secrets metrics" diff --git a/infra/gcp/ensure-main-project.sh b/infra/gcp/ensure-main-project.sh index 3e5eede1866..b523925405f 100755 --- a/infra/gcp/ensure-main-project.sh +++ b/infra/gcp/ensure-main-project.sh @@ -397,17 +397,17 @@ function ensure_prow_special_cases { fi local project="${1}" - local bucket principal secret + local principal secret color 6 "Special case: ensuring k8s-infra-ci-robot-github-token accessible by k8s-infra-prow-build-trusted" principal="serviceAccount:$(svc_acct_email "k8s-infra-prow-build-trusted" "kubernetes-external-secrets")" secret=$(secret_full_name "${project}" "k8s-infra-ci-robot-github-token") ensure_secret_role_binding "${secret}" "${principal}" "roles/secretmanager.secretAccessor" 2>&1 | indent - color 6 "Special case: ensuring gs://k8s-project-metrics exists for gs://k8s-metrics migration" + color 6 "Special case: ensuring gs://k8s-metrics exists" ( - bucket="gs://k8s-project-metrics" - owners="k8s-infra-prow-oncall@kubernetes.io" + local bucket="gs://k8s-metrics" + local owners="k8s-infra-prow-oncall@kubernetes.io" local old_service_account="triage@k8s-gubernator.iam.gserviceaccount.com" ensure_public_gcs_bucket "${project}" "${bucket}" @@ -416,15 +416,20 @@ function ensure_prow_special_cases { empower_gcs_admins "${project}" "${bucket}" # bucket owners can admin this bucket empower_group_to_admin_gcs_bucket "${owners}" "${bucket}" - # TODO(spiffxp): copy pasted to flip to ensure_removed when migrated - # k8s-prow-builds can write to this bucket + # TODO(spiffxp): remove once bindings have been removed + # k8s-prow-builds can no longer write to this bucket principal="serviceAccount:${old_service_account}" - ensure_gcs_role_binding "${bucket}" "${principal}" "objectAdmin" - ensure_gcs_role_binding "${bucket}" "${principal}" "legacyBucketWriter" + ensure_removed_gcs_role_binding "${bucket}" "${principal}" "objectAdmin" + ensure_removed_gcs_role_binding "${bucket}" "${principal}" "legacyBucketWriter" # k8s-infra-prow-build-trusted can write to this bucket principal="serviceAccount:$(svc_acct_email "k8s-infra-prow-build-trusted" "k8s-metrics")" ensure_gcs_role_binding "${bucket}" "${principal}" "objectAdmin" ensure_gcs_role_binding "${bucket}" "${principal}" "legacyBucketWriter" + # TODO(spiffxp): this is a test to confirm we _can_ charge bigquery usage elsewhere + # and might prove convenient since there are datasets in this project, + # but this should probably not be the long-term home of usage billing + # k8s-infra-prow-build-trusted can charge bigquery usage to this project + ensure_project_role_binding "${project}" "${principal}" "roles/bigquery.user" ) 2>&1 | indent }