Skip to content

Commit

Permalink
Merge pull request #1737 from spiffxp/ensure-organization-followup
Browse files Browse the repository at this point in the history
ensure-organization.sh followup
  • Loading branch information
k8s-ci-robot authored Mar 11, 2021
2 parents c7048a1 + 4e5eeed commit 4295316
Show file tree
Hide file tree
Showing 18 changed files with 371 additions and 223 deletions.
3 changes: 2 additions & 1 deletion infra/gcp/clusters/modules/gke-project/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,9 @@ that is intended to host a GKE cluster created by the [`gke-cluster`] module:
- Project is linked to CNCF billing account
- Services necessary to support [`gke-cluster`] are enabled
- Some default IAM bindings are added:
- k8s-infra-cluster-admins@ gets `roles/compute.viewer`, `roles/container.admin`, `roles/ServiceAccountLister`
- k8s-infra-cluster-admins@ gets `roles/compute.viewer`, `roles/container.admin`, org role [`ServiceAccountLister`]
- gke-security-groups@ gets `roles/container.clusterViewer`

[`gke-cluster`]: /infra/gcp/clusters/modules/gke-cluster
[`gke-nodepool`]: /infra/gcp/clusters/modules/gke-nodepool
[`ServiceAccountLister`]: /infra/gcp/roles/iam.serviceAccountLister.yaml
38 changes: 28 additions & 10 deletions infra/gcp/clusters/modules/gke-project/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,31 @@
* limitations under the License.
*/

/// Create the project in which we're creating the cluster
// hardcoded org and billing account, not sure we want this
// reusable outside of k8s-infra
locals {
org_domain = "kubernetes.io"
billing_account = "018801-93540E-22A20E"
}

data "google_organization" "org" {
domain = local.org_domain
}

// TODO(spiffxp): explicitly not using a data source for this until
// I have a better sense of whether this requires more permissions
// than (are / should be) available for k8s-infra-prow-oncall and
// k8s-infra-cluster-admins
// data google_billing_account {
// billing_account = locals.billing_account
// }

// Create the project in which we're creating the cluster
resource "google_project" "project" {
name = var.project_name
project_id = var.project_name
org_id = "758905017065" // kubernetes.io
billing_account = "018801-93540E-22A20E"
org_id = data.google_organization.org.org_id
billing_account = local.billing_account
}

// Services we need
Expand Down Expand Up @@ -91,16 +110,15 @@ resource "google_project_iam_member" "cluster_admins_as_container_admin" {
role = "roles/container.admin"
member = "group:${var.cluster_admins_group}"
}
resource "google_project_iam_custom_role" "service_account_lister" {
project = google_project.project.project_id
role_id = "ServiceAccountLister"
title = "Service Account Lister"
description = "Can list ServiceAccounts."
permissions = ["iam.serviceAccounts.list"]

// Role created by infra/gcp/ensure-organization.sh, use a data source to ensure it exists
data "google_iam_role" "service_account_lister" {
name = "${data.google_organization.org.name}/roles/iam.serviceAccountLister"
}

resource "google_project_iam_member" "cluster_admins_as_service_account_lister" {
project = google_project.project.project_id
role = "projects/${google_project.project.project_id}/roles/${google_project_iam_custom_role.service_account_lister.role_id}"
role = data.google_iam_role.service_account_lister.name
member = "group:${var.cluster_admins_group}"
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,10 @@ locals {
boskos_janitor_sa_name = "boskos-janitor" // Name of the GSA and KSA used by boskos-janitor
}

data "google_organization" "org" {
domain = "kubernetes.io"
}

module "project" {
source = "../../../modules/gke-project"
project_id = local.project_id
Expand All @@ -46,11 +50,15 @@ resource "google_project_iam_member" "k8s_infra_prow_oncall" {
member = "group:[email protected]"
}

// Role created by infra/gcp/ensure-organization.sh, use a data source to ensure it exists
data "google_iam_role" "prow_viewer" {
name = "${data.google_organization.org.name}/roles/prow.viewer"
}

// Ensure [email protected] has prow.viewer access to this project
resource "google_project_iam_member" "k8s_infra_prow_viewers" {
project = local.project_id
# TODO: use data resource to get org role name instead of hardcode
role = "organizations/758905017065/roles/prow.viewer"
role = data.google_iam_role.prow_viewer.name
member = "group:[email protected]"
}

Expand Down
31 changes: 14 additions & 17 deletions infra/gcp/ensure-main-project.sh
Original file line number Diff line number Diff line change
Expand Up @@ -90,23 +90,20 @@ gcloud projects add-iam-policy-binding "${PROJECT}" \
--role roles/bigquery.admin

color 6 "Empowering cluster admins"
gcloud projects add-iam-policy-binding "${PROJECT}" \
--member "group:${CLUSTER_ADMINS_GROUP}" \
--role roles/compute.viewer
gcloud projects add-iam-policy-binding "${PROJECT}" \
--member "group:${CLUSTER_ADMINS_GROUP}" \
--role roles/container.admin
gcloud projects add-iam-policy-binding "${PROJECT}" \
--member "group:${CLUSTER_ADMINS_GROUP}" \
--role roles/compute.loadBalancerAdmin
ensure_custom_iam_role "${PROJECT}" \
ServiceAccountLister \
"Service Account Lister" \
"Can list ServiceAccounts." \
iam.serviceAccounts.list
gcloud projects add-iam-policy-binding "${PROJECT}" \
--member "group:${CLUSTER_ADMINS_GROUP}" \
--role "projects/${PROJECT}/roles/ServiceAccountLister"
# TODO: this can also be a custom role
cluster_admin_roles=(
roles/compute.viewer
roles/container.admin
roles/compute.loadBalancerAdmin
$(custom_org_role_name iam.serviceAccountLister)
)
for role in "${cluster_admin_roles[@]}"; do
ensure_project_role_binding "${PROJECT}" "group:${CLUSTER_ADMINS_GROUP}" "${role}"
done
# TODO(spiffxp): remove when bindings for custom project role are gone
ensure_removed_project_role_binding "${PROJECT}" "group:${CLUSTER_ADMINS_GROUP}" "$(custom_project_role_name "${PROJECT}" ServiceAccountLister)"
ensure_removed_project_role "${PROJECT}" "ServiceAccountLister"

gsutil iam ch \
"group:${CLUSTER_ADMINS_GROUP}:objectAdmin" \
"gs://${CLUSTER_TERRAFORM_BUCKET}"
Expand Down
66 changes: 34 additions & 32 deletions infra/gcp/ensure-organization.sh
Original file line number Diff line number Diff line change
Expand Up @@ -40,18 +40,32 @@ org_roles=(
secretmanager.secretLister
organization.admin
CustomRole
iam.serviceAccountLister
)

old_org_roles=(
StorageBucketLister
)

# TODO(https://github.com/kubernetes/k8s.io/issues/1659): obviated by organization.admin, remove when bindings gone
old_org_admin_roles=(
roles/billing.user
roles/iam.organizationRoleAdmin
roles/resourcemanager.organizationAdmin
roles/resourcemanager.projectCreator
roles/resourcemanager.projectDeleter
roles/servicemanagement.quotaAdmin
)

color 6 "Ensuring organization custom roles exist"
(
for role in "${org_roles[@]}"; do
color 6 "Ensuring organization custom role ${role}"
ensure_custom_iam_role_from_file "org" "${role}" "${SCRIPT_DIR}/roles/${role}.yaml"
ensure_custom_org_iam_role_from_file "${role}" "${SCRIPT_DIR}/roles/${role}.yaml"
done
) 2>&1 | indent

color 6 "Ensuring org-level IAM bindings exist"
color 6 "Ensuring organization IAM bindings exist"
(
# [email protected] should be able to browse org resources
ensure_org_role_binding "group:[email protected]" "roles/browser"
Expand All @@ -65,42 +79,30 @@ color 6 "Ensuring org-level IAM bindings exist"

# k8s-infra-gcp-auditors@
ensure_org_role_binding "group:[email protected]" "$(custom_org_role_name "audit.viewer")"
# TODO(https://github.com/kubernetes/k8s.io/issues/1659): obviated by audit.viewer, remove when bindings gone
old_audit_roles=(
"$(custom_org_role_name "StorageBucketLister")"
roles/compute.viewer
roles/dns.reader
roles/iam.securityReviewer
roles/resourcemanager.organizationViewer
roles/serviceusage.serviceUsageConsumer
)
for role in "${old_audit_roles[@]}"; do
ensure_removed_org_role_binding "group:[email protected]" "${role}"
done

echo "exiting early to confirm audit.viewer role migration has worked"
exit 0

# k8s-infra-org-admins@
# roles/owner has too many permissions to aggregate into a custom role,
# and some services (e.g. storage) add bindings based on membership in it
ensure_org_role_binding "group:[email protected]" "roles/owner"
# everything org admins need beyond roles/owner to manage the org
ensure_org_role_binding "group:[email protected]" "$(custom_org_role_name "organization.admin")"
# TODO(https://github.com/kubernetes/k8s.io/issues/1659): obviated by organization.admin, remove when bindings gone
old_org_admin_roles=(
roles/billing.user
roles/iam.organizationRoleAdmin
roles/resourcemanager.organizationAdmin
roles/resourcemanager.projectCreator
roles/resourcemanager.projectDeleter
roles/servicemanagement.quotaAdmin
)
for role in "${old_audit_roles[@]}"; do
# TODO(spiffxp): remove the extra super duper paranoia once we verify
# I haven't locked myself out via group membership
ensure_org_role_binding "user:[email protected]" "${role}"
ensure_org_role_binding "user:[email protected]" "${role}"
ensure_removed_org_role_binding "group:[email protected]" "${role}"
) 2>&1 | indent

color 6 "Ensuring removed organization IAM bindings do not exist"
(
# TODO(spiffxp): remove this once the old bindings are confirmed gone
for role in "${old_org_admin_roles[@]}"; do
ensure_removed_org_role_binding "user:[email protected]" "${role}"
ensure_removed_org_role_binding "user:[email protected]" "${role}"
done
) 2>&1 | indent

color 6 "Ensuring removed organization custom roles do not exist"
(
for role in "${old_org_roles[@]}"; do
color 6 "Ensuring removed organization custom role ${role}"
ensure_removed_custom_org_iam_role "${role}"
done
) 2>&1 | indent

color 6 "All done!"
Loading

0 comments on commit 4295316

Please sign in to comment.