diff --git a/infra/gcp/clusters/modules/gke-project/README.md b/infra/gcp/clusters/modules/gke-project/README.md index 61bc1975e9f..615372fb7bb 100644 --- a/infra/gcp/clusters/modules/gke-project/README.md +++ b/infra/gcp/clusters/modules/gke-project/README.md @@ -6,8 +6,9 @@ that is intended to host a GKE cluster created by the [`gke-cluster`] module: - Project is linked to CNCF billing account - Services necessary to support [`gke-cluster`] are enabled - Some default IAM bindings are added: - - k8s-infra-cluster-admins@ gets `roles/compute.viewer`, `roles/container.admin`, `roles/ServiceAccountLister` + - k8s-infra-cluster-admins@ gets `roles/compute.viewer`, `roles/container.admin`, org role [`ServiceAccountLister`] - gke-security-groups@ gets `roles/container.clusterViewer` [`gke-cluster`]: /infra/gcp/clusters/modules/gke-cluster [`gke-nodepool`]: /infra/gcp/clusters/modules/gke-nodepool +[`ServiceAccountLister`]: /infra/gcp/roles/iam.serviceAccountLister.yaml diff --git a/infra/gcp/clusters/modules/gke-project/main.tf b/infra/gcp/clusters/modules/gke-project/main.tf index c82d9b429ff..d3d8b200fc2 100644 --- a/infra/gcp/clusters/modules/gke-project/main.tf +++ b/infra/gcp/clusters/modules/gke-project/main.tf @@ -14,12 +14,31 @@ * limitations under the License. */ -/// Create the project in which we're creating the cluster +// hardcoded org and billing account, not sure we want this +// reusable outside of k8s-infra +locals { + org_domain = "kubernetes.io" + billing_account = "018801-93540E-22A20E" +} + +data "google_organization" "org" { + domain = local.org_domain +} + +// TODO(spiffxp): explicitly not using a data source for this until +// I have a better sense of whether this requires more permissions +// than (are / should be) available for k8s-infra-prow-oncall and +// k8s-infra-cluster-admins +// data google_billing_account { +// billing_account = locals.billing_account +// } + +// Create the project in which we're creating the cluster resource "google_project" "project" { name = var.project_name project_id = var.project_name - org_id = "758905017065" // kubernetes.io - billing_account = "018801-93540E-22A20E" + org_id = data.google_organization.org.org_id + billing_account = local.billing_account } // Services we need @@ -91,16 +110,15 @@ resource "google_project_iam_member" "cluster_admins_as_container_admin" { role = "roles/container.admin" member = "group:${var.cluster_admins_group}" } -resource "google_project_iam_custom_role" "service_account_lister" { - project = google_project.project.project_id - role_id = "ServiceAccountLister" - title = "Service Account Lister" - description = "Can list ServiceAccounts." - permissions = ["iam.serviceAccounts.list"] + +// Role created by infra/gcp/ensure-organization.sh, use a data source to ensure it exists +data "google_iam_role" "service_account_lister" { + name = "${data.google_organization.org.name}/roles/iam.serviceAccountLister" } + resource "google_project_iam_member" "cluster_admins_as_service_account_lister" { project = google_project.project.project_id - role = "projects/${google_project.project.project_id}/roles/${google_project_iam_custom_role.service_account_lister.role_id}" + role = data.google_iam_role.service_account_lister.name member = "group:${var.cluster_admins_group}" } diff --git a/infra/gcp/clusters/projects/k8s-infra-prow-build/prow-build/main.tf b/infra/gcp/clusters/projects/k8s-infra-prow-build/prow-build/main.tf index c62e6d6789f..1238379243b 100644 --- a/infra/gcp/clusters/projects/k8s-infra-prow-build/prow-build/main.tf +++ b/infra/gcp/clusters/projects/k8s-infra-prow-build/prow-build/main.tf @@ -33,6 +33,10 @@ locals { boskos_janitor_sa_name = "boskos-janitor" // Name of the GSA and KSA used by boskos-janitor } +data "google_organization" "org" { + domain = "kubernetes.io" +} + module "project" { source = "../../../modules/gke-project" project_id = local.project_id @@ -46,11 +50,15 @@ resource "google_project_iam_member" "k8s_infra_prow_oncall" { member = "group:k8s-infra-prow-oncall@kubernetes.io" } +// Role created by infra/gcp/ensure-organization.sh, use a data source to ensure it exists +data "google_iam_role" "prow_viewer" { + name = "${data.google_organization.org.name}/roles/prow.viewer" +} + // Ensure k8s-infra-prow-viewers@kuberentes.io has prow.viewer access to this project resource "google_project_iam_member" "k8s_infra_prow_viewers" { project = local.project_id - # TODO: use data resource to get org role name instead of hardcode - role = "organizations/758905017065/roles/prow.viewer" + role = data.google_iam_role.prow_viewer.name member = "group:k8s-infra-prow-viewers@kubernetes.io" } diff --git a/infra/gcp/ensure-main-project.sh b/infra/gcp/ensure-main-project.sh index 7cb20544dfd..71a679b2915 100755 --- a/infra/gcp/ensure-main-project.sh +++ b/infra/gcp/ensure-main-project.sh @@ -90,23 +90,20 @@ gcloud projects add-iam-policy-binding "${PROJECT}" \ --role roles/bigquery.admin color 6 "Empowering cluster admins" -gcloud projects add-iam-policy-binding "${PROJECT}" \ - --member "group:${CLUSTER_ADMINS_GROUP}" \ - --role roles/compute.viewer -gcloud projects add-iam-policy-binding "${PROJECT}" \ - --member "group:${CLUSTER_ADMINS_GROUP}" \ - --role roles/container.admin -gcloud projects add-iam-policy-binding "${PROJECT}" \ - --member "group:${CLUSTER_ADMINS_GROUP}" \ - --role roles/compute.loadBalancerAdmin -ensure_custom_iam_role "${PROJECT}" \ - ServiceAccountLister \ - "Service Account Lister" \ - "Can list ServiceAccounts." \ - iam.serviceAccounts.list -gcloud projects add-iam-policy-binding "${PROJECT}" \ - --member "group:${CLUSTER_ADMINS_GROUP}" \ - --role "projects/${PROJECT}/roles/ServiceAccountLister" +# TODO: this can also be a custom role +cluster_admin_roles=( + roles/compute.viewer + roles/container.admin + roles/compute.loadBalancerAdmin + $(custom_org_role_name iam.serviceAccountLister) +) +for role in "${cluster_admin_roles[@]}"; do + ensure_project_role_binding "${PROJECT}" "group:${CLUSTER_ADMINS_GROUP}" "${role}" +done +# TODO(spiffxp): remove when bindings for custom project role are gone +ensure_removed_project_role_binding "${PROJECT}" "group:${CLUSTER_ADMINS_GROUP}" "$(custom_project_role_name "${PROJECT}" ServiceAccountLister)" +ensure_removed_project_role "${PROJECT}" "ServiceAccountLister" + gsutil iam ch \ "group:${CLUSTER_ADMINS_GROUP}:objectAdmin" \ "gs://${CLUSTER_TERRAFORM_BUCKET}" diff --git a/infra/gcp/ensure-organization.sh b/infra/gcp/ensure-organization.sh index 2f628ce3dd6..3b066ef8512 100755 --- a/infra/gcp/ensure-organization.sh +++ b/infra/gcp/ensure-organization.sh @@ -40,18 +40,32 @@ org_roles=( secretmanager.secretLister organization.admin CustomRole + iam.serviceAccountLister +) + +old_org_roles=( StorageBucketLister ) +# TODO(https://github.com/kubernetes/k8s.io/issues/1659): obviated by organization.admin, remove when bindings gone +old_org_admin_roles=( + roles/billing.user + roles/iam.organizationRoleAdmin + roles/resourcemanager.organizationAdmin + roles/resourcemanager.projectCreator + roles/resourcemanager.projectDeleter + roles/servicemanagement.quotaAdmin +) + color 6 "Ensuring organization custom roles exist" ( for role in "${org_roles[@]}"; do color 6 "Ensuring organization custom role ${role}" - ensure_custom_iam_role_from_file "org" "${role}" "${SCRIPT_DIR}/roles/${role}.yaml" + ensure_custom_org_iam_role_from_file "${role}" "${SCRIPT_DIR}/roles/${role}.yaml" done ) 2>&1 | indent -color 6 "Ensuring org-level IAM bindings exist" +color 6 "Ensuring organization IAM bindings exist" ( # k8s-infra-prow-oncall@kubernetes.io should be able to browse org resources ensure_org_role_binding "group:k8s-infra-prow-oncall@kubernetes.io" "roles/browser" @@ -65,21 +79,6 @@ color 6 "Ensuring org-level IAM bindings exist" # k8s-infra-gcp-auditors@ ensure_org_role_binding "group:k8s-infra-gcp-auditors@kubernetes.io" "$(custom_org_role_name "audit.viewer")" - # TODO(https://github.com/kubernetes/k8s.io/issues/1659): obviated by audit.viewer, remove when bindings gone - old_audit_roles=( - "$(custom_org_role_name "StorageBucketLister")" - roles/compute.viewer - roles/dns.reader - roles/iam.securityReviewer - roles/resourcemanager.organizationViewer - roles/serviceusage.serviceUsageConsumer - ) - for role in "${old_audit_roles[@]}"; do - ensure_removed_org_role_binding "group:k8s-infra-gcp-auditors@kubernetes.io" "${role}" - done - - echo "exiting early to confirm audit.viewer role migration has worked" - exit 0 # k8s-infra-org-admins@ # roles/owner has too many permissions to aggregate into a custom role, @@ -87,20 +86,23 @@ color 6 "Ensuring org-level IAM bindings exist" ensure_org_role_binding "group:k8s-infra-gcp-org-admins@kubernetes.io" "roles/owner" # everything org admins need beyond roles/owner to manage the org ensure_org_role_binding "group:k8s-infra-gcp-org-admins@kubernetes.io" "$(custom_org_role_name "organization.admin")" - # TODO(https://github.com/kubernetes/k8s.io/issues/1659): obviated by organization.admin, remove when bindings gone - old_org_admin_roles=( - roles/billing.user - roles/iam.organizationRoleAdmin - roles/resourcemanager.organizationAdmin - roles/resourcemanager.projectCreator - roles/resourcemanager.projectDeleter - roles/servicemanagement.quotaAdmin - ) - for role in "${old_audit_roles[@]}"; do - # TODO(spiffxp): remove the extra super duper paranoia once we verify - # I haven't locked myself out via group membership - ensure_org_role_binding "user:thockin@google.com" "${role}" - ensure_org_role_binding "user:davanum@gmail.com" "${role}" - ensure_removed_org_role_binding "group:k8s-infra-gcp-org-admins@kubernetes.io" "${role}" +) 2>&1 | indent + +color 6 "Ensuring removed organization IAM bindings do not exist" +( + # TODO(spiffxp): remove this once the old bindings are confirmed gone + for role in "${old_org_admin_roles[@]}"; do + ensure_removed_org_role_binding "user:thockin@google.com" "${role}" + ensure_removed_org_role_binding "user:davanum@gmail.com" "${role}" done ) 2>&1 | indent + +color 6 "Ensuring removed organization custom roles do not exist" +( + for role in "${old_org_roles[@]}"; do + color 6 "Ensuring removed organization custom role ${role}" + ensure_removed_custom_org_iam_role "${role}" + done +) 2>&1 | indent + +color 6 "All done!" diff --git a/infra/gcp/lib_iam.sh b/infra/gcp/lib_iam.sh index 5e2c72d48df..c8f937c5603 100755 --- a/infra/gcp/lib_iam.sh +++ b/infra/gcp/lib_iam.sh @@ -21,113 +21,112 @@ # # This MUST NOT be used directly. Source it via lib.sh instead. -# Ensure that custom IAM role exists, creating one if needed +readonly tmp_dir=$(mktemp -d "/tmp/k8sinfra-lib_iam.XXXXX") +trap 'rm -rf "${tmp_dir}"' EXIT + +# Ensure that custom IAM role exists in organization and in sync with definition in file # Arguments: -# $1: The GCP project -# $2: The role name (e.g. "ServiceAccountLister") -# $3: The role title (e.g. "Service Account Lister") -# $4: The role description (e.g. "Can list ServiceAccounts.") -# $5+: The role permissions (e.g. "iam.serviceAccounts.list") -# Example usage: -# ensure_custom_iam_role \ -# kubernetes-public \ -# ServiceAccountLister \ -# "Service Account Lister" \ -# "Can list ServiceAccounts." \ -# iam.serviceAccounts.list -function ensure_custom_iam_role() { - if [ $# -lt 5 ] || [ -z "${1}" ] || [ -z "${2}" ] || [ -z "${3}" ] \ - || [ -z "${4}" ] || [ -z "${5}" ] - then - echo -n "ensure_custom_iam_role(gcp_project, name, title," >&2 - echo " description, permission...) requires at least 5 arguments" >&2 - return 1 - fi - - local gcp_project="${1}"; shift - local name="${1}"; shift - local title="${1}"; shift - local description="${1}"; shift - local permissions; permissions=$(join_by , "$@") - - if ! gcloud --project "${gcp_project}" iam roles describe "${name}" \ - >/dev/null 2>&1 - then - gcloud --project "${gcp_project}" --quiet \ - iam roles create "${name}" \ - --title "${title}" \ - --description "${description}" \ - --stage GA \ - --permissions "${permissions}" +# $1: The role name (e.g. "foo.barrer") +# $2: The file (e.g. "/path/to/file.yaml") +function ensure_custom_org_iam_role_from_file() { + if [ ! $# -eq 2 -o -z "$1" -o -z "$2" ]; then + echo "ensure_custom_org_iam_role_from_file(name, file) requires 2 arguments" >&2 + return 1 fi + + local organization="${GCP_ORG}" + local name="${1}" + local file="${2}" + + _ensure_custom_iam_role_from_file "organization" "${organization}" "${name}" "${file}" } -# Ensure that custom IAM role exists and is in sync with definition in file +# Ensure that custom IAM role exists in project and in sync with definition in file # Arguments: -# $1: The scope of the role (e.g. "org", "project:foobar") -# $2: The role name (e.g. "prow.viewer") +# $1: The id of the project (e.g. "k8s-infra-foo") +# $2: The role name (e.g. "foo.barrer") # $3: The file (e.g. "/path/to/file.yaml") -function ensure_custom_iam_role_from_file() { +function ensure_custom_project_iam_role_from_file() { if [ ! $# -eq 3 -o -z "$1" -o -z "$2" -o -z "$3" ]; then - echo "ensure_custom_iam_role_from_file(scope, name, file) requires 3 arguments" >&2 + echo "ensure_custom_project_iam_role_from_file(project, name, file) requires 3 arguments" >&2 return 1 fi - local scope="${1}" + local project="${1}" local name="${2}" local file="${3}" - local full_name="${name}" - scope_flag="" - if [[ "${scope}" == "org" ]]; then - scope_flag="--organization ${GCP_ORG}" - full_name="organizations/${GCP_ORG}/roles/${name}" - elif [[ "${scope}" =~ "^project:" ]]; then - project=$(echo "${scope}" | cut -d: -f2-) - scope_flag="--project ${project}" - full_name="projects/${project}/roles/${name}" - else - echo "ensure_custom_iam_role_from_file(scope, name, file) scope must be one of 'org' or 'project:project-id'" >&2 + _ensure_custom_iam_role_from_file "project" "${project}" "${name}" "${file}" +} + +# Ensure that custom IAM role has been removed from organization +# Arguments: +# $1: The role name (e.g. "foo.barrer") +function ensure_removed_custom_org_iam_role() { + if [ ! $# -eq 1 -o -z "$1" ]; then + echo "ensure_removed_custom_org_iam_role(name) requires 1 arguments" >&2 return 1 fi - tmp_dir=$(mktemp -d "/tmp/ensure-role-${name}-XXXXX") - trap 'rm -rf "${tmp_dir}"' EXIT - before="${tmp_dir}/before.${role}.yaml" - ready="${tmp_dir}/ready.${role}.yaml" - after="${tmp_dir}/after.${role}.yaml" + local organization="${GCP_ORG}" + local name="${1}" - # detect if we should create or update and dump role; silently ignore error - verb="update" - if ! (gcloud iam roles describe ${scope_flag} "${name}" >"${before}") >/dev/null 2>&1; then - verb="create" + _ensure_removed_custom_iam_role "organization" "${organization}" "${name}" +} + +# Ensure that custom IAM role has been removed from project +# Arguments: +# $1: The id of the project (e.g. "k8s-infra-foo") +# $2: The role name (e.g. "foo.barrer") +function ensure_removed_custom_project_iam_role() { + if [ ! $# -eq 2 -o -z "$1" -o -z "$2" ]; then + echo "ensure_removed_custom_project_iam_role(project, name) requires 2 arguments" >&2 + return 1 fi - # name is foo.bar, but gcloud wants scope/id/role/foo.bar in the file - <"${file}" sed -e "s|^name: ${name}|name: ${full_name}|" >"${ready}" - gcloud iam roles "${verb}" ${scope_flag} "${name}" --file "${ready}" > "${after}" + local project="${1}" + local name="${2}" - # if they differ, ignore the error - diff "${before}" "${after}" || true + _ensure_removed_custom_iam_role "project" "${project}" "${name}" } # Return the full name of a custom IAM role defined at the org level # Arguments: -# $1: The role name (e.g. "prow.viewer") +# $1: The role name (e.g. "foo.barrer") function custom_org_role_name() { if [ ! $# -eq 1 -o -z "$1" ]; then echo "custom_org_role_name(name) requires 1 arguments" >&2 return 1 fi - + local name="${1}" + # the equivalent gcloud command takes longer and may require more privileges + # gcloud iam roles describe --organization ${organization} ${name} --format='value(name)' echo "organizations/${GCP_ORG}/roles/${name}" } -# Ensure that IAM binding exists at org level +# Return the full name of a custom IAM role defined at the org level # Arguments: -# $1: The role name (e.g. "prow.viewer") +# $1: The is of the project (e.g. "k8s-infra-foo") +# $2: The role name (e.g. "foo.barrer") +function custom_project_role_name() { + if [ ! $# -eq 2 -o -z "$1" -o -z "$2" ]; then + echo "custom_project_role_name(project, name) requires 2 arguments" >&2 + return 1 + fi + + local project="${1}" + local name="${2}" + + # the equivalent gcloud command takes longer and may require more privileges + # gcloud iam roles describe --projects ${project} ${name} --format='value(name)' + echo "projects/${project}/roles/${name}" +} + +# Ensure that IAM binding is present for organization +# Arguments: +# $1: The role name (e.g. "foo.barrer") # $2: The file (e.g. "/path/to/file.yaml") function ensure_org_role_binding() { if [ ! $# -eq 2 -o -z "$1" -o -z "$2" ]; then @@ -135,17 +134,14 @@ function ensure_org_role_binding() { return 1 fi - local org="${GCP_ORG}" + local organization="${GCP_ORG}" local principal="${1}" local role="${2}" - gcloud \ - organizations add-iam-policy-binding "${GCP_ORG}" \ - --member "${principal}" \ - --role "${role}" + _ensure_resource_role_binding "organizations" "${organization}" "${principal}" "${role}" } -# Ensure that IAM binding exists at project level +# Ensure that IAM binding is present for project # Arguments: # $1: The project id (e.g. "k8s-infra-foo") # $2: The principal (e.g. "group:k8s-infra-foo@kubernetes.io") @@ -160,10 +156,24 @@ function ensure_project_role_binding() { local principal="${2}" local role="${3}" - gcloud \ - projects add-iam-policy-binding "${project}" \ - --member "${principal}" \ - --role "${role}" + _ensure_resource_role_binding "projects" "${project}" "${principal}" "${role}" +} + +# Ensure that IAM binding has been removed from organization +# Arguments: +# $1: The principal (e.g. "group:k8s-infra-foo@kubernetes.io") +# $2: The role name (e.g. "roles/foo.bar") +function ensure_removed_org_role_binding() { + if [ ! $# -eq 2 -o -z "$1" -o -z "$2" ]; then + echo "ensure_removed_org_role_binding(principal, role) requires 2 arguments" >&2 + return 1 + fi + + local organization="${GCP_ORG}" + local principal="${1}" + local role="${2}" + + _ensure_removed_resource_role_binding "organizations" "${organization}" "${principal}" "${role}" } # Ensure that IAM binding has been removed from project @@ -184,24 +194,131 @@ function ensure_removed_project_role_binding() { _ensure_removed_resource_role_binding "projects" "${project}" "${principal}" "${role}" } -# Ensure that IAM binding has been removed from organization +# Ensure that custom IAM role exists in scope and in sync with definition in file # Arguments: -# $1: The principal (e.g. "group:k8s-infra-foo@kubernetes.io") -# $2: The role name (e.g. "roles/foo.bar") -function ensure_removed_org_role_binding() { - if [ ! $# -eq 2 -o -z "$1" -o -z "$2" ]; then - echo "ensure_removed_org_role_binding(principal, role) requires 2 arguments" >&2 +# $1: The scope of the role (e.g. "organization", "project") +# $2: The id of the scope (e.g. "12345819", "k8s-infra-foo") +# $3: The role name (e.g. "foo.barrer") +# $4: The file (e.g. "/path/to/file.yaml") +function _ensure_custom_iam_role_from_file() { + if [ ! $# -eq 4 -o -z "$1" -o -z "$2" -o -z "$3" -o -z "$4" ]; then + echo "_ensure_custom_iam_role_from_file(scope, id, name, file) requires 4 arguments" >&2 return 1 fi - local organization="${GCP_ORG}" - local principal="${1}" - local role="${2}" + local scope="${1}" + local id="${2}" + local name="${3}" + local file="${4}" + case "${scope}" in + organization | project ) ;; + * ) + echo "_ensure_custom_iam_role_from_file(scope, id, name, file) scope must 'organization' or 'project'" >&2 + return 1 + ;; + esac - _ensure_removed_resource_role_binding "organizations" "${organization}" "${principal}" "${role}" + local scope_flag="--${scope} ${id}" + + local before="${tmp_dir}/custom-role.before.yaml" + local ready="${tmp_dir}/custom-role.ready.yaml" + local after="${tmp_dir}/custom-role.after.yaml" + + # detect if we should create or update and dump role; silently ignore error + verb="update" + if ! (gcloud iam roles describe ${scope_flag} "${name}" | yq -Y 'del(.etag)' >"${before}") >/dev/null 2>&1; then + verb="create" + fi + + # deleted roles can be undeleted within 7 days; after that must wait 30 days to create a role with same id + # ref: https://cloud.google.com/iam/docs/creating-custom-roles#deleting-custom-role + if <"${before}" grep -q "^deleted: true"; then + gcloud iam roles undelete ${scope_flag} "${name}" + fi + + # name is foo.bar, but gcloud wants scopes/id/role/foo.bar in the file + local full_name="${scope}s/${id}/roles/${name}" + <"${file}" sed -e "s|^name: ${name}|name: ${full_name}|" >"${ready}" + gcloud iam roles "${verb}" ${scope_flag} "${name}" --file "${ready}" | yq -Y 'del(.etag)' > "${after}" + + # if they differ, ignore the error + diff -u "${before}" "${after}" || true +} + +# Ensure that custom IAM role exists in scope and in sync with definition in file +# Arguments: +# $1: The scope of the role (e.g. "organization", "project") +# $2: The id of the scope (e.g. "12345819", "k8s-infra-foo") +# $3: The role name (e.g. "foo.barrer") +function _ensure_removed_custom_iam_role() { + if [ ! $# -eq 3 -o -z "$1" -o -z "$2" -o -z "$3" ]; then + echo "_ensure_removed_custom_iam_role(scope, id, name) requires 3 arguments" >&2 + return 1 + fi + + local scope="${1}" + local id="${2}" + local name="${3}" + case "${scope}" in + organization | project ) ;; + * ) + echo "_ensure_removed_custom_iam_role(scope, id, name) scope must 'organization' or 'project'" >&2 + return 1 + ;; + esac + + local scope_flag="--${scope} ${id}" + + local before="${tmp_dir}/iam-bind.before.txt" + + # gcloud iam roles delete errors if role doesn't exist, so confirm it does + if ! gcloud iam roles describe ${scope_flag} ${name} --format="value(deleted)" > "${before}"; then + # not found, or can't see... no point in continuing + return + fi + # gcloud iam roles delete errors if role has already been deleted, so confirm it has not + if [ "$(cat "${before}")" == "True" ]; then + # already deleted, nothing to do + return + fi + gcloud iam roles delete ${scope_flag} "${name}" +} + +# Ensure that IAM binding is present for resource +# Arguments: +# $1: The resource type (e.g. "projects", "organizations", "secrets" ) +# $2: The id of the resource (e.g. "k8s-infra-foo", "12345") +# $3: The principal (e.g. "group:k8s-infra-foo@kubernetes.io") +# $4: The role name (e.g. "roles/foo.bar") +function _ensure_resource_role_binding() { + if [ ! $# -eq 4 -o -z "$1" -o -z "$2" -o -z "$3" -o -z "$4" ]; then + echo "_ensure_resource_role_binding(resource, id, principal, role) requires 4 arguments" >&2 + return 1 + fi + + local resource="${1}" + local id="${2}" + local principal="${3}" + local role="${4}" + + local before="${tmp_dir}/iam-bind.before.yaml" + local after="${tmp_dir}/iam-bind.after.yaml" + + # gcloud add-iam-policy-binding will not error on adding a duplicate binding + # TODO: that said, it is annoying to see lots of "updated iam policy for X" when nothing changed, + # so consider avoiding the call + gcloud "${resource}" get-iam-policy "${id}" | yq -Y 'del(.etag)' > "${before}" + # add the binding + gcloud \ + "${resource}" add-iam-policy-binding "${id}" \ + --member "${principal}" \ + --role "${role}" | \ + yq -Y 'del(.etag)' > "${after}" + # if they differ, ignore the error + diff -u "${before}" "${after}" || true } -# Ensure that IAM binding has been removed at resource level +# Ensure that IAM binding has been removed from resource # Arguments: # $1: The resource type (e.g. "projects", "organizations", "secrets" ) # $2: The id of the resource (e.g. "k8s-infra-foo", "12345") @@ -209,7 +326,7 @@ function ensure_removed_org_role_binding() { # $4: The role name (e.g. "roles/foo.bar") function _ensure_removed_resource_role_binding() { if [ ! $# -eq 4 -o -z "$1" -o -z "$2" -o -z "$3" -o -z "$4" ]; then - echo "ensure_removed_project_role_binding(resource, id, principal, role) requires 4 arguments" >&2 + echo "_ensure_removed_resource_role_binding(resource, id, principal, role) requires 4 arguments" >&2 return 1 fi diff --git a/infra/gcp/lib_util.sh b/infra/gcp/lib_util.sh index 6c90d46ebd4..effcd52121a 100755 --- a/infra/gcp/lib_util.sh +++ b/infra/gcp/lib_util.sh @@ -21,6 +21,9 @@ # # This MUST NOT be used directly. Source it via lib.sh instead. +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +repo_root=$(cd "${script_dir}/../.." && pwd) + function _color() { tput setf "$1" || true } @@ -35,7 +38,7 @@ function _nocolor() { function color() { _color "$1" shift - echo "$@$(_nocolor)" + echo "$*$(_nocolor)" } # ensure_gnu_sed @@ -45,20 +48,39 @@ function color() { # SED: The name of the gnu-sed binary # function ensure_gnu_sed() { - sed_help="$(LANG=C sed --help 2>&1 || true)" - if echo "${sed_help}" | grep -q "GNU\|BusyBox"; then - SED="sed" - elif command -v gsed &>/dev/null; then - SED="gsed" - else - >&2 echo "Failed to find GNU sed as sed or gsed. If you are on Mac: brew install gnu-sed" - return 1 - fi - export SED + sed_help="$(LANG=C sed --help 2>&1 || true)" + if echo "${sed_help}" | grep -q "GNU\|BusyBox"; then + SED="sed" + elif command -v gsed &>/dev/null; then + SED="gsed" + else + >&2 echo "Failed to find GNU sed as sed or gsed. If you are on Mac: brew install gnu-sed" + return 1 + fi + export SED +} + +function verify_prereqs() { + # indent relies on sed -u which isn't available in macOS's sed + if ! ensure_gnu_sed; then + exit 1 + fi + # ensure-e2e-projects, ensure-main-project, ensure-namespaces rely on this + # we're not checking for a specific version; 1.6 has not yet made it to distributions + if ! command -v jq &>/dev/null; then + >&2 echo "jq not found. Please install: https://stedolan.github.io/jq/download/" + exit 1 + fi + # generate-role-yaml relies on this + # opting for https://kislyuk.github.io/yq/ over https://github.com/mikefarah/yq due to + # parity with jq, but may be worth reconsidering + if ! command -v yq &>/dev/null; then + >&2 echo "yq not found. Please install, e.g. pip3 install -r ${repo_root}/requirements.txt" + exit 1 + fi } -# indent relies on sed -u which isn't available in macOS's sed -if ! ensure_gnu_sed; then +if ! verify_prereqs; then exit 1 fi diff --git a/infra/gcp/prow/ensure-e2e-projects.sh b/infra/gcp/prow/ensure-e2e-projects.sh index 32b2c596914..9c5a1362b78 100755 --- a/infra/gcp/prow/ensure-e2e-projects.sh +++ b/infra/gcp/prow/ensure-e2e-projects.sh @@ -113,12 +113,7 @@ for prj; do color 6 "Ensure stale role bindings have been removed from e2e project: ${prj}" ( - # TODO(https://github.com/kubernetes/k8s.io/issues/1661): remove once verified as consistent - color 6 "group:k8s-infra-prow-viewers@kubernetes.io should not have roles/viewer (ref: https://github.com/kubernetes/k8s.io/issues/1661)" - ensure_removed_project_role_binding "${prj}" "group:k8s-infra-prow-viewers@kubernetes.io" "roles/viewer" - # TODO(https://github.com/kubernetes/k8s.io/issues/299): remove once smarter logic folded into ensure_project - color 6 "user:* should not have roles/owner for projects (ref: https://github.com/kubernetes/k8s.io/issues/299)" - ensure_removed_project_role_binding "${prj}" "spiffxp@google.com" "roles/owner" + echo "no stale bindings slated for removal" ) 2>&1 | indent color 6 "Enabling APIs necessary for kubernetes e2e jobs to use e2e project: ${prj}" diff --git a/infra/gcp/roles/CustomRole.yaml b/infra/gcp/roles/CustomRole.yaml index 57396ca2e16..cd0bf1aa9be 100644 --- a/infra/gcp/roles/CustomRole.yaml +++ b/infra/gcp/roles/CustomRole.yaml @@ -1,4 +1,4 @@ -#### generated by generate-role-yaml.sh from the following spec: +#### generated by generate-role-yaml.sh from ./roles/specs/CustomRole.yaml # # # This was retroactively put together to match a role that is already in # # production. It's not entirely clear why it was built this way, for example diff --git a/infra/gcp/roles/StorageBucketLister.yaml b/infra/gcp/roles/StorageBucketLister.yaml deleted file mode 100644 index a4011f70983..00000000000 --- a/infra/gcp/roles/StorageBucketLister.yaml +++ /dev/null @@ -1,26 +0,0 @@ -#### generated by generate-role-yaml.sh from the following spec: -# -# # allow listing of buckets -# # TODO(https://github.com/kubernetes/k8s.io/issues/1659): remove once auditor.viewer is used instead -# title: Storage Bucket Lister -# description: Can list storage buckets -# name: StorageBucketLister -# include: -# roles: -# - roles/storage.admin -# permissionRegexes: -# - ^storage.buckets. -# exclude: -# permissionRegexes: -# - create$ -# - update$ -# - delete$ -# - IamPolicy$ -# -description: Can list storage buckets -includedPermissions: - - storage.buckets.get - - storage.buckets.list -name: StorageBucketLister -stage: GA -title: Storage Bucket Lister diff --git a/infra/gcp/roles/audit.viewer.yaml b/infra/gcp/roles/audit.viewer.yaml index 1c65f1e93f9..5a381c9a4ff 100644 --- a/infra/gcp/roles/audit.viewer.yaml +++ b/infra/gcp/roles/audit.viewer.yaml @@ -1,4 +1,4 @@ -#### generated by generate-role-yaml.sh from the following spec: +#### generated by generate-role-yaml.sh from ./roles/specs/audit.viewer.yaml # # # an aggregate of different service roles to allow auditing of all resources # # in the organization hierarchy using GCP console or gcloud commands diff --git a/infra/gcp/roles/generate-role-yaml.sh b/infra/gcp/roles/generate-role-yaml.sh index a47802e4033..3939b3cbdc1 100755 --- a/infra/gcp/roles/generate-role-yaml.sh +++ b/infra/gcp/roles/generate-role-yaml.sh @@ -77,8 +77,10 @@ function output_role_yaml() { exclude_regex=$(<"${spec}" yq -r '.exclude? | .permissionRegexes//[] | join("|")') local output_path="${output_dir}/${name}.yaml" + + echo "generating custom role, spec: ${spec}, output: ${output_path}" ( - echo "#### generated by ${script_name} from the following spec:" + echo "#### generated by ${script_name} from ${spec}" echo "#" <"${spec}" sed -e 's/^/# /' echo "#" diff --git a/infra/gcp/roles/iam.serviceAccountLister.yaml b/infra/gcp/roles/iam.serviceAccountLister.yaml new file mode 100644 index 00000000000..3092e20b335 --- /dev/null +++ b/infra/gcp/roles/iam.serviceAccountLister.yaml @@ -0,0 +1,18 @@ +#### generated by generate-role-yaml.sh from ./roles/specs/iam.serviceAccountLister.yaml +# +# # formerly a project-level custom role for any project with GKE clusters, +# # not actually sure why this account is necessary but here we are +# title: ServiceAccount Lister +# description: Can list ServiceAccounts +# name: iam.serviceAccountLister +# include: +# permissions: +# - iam.serviceAccounts.list +# +# +description: Can list ServiceAccounts +includedPermissions: + - iam.serviceAccounts.list +name: iam.serviceAccountLister +stage: GA +title: ServiceAccount Lister diff --git a/infra/gcp/roles/organization.admin.yaml b/infra/gcp/roles/organization.admin.yaml index 4668979c0cf..3f3593f1ff3 100644 --- a/infra/gcp/roles/organization.admin.yaml +++ b/infra/gcp/roles/organization.admin.yaml @@ -1,4 +1,4 @@ -#### generated by generate-role-yaml.sh from the following spec: +#### generated by generate-role-yaml.sh from ./roles/specs/organization.admin.yaml # # # permissions that are required beyond roles/owner to effectively manage # # our GCP organization, at least according to the role bindings that have diff --git a/infra/gcp/roles/prow.viewer.yaml b/infra/gcp/roles/prow.viewer.yaml index 7c587719d8e..6d6bb4c594e 100644 --- a/infra/gcp/roles/prow.viewer.yaml +++ b/infra/gcp/roles/prow.viewer.yaml @@ -1,4 +1,4 @@ -#### generated by generate-role-yaml.sh from the following spec: +#### generated by generate-role-yaml.sh from ./roles/specs/prow.viewer.yaml # # # a subset of convenience role roles/viewer to allow community members # # to view resources used by prow or jobs that lease e2e projects from boskos @@ -373,6 +373,7 @@ includedPermissions: - logging.queries.delete - logging.queries.get - logging.queries.list + - logging.queries.listShared - logging.queries.update - logging.sinks.get - logging.sinks.list diff --git a/infra/gcp/roles/secretmanager.secretLister.yaml b/infra/gcp/roles/secretmanager.secretLister.yaml index 28c71cf75d5..5b3571be937 100644 --- a/infra/gcp/roles/secretmanager.secretLister.yaml +++ b/infra/gcp/roles/secretmanager.secretLister.yaml @@ -1,4 +1,4 @@ -#### generated by generate-role-yaml.sh from the following spec: +#### generated by generate-role-yaml.sh from ./roles/specs/secretmanager.secretLister.yaml # # # allow principals with roles/secretmanager.admin on specific secrets # # to manage those secrets via the GCP console diff --git a/infra/gcp/roles/specs/StorageBucketLister.yaml b/infra/gcp/roles/specs/StorageBucketLister.yaml deleted file mode 100644 index 8ccacda3314..00000000000 --- a/infra/gcp/roles/specs/StorageBucketLister.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# allow listing of buckets -# TODO(https://github.com/kubernetes/k8s.io/issues/1659): remove once auditor.viewer is used instead -title: Storage Bucket Lister -description: Can list storage buckets -name: StorageBucketLister -include: - roles: - - roles/storage.admin - permissionRegexes: - - ^storage.buckets. -exclude: - permissionRegexes: - - create$ - - update$ - - delete$ - - IamPolicy$ diff --git a/infra/gcp/roles/specs/iam.serviceAccountLister.yaml b/infra/gcp/roles/specs/iam.serviceAccountLister.yaml new file mode 100644 index 00000000000..561754b3eb4 --- /dev/null +++ b/infra/gcp/roles/specs/iam.serviceAccountLister.yaml @@ -0,0 +1,9 @@ +# formerly a project-level custom role for any project with GKE clusters, +# not actually sure why this account is necessary but here we are +title: ServiceAccount Lister +description: Can list ServiceAccounts +name: iam.serviceAccountLister +include: + permissions: + - iam.serviceAccounts.list +