diff --git a/audit/audit-gcp.sh b/audit/audit-gcp.sh index 6e65eff3c84..9e9bf4e3030 100755 --- a/audit/audit-gcp.sh +++ b/audit/audit-gcp.sh @@ -20,24 +20,62 @@ set -o pipefail REPO_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P) readonly REPO_ROOT -. "${REPO_ROOT}/infra/gcp/lib.sh" -readonly KUBERNETES_IO_GCP_ORG="${GCP_ORG}" +# TODO: Including this automatically calls verify_prereqs, which looks for yq, +# which is not present in gcr.io/k8s-staging-releng/releng-ci:latest, the +# image used to run this script at present. Update to use an image that +# does have it installed, or at least pip3. In the meantime, copy-paste +# the indent function. +# . "${REPO_ROOT}/infra/gcp/lib.sh" + +# ensure_gnu_sed +# Determines which sed binary is gnu-sed on linux/darwin +# +# Sets: +# SED: The name of the gnu-sed binary +# +function ensure_gnu_sed() { + sed_help="$(LANG=C sed --help 2>&1 || true)" + if echo "${sed_help}" | grep -q "GNU\|BusyBox"; then + SED="sed" + elif command -v gsed &>/dev/null; then + SED="gsed" + else + >&2 echo "Failed to find GNU sed as sed or gsed. If you are on Mac: brew install gnu-sed" + return 1 + fi + export SED +} + +# Indent each line of stdin. +# example: 2>&1 | indent +function indent() { + ${SED} -u 's/^/ /' +} + readonly AUDIT_DIR="${REPO_ROOT}/audit" +readonly KUBERNETES_IO_GCP_ORG="758905017065" # kubernetes.io -# TODO: this should maybe just be a call to verify_prereqs from lib_util.sh, -# but that currently enforces presence of `yq` which I'm not sure is -# present on the image used by the prowjob that runs this script +# TODO: this should delegate to verify_prereqs from infra/gcp/lib_util.sh once +# we can guarantee this runs in an image with `yq` and/or pip3 installed function ensure_dependencies() { + # indent relies on sed -u which isn't available in macOS's sed + if ! ensure_gnu_sed; then + exit 1 + fi + if ! command -v jq &>/dev/null; then - >&2 echo "jq not found. Please install: https://stedolan.github.io/jq/download/" - exit 1 + echo "jq not found. Please install: https://stedolan.github.io/jq/download/" >&2 + exit 1 fi # the 'bq show' command is called as a hack to dodge the config prompts that bq presents # the first time it is run. A newline is passed to stdin to skip the prompt for default project # when the service account in use has access to multiple projects. - bq show <<< $'\n' >/dev/null + if ! bq show <<< $'\n' >/dev/null; then + # ignore errors from bq while doing this hack + true + fi # right now most of this script assumes it's been run within the audit dir pushd "${AUDIT_DIR}" >/dev/null @@ -313,7 +351,7 @@ function audit_k8s_infra_gcp() { echo "Removing all existing GCP project audit files" remove_all_gcp_project_audit_files 2>&1 | indent - echo "Exporting GCP organization: ${organization}" + echo "Exporting GCP organization: ${KUBERNETES_IO_GCP_ORG}" audit_gcp_organization "${KUBERNETES_IO_GCP_ORG}" 2>&1 | indent # TODO: this will miss projects that are under folders diff --git a/audit/create-or-update-audit-pr.sh b/audit/create-or-update-audit-pr.sh new file mode 100755 index 00000000000..b36fe98cadd --- /dev/null +++ b/audit/create-or-update-audit-pr.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash + +# Copyright 2021 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Run the audit script and create or update a PR containing any changes + +# NOTE: This is intended to run on k8s-infra-prow-build-trusted as +# k8s-infra-gcp-auditor@kubernetes-public.iam.gserviceaccount.com + +# TODO: Running locally is a work in progress, there are assumptions +# made about the environment in which this runs: +# - must have certain env vars present +# - must have kubernetes/test-infra in a certain location +# - must be able to build kubernetes/test-infra +# - must have gcloud already authenticated as someone who has the +# custom org role "audit.viewer" + +set -o errexit +set -o nounset +set -o pipefail + +GH_USER=cncf-ci +GH_NAME="CNCF CI Bot" +GH_EMAIL="cncf-ci@ii.coop" +FORK_GH_REPO=k8s.io +FORK_GH_BRANCH=autoaudit-${PROW_INSTANCE_NAME:-prow} + +echo "Ensure git configured" >&2 +git config user.name "${GH_NAME}" +git config user.email "${GH_EMAIL}" + +echo "Ensure gcloud creds are working" >&2 +gcloud config list + +echo "Running Audit Script to dump GCP configuration to yaml" >&2 +pushd ./audit +bash ./audit-gcp.sh +popd + +echo "Determining whether there are changes to push" >&2 +git add --all audit +git commit -m "audit: update as of $(date +%Y-%m-%d)" +git remote add fork "https://github.com/${GH_USER}/${FORK_GH_BRANCH}" +if git fetch fork "${FORK_GH_BRANCH}"; then + if git diff --quiet HEAD "fork/${FORK_GH_BRANCH}" -- audit; then + echo "No new changes to push, exiting early..." >&2 + exit + fi +fi + +echo "Generating pr-creator binary from k/test-infra/robots" >&2 +pushd ../../kubernetes/test-infra +go build -o /workspace/pr-creator robots/pr-creator/main.go +popd + +echo "Pushing commit to github.com/${GH_USER}/${FORK_GH_REPO}..." >&2 +GH_TOKEN=$(cat /etc/github-token/token) +git push -f "https://${GH_USER}:${GH_TOKEN}@github.com/${GH_USER}/${FORK_GH_REPO}" "HEAD:${FORK_GH_BRANCH}" 2>/dev/null + +echo "Creating or updating PR to merge ${GH_USER}:${FORK_GH_BRANCH} into kubernetes:main..." >&2 +/workspace/pr-creator \ + --github-token-path=/etc/github-token/token \ + --org=kubernetes --repo=k8s.io --branch=main \ + --source="${GH_USER}:${FORK_GH_BRANCH}" \ + --head-branch="${FORK_GH_BRANCH}" \ + --title="audit: update as of $(date +%Y-%m-%d)" \ + --body="Audit Updates wg-k8s-infra" \ + --confirm diff --git a/infra/gcp/lib_util.sh b/infra/gcp/lib_util.sh index 08639f43458..d4c61fb3c52 100644 --- a/infra/gcp/lib_util.sh +++ b/infra/gcp/lib_util.sh @@ -70,7 +70,7 @@ function verify_prereqs() { >&2 echo "jq not found. Please install: https://stedolan.github.io/jq/download/" exit 1 fi - # generate-role-yaml relies on this + # generate-role-yaml, lib_iam, lib_gcs, lib_services rely on this # opting for https://kislyuk.github.io/yq/ over https://github.com/mikefarah/yq due to # parity with jq, but may be worth reconsidering if ! command -v yq &>/dev/null; then