Merge pull request #2104 from spiffxp/unbreak-audit-ci

audit: unbreak ci-k8sio-audit job

audit/audit-gcp.sh

@@ -20,24 +20,62 @@ set -o pipefail
 
 REPO_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)
 readonly REPO_ROOT
-. "${REPO_ROOT}/infra/gcp/lib.sh"
 
-readonly KUBERNETES_IO_GCP_ORG="${GCP_ORG}"
+# TODO: Including this automatically calls verify_prereqs, which looks for yq,
+#       which is not present in gcr.io/k8s-staging-releng/releng-ci:latest, the
+#       image used to run this script at present. Update to use an image that
+#       does have it installed, or at least pip3. In the meantime, copy-paste
+#       the indent function.
+# . "${REPO_ROOT}/infra/gcp/lib.sh"
+
+# ensure_gnu_sed
+# Determines which sed binary is gnu-sed on linux/darwin
+#
+# Sets:
+#  SED: The name of the gnu-sed binary
+#
+function ensure_gnu_sed() {
+    sed_help="$(LANG=C sed --help 2>&1 || true)"
+    if echo "${sed_help}" | grep -q "GNU\|BusyBox"; then
+        SED="sed"
+    elif command -v gsed &>/dev/null; then
+        SED="gsed"
+    else
+        >&2 echo "Failed to find GNU sed as sed or gsed. If you are on Mac: brew install gnu-sed"
+        return 1
+    fi
+    export SED
+}
+
+# Indent each line of stdin.
+# example: <command> 2>&1 | indent
+function indent() {
+    ${SED} -u 's/^/  /'
+}
+
 readonly AUDIT_DIR="${REPO_ROOT}/audit"
+readonly KUBERNETES_IO_GCP_ORG="758905017065" # kubernetes.io
 
-# TODO: this should maybe just be a call to verify_prereqs from lib_util.sh,
-#       but that currently enforces presence of `yq` which I'm not sure is
-#       present on the image used by the prowjob that runs this script
+# TODO: this should delegate to verify_prereqs from infra/gcp/lib_util.sh once
+#       we can guarantee this runs in an image with `yq` and/or pip3 installed
 function ensure_dependencies() {
+    # indent relies on sed -u which isn't available in macOS's sed
+    if ! ensure_gnu_sed; then
+        exit 1
+    fi
+
     if ! command -v jq &>/dev/null; then
-      >&2 echo "jq not found. Please install: https://stedolan.github.io/jq/download/"
-      exit 1
+        echo "jq not found. Please install: https://stedolan.github.io/jq/download/" >&2
+        exit 1
     fi
 
     # the 'bq show' command is called as a hack to dodge the config prompts that bq presents
     # the first time it is run. A newline is passed to stdin to skip the prompt for default project
     # when the service account in use has access to multiple projects.
-    bq show <<< $'\n' >/dev/null
+    if ! bq show <<< $'\n' >/dev/null; then
+        # ignore errors from bq while doing this hack
+        true
+    fi
 
     # right now most of this script assumes it's been run within the audit dir
     pushd "${AUDIT_DIR}" >/dev/null
@@ -313,7 +351,7 @@ function audit_k8s_infra_gcp() {
     echo "Removing all existing GCP project audit files"
     remove_all_gcp_project_audit_files 2>&1 | indent
 
-    echo "Exporting GCP organization: ${organization}"
+    echo "Exporting GCP organization: ${KUBERNETES_IO_GCP_ORG}"
     audit_gcp_organization "${KUBERNETES_IO_GCP_ORG}" 2>&1 | indent
 
     # TODO: this will miss projects that are under folders

audit/create-or-update-audit-pr.sh

@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+
+# Copyright 2021 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Run the audit script and create or update a PR containing any changes
+
+# NOTE: This is intended to run on k8s-infra-prow-build-trusted as
+#       [email protected]
+
+# TODO: Running locally is a work in progress, there are assumptions
+#       made about the environment in which this runs:
+#       - must have certain env vars present
+#       - must have kubernetes/test-infra in a certain location
+#       - must be able to build kubernetes/test-infra
+#       - must have gcloud already authenticated as someone who has the
+#         custom org role "audit.viewer"
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+GH_USER=cncf-ci
+GH_NAME="CNCF CI Bot"
+GH_EMAIL="[email protected]"
+FORK_GH_REPO=k8s.io
+FORK_GH_BRANCH=autoaudit-${PROW_INSTANCE_NAME:-prow}
+
+echo "Ensure git configured" >&2
+git config user.name "${GH_NAME}"
+git config user.email "${GH_EMAIL}"
+
+echo "Ensure gcloud creds are working" >&2
+gcloud config list
+
+echo "Running Audit Script to dump GCP configuration to yaml" >&2
+pushd ./audit
+bash ./audit-gcp.sh
+popd
+
+echo "Determining whether there are changes to push" >&2
+git add --all audit
+git commit -m "audit: update as of $(date +%Y-%m-%d)"
+git remote add fork "https://github.com/${GH_USER}/${FORK_GH_BRANCH}"
+if git fetch fork "${FORK_GH_BRANCH}"; then
+    if git diff --quiet HEAD "fork/${FORK_GH_BRANCH}" -- audit; then
+    echo "No new changes to push, exiting early..." >&2
+    exit
+    fi
+fi
+
+echo "Generating pr-creator binary from k/test-infra/robots" >&2
+pushd ../../kubernetes/test-infra
+go build -o /workspace/pr-creator robots/pr-creator/main.go
+popd
+
+echo "Pushing commit to github.com/${GH_USER}/${FORK_GH_REPO}..." >&2
+GH_TOKEN=$(cat /etc/github-token/token)
+git push -f "https://${GH_USER}:${GH_TOKEN}@github.com/${GH_USER}/${FORK_GH_REPO}" "HEAD:${FORK_GH_BRANCH}" 2>/dev/null
+
+echo "Creating or updating PR to merge ${GH_USER}:${FORK_GH_BRANCH} into kubernetes:main..." >&2
+/workspace/pr-creator \
+    --github-token-path=/etc/github-token/token \
+    --org=kubernetes --repo=k8s.io --branch=main \
+    --source="${GH_USER}:${FORK_GH_BRANCH}" \
+    --head-branch="${FORK_GH_BRANCH}" \
+    --title="audit: update as of $(date +%Y-%m-%d)" \
+    --body="Audit Updates wg-k8s-infra" \
+    --confirm

infra/gcp/lib_util.sh

@@ -70,7 +70,7 @@ function verify_prereqs() {
         >&2 echo "jq not found. Please install: https://stedolan.github.io/jq/download/"
         exit 1
     fi
-    # generate-role-yaml relies on this
+    # generate-role-yaml, lib_iam, lib_gcs, lib_services rely on this
     # opting for https://kislyuk.github.io/yq/ over https://github.com/mikefarah/yq due to
     # parity with jq, but may be worth reconsidering
     if ! command -v yq &>/dev/null; then