Add bazel-based presubmit (#262)

This change adds a cloudbuild.yaml presubmit to enkit.

This presubmit:
* rebases the PR branch onto the latest master
* detects affected targets between master and the tip of the PR branch
* builds all affected targets
* runs all affected tests

This change does not include:

* Filtering based on tags - we need some way to exclude tests from being
  run in presubmit, if they are expensive or flaky
* Optimizations - getting to the build phase takes ~8.5 minutes, which
  is very long and could possibly be reduced.

Tested: https://console.cloud.google.com/cloud-build/builds;region=global/da53f43e-0c13-4103-bafc-5bc0aa400f4c?project=cloud-build-290921

infra/cloudbuild/helpers/git_rebase_pr.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+set -e
+
+# Rebase PR onto the tip of master.
+# Requires:
+# * Image with `git`
+# * Mounted `/root/.ssh` volume
+#
+# Example usage:
+#  - name: gcr.io/cloud-builders/git
+#    entrypoint: bash
+#    args:
+#      - -c
+#      - infra/cloudbuild/helpers/git_rebase_pr.sh 100
+#    volumes:
+#      - name: ssh
+#        path: /root/.ssh
+
+# Depth to search for a common merge-base between the current commit and the
+# base branch (typically master)
+readonly FETCH_DEPTH="$1"
+
+# Impersonate PR author when rebasing
+# Rebasing will reauthor commits, and as no author is configured by default,
+# git will complain and bail. Configure the author and email for this build
+# from the last commit, which should match the PR author, as long as PRs
+# don't have multiple collaborators. This way, anything that reads the commit
+# author name/email won't be bamboozled by e.g. a dummy value here.
+git config user.email "$(git log --format='%ae' -1)"
+git config user.name "$(git log --format='%an' -1)"
+
+
+# Give the PR head a local branch name
+# This branch name is referenced in future rebasing steps.
+git checkout -b github_pr
+
+# Fetch more commits of the history
+# We need to find a common ancestor between the PR commits and master in order
+# for the rebase to succeed.  The `deepen` value likely needs to be the maximum
+# of "number of commits in a PR" and "number of commits master is allowed to
+# move ahead by". Potential scalability problem here: master moves at a rate
+# proportional to the number of devs. So as more people join, we need to fetch
+# more master to find a common ancestor.
+#
+# Realistically, this probably becomes "number of commits master moves in time
+# period t" where t is something like one week, and then we take that position
+# that PRs must be rebased at least weekly if one wants presubmits to run
+# properly.
+git fetch --deepen="${FETCH_DEPTH}"
+
+# Rebase or error
+# TODO(scott): Add instructions for rebasing or a pointer to such instructions
+# in the error message.
+readonly COMMON_ANCESTOR="$(git merge-base origin/master github_pr)"
+git rebase "${COMMON_ANCESTOR}" github_pr --onto origin/master || { echo "
+********************************************************************************
+** Auto-rebase failure **                                                      *
+********************************************************************************
+* Presubmits rebase your PR onto the latest master before running, but this    *
+* has failed because your PR is too out-of-date. Please rebase your PR to pick *
+* updates from master and re-push.                                             *
+********************************************************************************
+"; /bin/false; }

infra/cloudbuild/helpers/git_ssh_setup.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+set -e
+
+# SSH setup for pulling repositories from Github
+# More info: https://cloud.google.com/build/docs/access-github-from-build
+# Requires:
+# * Image with `git`
+# * Mounted `/root/.ssh` volume
+# * SSH_KEY defined in the environment
+#
+# Example usage:
+#  - name: gcr.io/cloud-builders/git
+#    entrypoint: bash
+#    args:
+#      - '-c'
+#      - infra/cloudbuild/helpers/git_ssh_setup.sh enfabrica/internal
+#    secretEnv:
+#      - SSH_KEY
+#    volumes:
+#      - name: ssh
+#        path: /root/.ssh
+
+readonly REPO="$1"
+
+# Spill the private SSH key to the root user's SSH dir
+echo "${SSH_KEY}" >> /root/.ssh/id_rsa
+chmod 400 /root/.ssh/id_rsa
+
+# Copy a known_hosts file containing Github to the root user's SSH dir
+cp infra/cloudbuild/helpers/known_hosts /root/.ssh/known_hosts
+
+# Rewrite the origin URL to use SSH instead of HTTPS
+git remote set-url origin "[email protected]:${REPO}"

infra/cloudbuild/helpers/known_hosts

@@ -0,0 +1,2 @@
+# Github's public SSH key, so that repos can be fetched during the build
+github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==

infra/cloudbuild/helpers/log_affected_targets.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+set -e
+
+# Log affected targets
+# Requires:
+# * Image with `bash`
+# * argv[1]: Path to list of changed targets
+# * argv[2]: Path to list of changed tests
+#
+# Example usage:
+#  - name: gcr.io/cloud-builders/git
+#    entrypoint: bash
+#    args:
+#      - -c
+#      - infra/cloudbuild/helpers/log_affected_targets.sh /affected-targets/build.txt /affected-targets/test.txt
+#    volumes:
+#      - name: affected-targets
+#        path: /affected-targets
+
+
+readonly CHANGED_TARGETS_FILE="$1"
+readonly CHANGED_TESTS_FILE="$2"
+
+echo "Building affected targets:"
+cat "${CHANGED_TARGETS_FILE}"
+echo ""
+echo "Running affected tests:"
+cat "${CHANGED_TESTS_FILE}"

infra/cloudbuild/presubmit_bazel.yaml

@@ -0,0 +1,76 @@
+# Bazel Presubmit Cloud Build
+#
+# This workflow defines a Bazel {build, test} presubmit flow that auto-detects
+# changed targets from the triggering PR vs. the latest master.
+
+steps:
+  - name: gcr.io/cloud-builders/git
+    entrypoint: bash
+    args:
+      - "-c"
+      - infra/cloudbuild/helpers/git_ssh_setup.sh enfabrica/enkit
+    secretEnv:
+      - SSH_KEY
+    volumes:
+      - name: ssh
+        path: /root/.ssh
+
+  - name: gcr.io/cloud-builders/git
+    entrypoint: bash
+    args:
+      - -c
+      - infra/cloudbuild/helpers/git_rebase_pr.sh 100
+    volumes:
+      - name: ssh
+        path: /root/.ssh
+
+  - name: gcr.io/devops-284019/developer_testing:scott_presubmit_test
+    entrypoint: /opt/enfabrica/bin/enkit
+    args:
+      - bazel
+      - affected-targets
+      - list
+      - --start=origin/master
+      - --end=github_pr
+      - --affected_targets_file=/affected-targets/build.txt
+      - --affected_tests_file=/affected-targets/test.txt
+    volumes:
+      - name: affected-targets
+        path: /affected-targets
+
+  - name: gcr.io/cloud-builders/git
+    entrypoint: bash
+    args:
+      - -c
+      - infra/cloudbuild/helpers/log_affected_targets.sh /affected-targets/build.txt /affected-targets/test.txt
+    volumes:
+      - name: affected-targets
+        path: /affected-targets
+
+  - name: gcr.io/devops-284019/developer_testing:scott_presubmit_test
+    entrypoint: bash
+    args:
+      - -c
+      - cat /affected-targets/build.txt | xargs bazel build
+    volumes:
+      - name: affected-targets
+        path: /affected-targets
+
+  - name: gcr.io/devops-284019/developer_testing:scott_presubmit_test
+    entrypoint: bash
+    args:
+      - -c
+      - cat /affected-targets/test.txt | xargs bazel test
+    volumes:
+      - name: affected-targets
+        path: /affected-targets
+
+availableSecrets:
+  secretManager:
+    # SSH key for fetching from enfabrica private repos
+    - versionName: projects/496137108493/secrets/github-enfabrica-bot-key/versions/latest
+      env: SSH_KEY
+
+timeout: 20m
+options:
+  machineType: E2_HIGHCPU_8