[ci] Try to use GCP Bazel cache for more jobs

azure-pipelines.yml

@@ -46,6 +46,7 @@ jobs:
   - publish: $(Pipeline.Workspace)/opentitan-repo.tar.gz
     artifact: opentitan-repo
     displayName: Upload repository
+
 - job: lint
   displayName: Quality (quick lint)
   # Run code quality checks (quick lint)
@@ -179,23 +180,13 @@ jobs:
   dependsOn: lint
   condition: and(succeeded(), eq(dependencies.lint.outputs['DetermineBuildType.onlyDocChanges'], '0'), eq(dependencies.lint.outputs['DetermineBuildType.onlyCdcChanges'], '0'))
   pool: ci-public
-  variables:
-    - name: bazelCacheGcpKeyPath
-      value: ''
   steps:
   - template: ci/checkout-template.yml
   - template: ci/install-package-dependencies.yml
-  - task: DownloadSecureFile@1
-    condition: eq(variables['Build.SourceBranchName'], 'master')
-    name: bazelCacheGcpKey
-    inputs:
-      secureFile: "bazel_cache_gcp_key.json"
-    # Set the remote cache GCP key path
-  - bash: echo "##vso[task.setvariable variable=bazelCacheGcpKeyPath]$(bazelCacheGcpKey.secureFilePath)"
-    condition: eq(variables['Build.SourceBranchName'], 'master')
-    displayName: GCP key path
+  - template: ci/load-bazel-cache-write-creds.yml
   - bash: |
       set -x -e
+
       # Check the entire build graph for conflicts in loading or analysis
       # phases. For context, see issue #18726.
       # First, test with an empty bitstream cache entry.
@@ -214,14 +205,13 @@ jobs:
       #   shallow exclusion; tests deeper under //hw will still be found.
       # * It excludes targets that depend on bitstream_splice rules, since the
       #   environment does not have access to Vivado.
-      export GCP_BAZEL_CACHE_KEY=$(bazelCacheGcpKeyPath)
       TARGET_PATTERN_FILE=target_pattern.txt
       echo //... > "${TARGET_PATTERN_FILE}"
       echo -//quality/... >> "${TARGET_PATTERN_FILE}"
       echo -//sw/otbn/crypto/... >> "${TARGET_PATTERN_FILE}"
       echo -//third_party/riscv-compliance/... >> "${TARGET_PATTERN_FILE}"
       echo -//hw:all >> "${TARGET_PATTERN_FILE}"
-      ./bazelisk.sh cquery \
+      ci/bazelisk.sh cquery \
         --noinclude_aspects \
         --output=starlark \
         --starlark:expr='"-{}".format(target.label)' \
@@ -247,8 +237,8 @@ jobs:
       ROM_REAL_TARGETS="//sw/device/silicon_creator/rom:package_real"
       ROM_FAKE_TARGETS="//sw/device/silicon_creator/rom:package_fake"
       QUERY_CMD_ARGS=(outquery-all --noinclude_aspects --noimplicit_deps)
-      ROM_REAL_FILES=($(./bazelisk.sh "${QUERY_CMD_ARGS[@]}" "${ROM_REAL_TARGETS}" | sort | uniq))
-      ROM_FAKE_FILES=($(./bazelisk.sh "${QUERY_CMD_ARGS[@]}" "${ROM_FAKE_TARGETS}" | sort | uniq))
+      ROM_REAL_FILES=($(ci/bazelisk.sh "${QUERY_CMD_ARGS[@]}" "${ROM_REAL_TARGETS}" | sort | uniq))
+      ROM_FAKE_FILES=($(ci/bazelisk.sh "${QUERY_CMD_ARGS[@]}" "${ROM_FAKE_TARGETS}" | sort | uniq))
       cp -Lvt "${ROM_TARGET}" "${ROM_FAKE_FILES[@]}" "${ROM_REAL_FILES[@]}"
   - template: ci/upload-artifacts-template.yml
     parameters:
@@ -260,21 +250,10 @@ jobs:
   timeoutInMinutes: 120
   dependsOn: sw_build
   pool: ci-public
-  variables:
-    - name: bazelCacheGcpKeyPath
-      value: ''
   steps:
   - template: ci/checkout-template.yml
   - template: ci/install-package-dependencies.yml
-  - task: DownloadSecureFile@1
-    condition: eq(variables['Build.SourceBranchName'], 'master')
-    name: bazelCacheGcpKey
-    inputs:
-      secureFile: "bazel_cache_gcp_key.json"
-    # Set the remote cache GCP key path
-  - bash: echo "##vso[task.setvariable variable=bazelCacheGcpKeyPath]$(bazelCacheGcpKey.secureFilePath)"
-    condition: eq(variables['Build.SourceBranchName'], 'master')
-    displayName: GCP key path
+  - template: ci/load-bazel-cache-write-creds.yml
   - download: current
     artifact: target_pattern_file
   - bash: |
@@ -317,24 +296,12 @@ jobs:
   pool: ci-public
   timeoutInMinutes: 240
   dependsOn: lint
-  variables:
-    - name: bazelCacheGcpKeyPath
-      value: ''
   steps:
   - template: ci/checkout-template.yml
   - template: ci/install-package-dependencies.yml
-  - task: DownloadSecureFile@1
-    condition: eq(variables['Build.SourceBranchName'], 'master')
-    name: bazelCacheGcpKey
-    inputs:
-      secureFile: "bazel_cache_gcp_key.json"
-  - bash: echo "##vso[task.setvariable variable=bazelCacheGcpKeyPath]$(bazelCacheGcpKey.secureFilePath)"
-    condition: eq(variables['Build.SourceBranchName'], 'master')
-    displayName: GCP key path
-    # Set the remote cache GCP key path
+  - template: ci/load-bazel-cache-write-creds.yml
   - bash: |
       set -x -e
-      export GCP_BAZEL_CACHE_KEY=$(bazelCacheGcpKeyPath)
       ci/scripts/run-verilator-tests.sh
     displayName: Build & execute tests
   - template: ci/publish-bazel-test-results.yml
@@ -347,7 +314,7 @@ jobs:
   - bash: |
       . util/build_consts.sh
       mkdir -p "$BIN_DIR/hw/top_earlgrey/"
-      cp $(./bazelisk.sh outquery-build.verilator //sw/device/tests:uart_smoketest_sim_verilator) \
+      cp $(ci/bazelisk.sh outquery-build.verilator //sw/device/tests:uart_smoketest_sim_verilator) \
         "$BIN_DIR/hw/top_earlgrey/Vchip_earlgrey_verilator"
     displayName: Copy verilated binary to $BIN_DIR
   - template: ci/upload-artifacts-template.yml
@@ -374,6 +341,7 @@ jobs:
     parameters:
       downloadPartialBuildBinFrom:
         - chip_englishbreakfast_verilator
+  - template: ci/load-bazel-cache-write-creds.yml
   - bash: |
       . util/build_consts.sh
       ci/scripts/run-english-breakfast-verilator-tests.sh
@@ -435,15 +403,7 @@ jobs:
   steps:
   - template: ci/checkout-template.yml
   - template: ci/install-package-dependencies.yml
-  - task: DownloadSecureFile@1
-    condition: eq(variables['Build.SourceBranchName'], 'master')
-    name: bazelCacheGcpKey
-    inputs:
-      secureFile: "bazel_cache_gcp_key.json"
-  - bash: echo "##vso[task.setvariable variable=bazelCacheGcpKeyPath]$(bazelCacheGcpKey.secureFilePath)"
-    condition: eq(variables['Build.SourceBranchName'], 'master')
-    displayName: GCP key path
-    # Set the remote cache GCP key path
+  - template: ci/load-bazel-cache-write-creds.yml
   - bash: |
       ci/bazelisk.sh test --test_tag_filters=-nightly //sw/otbn/crypto/...
     displayName: Execute tests
@@ -560,6 +520,7 @@ jobs:
       downloadPartialBuildBinFrom:
         - chip_earlgrey_cw310
         - sw_build
+  - template: ci/load-bazel-cache-write-creds.yml
   - bash: |
       set -e
       . util/build_consts.sh
@@ -586,6 +547,7 @@ jobs:
       downloadPartialBuildBinFrom:
         - chip_earlgrey_cw310
         - sw_build
+  - template: ci/load-bazel-cache-write-creds.yml
   - bash: |
       set -e
       . util/build_consts.sh
@@ -613,6 +575,7 @@ jobs:
         - chip_earlgrey_cw310
         - chip_earlgrey_cw310_hyperdebug
         - sw_build
+  - template: ci/load-bazel-cache-write-creds.yml
   - bash: |
       set -e
       . util/build_consts.sh
@@ -639,6 +602,7 @@ jobs:
       downloadPartialBuildBinFrom:
         - chip_earlgrey_cw310_hyperdebug
         - sw_build
+  - template: ci/load-bazel-cache-write-creds.yml
   # We run the update command twice to workaround an issue with udev on the container.
   # Where rusb cannot dynamically update its device list in CI (udev is not completely
   # functional). If the device is in normal mode, the first thing that opentitantool
@@ -678,6 +642,7 @@ jobs:
       downloadPartialBuildBinFrom:
         - chip_earlgrey_cw340
         - sw_build
+  - template: ci/load-bazel-cache-write-creds.yml
   - bash: |
       set -e
       . util/build_consts.sh
@@ -704,6 +669,7 @@ jobs:
       downloadPartialBuildBinFrom:
         - chip_earlgrey_cw310
         - sw_build
+  - template: ci/load-bazel-cache-write-creds.yml
   - bash: |
       set -e
       . util/build_consts.sh

bazelisk.sh

@@ -93,10 +93,15 @@ function outquery_starlark_expr() {
     esac
 }
 
+# Arguments:
+# $qexpr: starlark expression - see `outquery_starlark_expr`
+# $name: name of an array containing Bazel arguments that should come _before_
+#        the subcommand (e.g. `--bazelrc=...`).
 function do_outquery() {
     local qexpr="$1"
     shift
-    "$file" cquery "$@" \
+
+    "$file" "${pre_cmd_args[@]}" cquery "$@" \
         --output=starlark --starlark:expr="$qexpr" \
         --ui_event_filters=-info --noshow_progress
 }
@@ -121,6 +126,14 @@ function main() {
         fi
     fi
 
+    # Shift all flags (starting with `-`) that come before the subcommand
+    # into an array.
+    pre_cmd_args=()
+    while [[ "${1-}" == -* ]]; do
+        pre_cmd_args+=("$1")
+        shift
+    done
+
     case "${1-}" in
         outquery*)
             # The custom 'outquery' command can be used to query bazel for the
@@ -150,13 +163,13 @@ function main() {
             local qexpr outfile
             qexpr="$(outquery_starlark_expr outquery)"
             outfile=$(do_outquery "$qexpr" "$@")
-            "$file" build "$@"
+            "$file" "${pre_cmd_args[@]}" build "$@"
             # shellcheck disable=SC2059
             # We are intentionally using $command_template as a format string.
             eval "$(printf "$command_template" "$outfile")"
             ;;
         *)
-            exec "$file" "$@"
+            exec "$file" "${pre_cmd_args[@]}" "$@"
             ;;
     esac
 }

ci/bazelisk.sh

@@ -11,18 +11,19 @@ if [[ -n "${PWD_OVERRIDE}" ]]; then
     cd "${PWD_OVERRIDE}" || exit
 fi
 
-echo "Running bazelisk in $(pwd)."
+echo "Running bazelisk in $(pwd)." >&2
 
 # An additional bazelrc must be synthesized to specify precisely how to use the
 # GCP bazel cache.
+GCP_CREDS_FILE="$GCP_BAZEL_CACHE_KEY_SECUREFILEPATH"
 GCP_BAZELRC="$(mktemp /tmp/XXXXXX.bazelrc)"
 trap 'rm ${GCP_BAZELRC}' EXIT
 
-if [[ -n "${GCP_BAZEL_CACHE_KEY}" && -f "${GCP_BAZEL_CACHE_KEY}" ]]; then
-    echo "Applying GCP cache key; will upload to the cache."
-    echo "build --google_credentials=${GCP_BAZEL_CACHE_KEY}" >> "${GCP_BAZELRC}"
+if [[ -n "$GCP_CREDS_FILE" && -f "$GCP_CREDS_FILE" ]]; then
+    echo "Applying GCP cache key; will upload to the cache." >&2
+    echo "build --google_credentials=${GCP_CREDS_FILE}" >> "${GCP_BAZELRC}"
 else
-    echo "No key/invalid path to key. Download from cache only."
+    echo "No key/invalid path to key. Download from cache only." >&2
     echo "build --remote_upload_local_results=false" >> "${GCP_BAZELRC}"
 fi
 

ci/load-bazel-cache-write-creds.yml

@@ -0,0 +1,17 @@
+# Copyright lowRISC contributors.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+# Azure Pipelines template for downloading a "secure file" containing the write
+# credentials for a GCP bucket where we store a Bazel cache.
+#
+# The path to the downloaded file is automatically stored in an environment
+# variable `GCP_BAZEL_CACHE_KEY_SECUREFILEPATH`. This file is loaded by the
+# `ci/bazelisk.sh` script.
+
+steps:
+  - task: DownloadSecureFile@1
+    condition: eq(variables['Build.SourceBranchName'], 'master')
+    name: GCP_BAZEL_CACHE_KEY
+    inputs:
+      secureFile: "bazel_cache_gcp_key.json"

ci/scripts/check-bazel-banned-rules.sh

@@ -6,7 +6,7 @@
 
 set -e
 
-GIT_REPOS=$(./bazelisk.sh query "kind('(new_)?git_repository', //external:*)")
+GIT_REPOS=$(ci/bazelisk.sh query "kind('(new_)?git_repository', //external:*)")
 if [[ ${GIT_REPOS} ]]; then
   echo "Bazel's 'git_repository' rule is insecure and incompatible with OpenTitan's airgapping strategy."
   echo "Please replace $GIT_REPOS with our 'http_archive_or_local' rule and set a sha256 so it can be canonically reproducible."

ci/scripts/check-bazel-tags.sh

@@ -25,7 +25,7 @@ check_empty () {
 
 # This check ensures OpenTitan software can be built with a wildcard without
 # waiting for Verilator using --build_tag_filters=-verilator
-untagged=$(./bazelisk.sh query \
+untagged=$(ci/bazelisk.sh query \
   "rdeps(
       //...,
       //hw:verilator

ci/scripts/check-licence-headers.sh

@@ -30,7 +30,7 @@ set -o pipefail
 (for F in $(git diff --name-only --diff-filter=ACMRTUXB "$merge_base"); do
     echo "--test_arg=\"$F\""
 done)| \
-    xargs -r ./bazelisk.sh test //quality:license_check --test_output=streamed || {
+    xargs -r ci/bazelisk.sh test //quality:license_check --test_output=streamed || {
 
     echo >&2 -n "##vso[task.logissue type=error]"
     echo >&2 "Licence header check failed."

ci/scripts/check-module-ids.sh

@@ -24,7 +24,7 @@ query_elfs='
     )'
 target_file=$(mktemp)
 trap 'rm -f "$target_file"' EXIT
-./bazelisk.sh query "$query_elfs" >"$target_file"
+ci/bazelisk.sh query "$query_elfs" >"$target_file"
 # We now ask bazel to build all targets but we also add the module ID checker aspect
 # and we query the corresponding output group to force bazel to run the checks.
 ./ci/bazelisk.sh build \

ci/scripts/clang-format.sh

@@ -28,4 +28,4 @@ set -o pipefail
 (for F in $(git diff --name-only "$merge_base" -- "*.cpp" "*.cc" "*.c" "*.h" ':!*/vendor/*'); do
     echo "--test_arg=\"$F\""
 done) | \
-    xargs -r ./bazelisk.sh test //quality:clang_format_check --test_output=streamed
+    xargs -r ci/bazelisk.sh test //quality:clang_format_check --test_output=streamed

ci/scripts/get-bitstream-strategy.sh

@@ -33,12 +33,12 @@ else
 fi
 
 # Retrieve the most recent bitstream at or before HEAD.
-BITSTREAM="--refresh HEAD" ./bazelisk.sh build @bitstreams//:manifest
+BITSTREAM="--refresh HEAD" ci/bazelisk.sh build @bitstreams//:manifest
 
 # Find the bitstream commit from the manifest file.
 manifest_file=$(ci/scripts/target-location.sh @bitstreams//:manifest)
 
-bitstream_commit=$(./bazelisk.sh run //util/py/scripts:get_bitstream_build_id \
+bitstream_commit=$(ci/bazelisk.sh run //util/py/scripts:get_bitstream_build_id \
   -- --manifest ${manifest_file} \
   --schema ${REPO_TOP}/rules/scripts/bitstreams_manifest.schema.json \
   --design ${bitstream_design})

ci/scripts/lib/bazel_query.py

@@ -108,7 +108,7 @@ def query(self, query: str) -> List[str]:
             return self._backend(query)
 
         bazel = subprocess.run(
-            ["./bazelisk.sh", "query", "--output=label", query],
+            ["ci/bazelisk.sh", "query", "--output=label", query],
             stdout=subprocess.PIPE,
             stderr=subprocess.DEVNULL,
             encoding='utf-8',

ci/scripts/rust-format.sh

@@ -27,7 +27,7 @@ trap 'echo "code failed rustfmt_check fix with ./bazelisk.sh run //quality:rustf
 set -o pipefail
 if ! git diff --quiet $merge_base -- "*.rs" ':!*/vendor/*'; then
     echo "Rust files changed, running Rust lint checks"
-    ./bazelisk.sh test //quality:rustfmt_check --test_output=streamed
+    ci/bazelisk.sh test //quality:rustfmt_check --test_output=streamed
 else
     echo "Rust files unchanged, skipping Rust lint checks"
 fi