[ci] Produce and upload bazel execution logs

Execution logs can be used to debug caching and understand better what bazel is doing. This commit introduces a general mecanism to upload those logs. Only convert the SW build and test step for now. Signed-off-by: Amaury Pouly <[email protected]>
lowRISC · Nov 10, 2023 · 0e15c51 · 0e15c51
1 parent 03c5fee
commit 0e15c51
Show file tree

Hide file tree

Showing 2 changed files with 30 additions and 2 deletions.
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -18,6 +18,8 @@ variables:
   # This controls where builds happen, and gets picked up by build_consts.sh.
   BUILD_ROOT: $(Build.ArtifactStagingDirectory)
   VIVADO_VERSION: "2020.2"
+  # This controls where the bazel execution logs go
+  BAZEL_EXEC_LOG_ROOT: $(Build.ArtifactStagingDirectory)/bazel_exec_logs
 
 trigger:
   batch: true
@@ -216,6 +218,7 @@ jobs:
       #   environment does not have access to Vivado.
       export GCP_BAZEL_CACHE_KEY=$(bazelCacheGcpKeyPath)
       TARGET_PATTERN_FILE=$(mktemp)
+      mkdir "$BAZEL_EXEC_LOG_ROOT"
       echo //... > "${TARGET_PATTERN_FILE}"
       echo -//quality/... >> "${TARGET_PATTERN_FILE}"
       echo -//sw/otbn/crypto/... >> "${TARGET_PATTERN_FILE}"
@@ -232,15 +235,18 @@ jobs:
         --build_tests_only=false \
         --define DISABLE_VERILATOR_BUILD=true \
         --test_tag_filters=-broken,-cw310,-verilator,-dv \
-        --target_pattern_file="${TARGET_PATTERN_FILE}"
+        --target_pattern_file="${TARGET_PATTERN_FILE}" \
+        --execution_log_binary_file="$BAZEL_EXEC_LOG_ROOT/build.log"
       ci/bazelisk.sh test \
         --build_tests_only=false \
         --test_output=errors \
         --define DISABLE_VERILATOR_BUILD=true \
         --test_tag_filters=-broken,-cw310,-verilator,-dv \
-        --target_pattern_file="${TARGET_PATTERN_FILE}"
+        --target_pattern_file="${TARGET_PATTERN_FILE}" \
+        --execution_log_binary_file="$BAZEL_EXEC_LOG_ROOT/test.log"
     displayName: Build & test SW
   - template: ci/publish-bazel-test-results.yml
+  - template: ci/publish-bazel-exec-logs.yml
   - bash: |
       set -x -e
       . util/build_consts.sh

diff --git a/ci/publish-bazel-exec-logs.yml b/ci/publish-bazel-exec-logs.yml
@@ -0,0 +1,22 @@
+# Copyright lowRISC contributors.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+# Azure template for publishing bazel execution logs so they can be analyzed
+# offline.
+#
+# This template expects that a variable $BAZEL_EXEC_LOG_ROOT is set and it
+# will upload a zip file containing all the files in this directory.
+
+steps:
+  - bash: |
+      set -e
+      cd $BAZEL_EXEC_LOG_ROOT
+      zip $(Build.ArtifactStagingDirectory)/exec_logs.zip *
+    displayName: Create zip with execution logs
+  - publish: "$(Build.ArtifactStagingDirectory)/exec_logs.zip"
+    # The PhaseName is the string after the "job" key in the build description,
+    # e.g. "job: my_phase_name".
+    artifact: exec-logs-$(System.PhaseName)
+    displayName: Upload execution logs
+