[ci] Produce and upload bazel execution logs

Execution logs can be used to debug caching and understand better what bazel is doing. This commit introduces a general mecanism to upload those logs. Only convert the SW build and test step for now. Signed-off-by: Amaury Pouly <[email protected]>
lowRISC · Jan 16, 2024 · 61c8231 · 61c8231
1 parent 890b751
commit 61c8231
Show file tree

Hide file tree

Showing 2 changed files with 32 additions and 2 deletions.
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -18,6 +18,8 @@ variables:
   # This controls where builds happen, and gets picked up by build_consts.sh.
   BUILD_ROOT: $(Build.ArtifactStagingDirectory)
   VIVADO_VERSION: "2021.1"
+  # This controls where the bazel execution logs go
+  BAZEL_EXEC_LOG_ROOT: $(Build.ArtifactStagingDirectory)/bazel_exec_logs
 
 trigger:
   batch: true
@@ -216,6 +218,7 @@ jobs:
       #   environment does not have access to Vivado.
       export GCP_BAZEL_CACHE_KEY=$(bazelCacheGcpKeyPath)
       TARGET_PATTERN_FILE=target_pattern.txt
+      mkdir "$BAZEL_EXEC_LOG_ROOT"
       echo //... > "${TARGET_PATTERN_FILE}"
       echo -//quality/... >> "${TARGET_PATTERN_FILE}"
       echo -//sw/otbn/crypto/... >> "${TARGET_PATTERN_FILE}"
@@ -234,8 +237,11 @@ jobs:
         --build_tests_only=false \
         --define DISABLE_VERILATOR_BUILD=true \
         --test_tag_filters=-broken,-cw310,-verilator,-dv \
-        --target_pattern_file="${TARGET_PATTERN_FILE}"
+        --target_pattern_file="${TARGET_PATTERN_FILE}" \
+        --execution_log_json_file="$BAZEL_EXEC_LOG_ROOT/build.log" \
+        --noexecution_log_sort
     displayName: Build SW
+  - template: ci/publish-bazel-exec-logs.yml
   - publish: target_pattern.txt
     artifact: target_pattern_file
   - bash: |
@@ -284,8 +290,11 @@ jobs:
         --test_output=errors \
         --define DISABLE_VERILATOR_BUILD=true \
         --test_tag_filters=-broken,-cw310,-verilator,-dv,-silicon \
-        --target_pattern_file="${TARGET_PATTERN_FILE}"
+        --target_pattern_file="${TARGET_PATTERN_FILE}" \
+        --execution_log_json_file="$BAZEL_EXEC_LOG_ROOT/test.log" \
+        --noexecution_log_sort
     displayName: Build & test SW
+  - template: ci/publish-bazel-exec-logs.yml
   - template: ci/publish-bazel-test-results.yml
 
 - job: chip_englishbreakfast_verilator

diff --git a/ci/publish-bazel-exec-logs.yml b/ci/publish-bazel-exec-logs.yml
@@ -0,0 +1,21 @@
+# Copyright lowRISC contributors.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+# Azure template for publishing bazel execution logs so they can be analyzed
+# offline.
+#
+# This template expects that a variable $BAZEL_EXEC_LOG_ROOT is set and it
+# will upload a zip file containing all the files in this directory.
+
+steps:
+  - bash: |
+      set -e
+      cd $BAZEL_EXEC_LOG_ROOT
+      zip $(Build.ArtifactStagingDirectory)/exec_logs.zip *
+    displayName: Create zip with execution logs
+  - publish: "$(Build.ArtifactStagingDirectory)/exec_logs.zip"
+    # The PhaseName is the string after the "job" key in the build description,
+    # e.g. "job: my_phase_name".
+    artifact: exec-logs-$(System.PhaseName)
+    displayName: Upload execution logs