fix(bootstrap): don't download bad cache if unstaged changes (#11198)

Prevents mistakes from having out-of-date artifacts. Relies on rebuild
patterns catching uncommitted files/changes

barretenberg/cpp/src/barretenberg/commitment_schemes/ipa/ipa.hpp

@@ -981,4 +981,4 @@ template <typename Curve_> class IPA {
     }
 };
 
-} // namespace bb
+} // namespace bb

ci3/cache_content_hash

@@ -2,13 +2,11 @@
 set -euo pipefail
 [ "${BUILD_SYSTEM_DEBUG:-}" = 1 ] && set -x
 
-# Ensure args are set
 if [[ "$#" = 0 ]]; then
-    echo "Error: No arguments provided."
-    exit 1
+  echo "Error: No arguments provided."
+  exit 1
 fi
 
-# If too many spurious cache misses: can be customized to pin artifacts to a specific version
 AZTEC_CACHE_COMMIT=${AZTEC_CACHE_COMMIT:-HEAD}
 PLATFORM_TAG="${PLATFORM_TAG:-${OSTYPE:-unknown}-$(uname -m)}"
 
@@ -22,23 +20,43 @@ for arg in "$@"; do
   fi
 done
 
+GREP_PATTERN=$(echo "$rebuild_patterns" | grep -v '^$')
 # Concatenate patterns with '|' and double escape backslashes for AWK
 # filter empty lines
 AWK_PATTERN=$(echo "$rebuild_patterns" | grep -v '^$' | sed 's/\\/\\\\/g' | tr '\n' '|' | sed 's/|$//')
-
 # use git repo root because that is where our patterns are focused
-cd $(git rev-parse --show-toplevel)
+cd "$(git rev-parse --show-toplevel)"
+
+unstaged_diff="$(git diff --name-only | grep -E "$GREP_PATTERN" || true)"
+staged_diff="$(git diff --staged --name-only | grep -E "$GREP_PATTERN" || true)"
+untracked_diff="$(git ls-files --others --exclude-standard | grep -E "$GREP_PATTERN" || true)"
+
+# Check for uncommitted files/changes
+if [[ -n "$unstaged_diff" ]] || [[ -n "$staged_diff" ]] || [[ -n "$untracked_diff" ]]; then
+  # Signal to cache_upload and cache_download to not touch this file.
+  echo "disabled-cache"
+  exit 0
+fi
+
+# Calculate a content hash for matched files
 # Use git ls-tree and AWK to filter files matching the rebuild patterns and extract their hashes
 # Sort the hashes and compute the content hash
-CONTENT_HASH=$(git ls-tree -r $AZTEC_CACHE_COMMIT | awk -v pattern="($AWK_PATTERN)" '$4 ~ pattern {print $3}' | sort | git hash-object --stdin| cut -c1-16)
-
-# Check if file list was empty by comparing against the result of 'echo '' | git hash-object --stdin | cut -c1-16'
-ECHO_BLANK_HASH="8b137891791fe969"
-if [ "$CONTENT_HASH" = "$ECHO_BLANK_HASH" ]; then
-    echo "No files matched the rebuild patterns $rebuild_patterns."
-    echo "Awk pattern expanded: $AWK_PATTERN."
-    exit 1
+CONTENT_HASH="$(
+  git ls-tree -r "$AZTEC_CACHE_COMMIT" \
+    | awk -v pattern="($AWK_PATTERN)" '$4 ~ pattern {print $3}' \
+    | sort \
+    | git hash-object --stdin \
+    | cut -c1-16
+)"
+
+# Check if file list was empty by comparing against the result of
+# 'echo '' | git hash-object --stdin | cut -c1-16'
+EMPTY_HASH="8b137891791fe969"
+if [[ "$CONTENT_HASH" == "$EMPTY_HASH" ]]; then
+  echo "No files matched the rebuild patterns $rebuild_patterns."
+  echo "Awk pattern expanded: $AWK_PATTERN."
+  exit 1
 fi
 
 # important: include architecture in content hash because we target x86_64 and arm64
-echo "$CONTENT_HASH-$(echo $PLATFORM_TAG)"
+echo "${CONTENT_HASH}-${PLATFORM_TAG}"

ci3/cache_download

@@ -7,6 +7,10 @@ if [ "$#" -lt 1 ]; then
   exit 1
 fi
 
+if [[ "$1" == *"disabled-cache"* ]]; then
+  echo "Not using cache for $1 due to uncommitted changes/files."
+  exit 1
+fi
 if [ "${USE_CACHE:-0}" -lt 1 ]; then
   # Only download if USE_CACHE is 1
   echo "Not using cache for $1 because USE_CACHE=0."
@@ -26,4 +30,4 @@ else
   # Attempt to download and extract the cache file
   (curl -s -f "$S3_ENDPOINT/build-cache/$TAR_FILE" | tar -xzf - -C "$OUT_DIR" 2>/dev/null) || (echo "Cache download of $TAR_FILE failed." >&2 && exit 1)
 fi
-echo "Cache download and extraction of $TAR_FILE complete." >&2
+echo "Cache download and extraction of $TAR_FILE complete." >&2

ci3/cache_download_flag

@@ -7,6 +7,10 @@ if [ "$#" -lt 1 ]; then
   exit 1
 fi
 
+if [[ "$1" == *"disabled-cache"* ]]; then
+  echo "Running test $1 due to uncommitted changes/files."
+  exit 1
+fi
 if [ "${USE_CACHE:-0}" != 1 ] ; then
   # Don't look if CI isn't set. No need to muddle with dev runs.
   echo "Running test $1 because USE_CACHE is not 1."

ci3/cache_upload

@@ -7,6 +7,10 @@ if [ "$#" -lt 2 ]; then
   exit 1
 fi
 
+if [[ "$1" == *"disabled-cache"* ]]; then
+  echo "Skipping upload of $1 due to uncommitted changes/files."
+  exit 0
+fi
 # Name, intended to have .tar.gz ending
 name="$1"
 # Now $@ = our binary path args
@@ -22,4 +26,4 @@ if tar -czf - "$@" | aws ${S3_BUILD_CACHE_AWS_PARAMS:-} s3 cp - "s3://aztec-ci-a
 else
   echo "Cache upload of $name failed." >&2
   exit 0
-fi
+fi

ci3/cache_upload_flag

@@ -7,8 +7,12 @@ if [ "$#" -lt 1 ]; then
   exit 1
 fi
 
+if [[ "$1" == *"disabled-cache"* ]]; then
+  echo "Not uploading test flag $1 due to uncommitted changes/files."
+  exit 0
+fi
 if [ "${CI:-0}" -lt 1 ]; then
-  # Don't upload if CI isn't set. No need to muddle with dev runs.A
+  # Don't upload if CI isn't set. No need to muddle with dev runs.
   echo "Not uploading test flag $1 because CI=0."
   exit 0
 fi

ci3/source

@@ -19,7 +19,12 @@ export ci3="$root/ci3"
 
 function hash_str {
   set -eu
-  echo $1 | git hash-object --stdin | cut -c1-16 | tr -d '\n'
+  if [[ "$1" == *"disabled-cache"* ]]; then
+    # We want to propagate cache being disabled so that cache_upload and cache_download can catch this
+    echo "disabled-cache"
+  else
+    echo $1 | git hash-object --stdin | cut -c1-16 | tr -d '\n'
+  fi
 }
 
 function echo_stderr {
@@ -28,4 +33,4 @@ function echo_stderr {
 
 export -f hash_str echo_stderr
 
-source $ci3/source_color
+source $ci3/source_color

ci3/source_bootstrap

@@ -11,4 +11,4 @@ case "${1:-}" in
   ""|"fast")
     export USE_CACHE=${USE_CACHE:-1}
     ;;
-esac
+esac

docs/bootstrap.sh

@@ -3,7 +3,7 @@ source $(git rev-parse --show-toplevel)/ci3/source_bootstrap
 
 cmd=${1:-}
 # combine yarn project hash
-hash=$(echo $(../yarn-project/bootstrap.sh hash) $(cache_content_hash .rebuild_patterns) | git hash-object --stdin)
+hash=$(hash_str "$(../yarn-project/bootstrap.sh hash) $(cache_content_hash .rebuild_patterns)")
 # TODO(ci3): build command
 case "$cmd" in
   "hash")
@@ -12,4 +12,4 @@ case "$cmd" in
   *)
     echo "Unknown command: $cmd"
     exit 1
-esac
+esac