From dfb4817f43b89d22c09177a13e4c685b52db0103 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Thu, 19 Dec 2024 12:56:29 +0000 Subject: [PATCH 1/5] fix --- barretenberg/cpp/bootstrap.sh | 2 +- noir-projects/noir-contracts/bootstrap.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/barretenberg/cpp/bootstrap.sh b/barretenberg/cpp/bootstrap.sh index 7ea2de2e510..aad3427e64a 100755 --- a/barretenberg/cpp/bootstrap.sh +++ b/barretenberg/cpp/bootstrap.sh @@ -115,7 +115,7 @@ case "$cmd" in test ;; "ci") - build_native + build test ;; "hash") diff --git a/noir-projects/noir-contracts/bootstrap.sh b/noir-projects/noir-contracts/bootstrap.sh index 7330bbc26e1..180b9fe0e04 100755 --- a/noir-projects/noir-contracts/bootstrap.sh +++ b/noir-projects/noir-contracts/bootstrap.sh @@ -97,7 +97,7 @@ function compile { contract_name=$(cat contracts/$1/src/main.nr | awk '/^contract / { print $2 }') local filename="$contract-$contract_name.json" local json_path="./target/$filename" - export REBUILD_PATTERNS="^noir-projects/noir-contracts/contracts/$contract/" + export REBUILD_PATTERNS="^noir-projects/noir-contracts/contracts/$contract/ ^noir-projects/aztec-nr/" contract_hash="$(cache_content_hash ../../noir/.rebuild_patterns ../../avm-transpiler/.rebuild_patterns)" if ! cache_download contract-$contract_hash.tar.gz &> /dev/null; then $NARGO compile --package $contract --silence-warnings --inliner-aggressiveness 0 From 841e740ef347b9ec0b3810657013ae64c7facf57 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Thu, 19 Dec 2024 13:09:02 +0000 Subject: [PATCH 2/5] denoise --- avm-transpiler/bootstrap.sh | 4 ++-- ci3/bootstrap_local | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/avm-transpiler/bootstrap.sh b/avm-transpiler/bootstrap.sh index 7e1f37c8ea6..d4429e25a4d 100755 --- a/avm-transpiler/bootstrap.sh +++ b/avm-transpiler/bootstrap.sh @@ -17,8 +17,8 @@ function build { } function test { - cargo fmt --check - cargo clippy + denoise cargo fmt --check + denoise cargo clippy } case "$cmd" in diff --git a/ci3/bootstrap_local b/ci3/bootstrap_local index 2b1c78d4e60..4fcb03ee5fd 100755 --- a/ci3/bootstrap_local +++ b/ci3/bootstrap_local @@ -26,6 +26,7 @@ docker run $interactive_args --rm \ -v $root:/aztec-packages-host:ro \ -v $HOME/.aws:/root/.aws \ aztecprotocol/ci:2.0 bash -c " + set -e /usr/local/share/docker-init.sh &> /dev/null git config --global --add safe.directory /aztec-packages-host/.git mkdir -p /root/aztec-packages && cd /root/aztec-packages From 78e9f54b7de7271076780be9c55c87dc80139350 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Thu, 19 Dec 2024 14:04:06 +0000 Subject: [PATCH 3/5] cache args improv --- build-images/bootstrap.sh | 2 +- ci3/bootstrap_local | 17 ++++--------- ci3/cache_content_hash | 25 +++++++++++-------- noir-projects/noir-contracts/bootstrap.sh | 8 ++++-- .../noir-protocol-circuits/bootstrap.sh | 2 +- 5 files changed, 28 insertions(+), 26 deletions(-) diff --git a/build-images/bootstrap.sh b/build-images/bootstrap.sh index b177159ed28..6b5d377ad96 100755 --- a/build-images/bootstrap.sh +++ b/build-images/bootstrap.sh @@ -3,7 +3,7 @@ source $(git rev-parse --show-toplevel)/ci3/source_bootstrap cmd=${1:-} -hash=$(REBUILD_PATTERNS="^build-images/Earthfile" cache_content_hash) +hash=$(cache_content_hash "^build-images/Earthfile") function build { github_group "build-images build" diff --git a/ci3/bootstrap_local b/ci3/bootstrap_local index 4fcb03ee5fd..7462eaf0a2a 100755 --- a/ci3/bootstrap_local +++ b/ci3/bootstrap_local @@ -9,18 +9,13 @@ root=$(git rev-parse --show-toplevel) source $root/ci3/source -current_commit=$(git rev-parse HEAD) cmd=${1:-"CI=1 ./bootstrap.sh fast || exec bash"} if ! git diff --quiet; then - echo "There are local changes to tracked files." - exit 1 + echo -e "${yellow}WARNING: There are local changes to tracked files.${reset}" fi -interactive_args="" -# Only add -ti and a name if we are not in CI. -[ "${CI:-0}" = "0" ] && interactive_args="--name aztec_build -ti" -docker run $interactive_args --rm \ +docker run --name aztec_build -ti --rm \ --privileged \ -v bootstrap_ci_local_docker:/var/lib/docker \ -v $root:/aztec-packages-host:ro \ @@ -29,11 +24,9 @@ docker run $interactive_args --rm \ set -e /usr/local/share/docker-init.sh &> /dev/null git config --global --add safe.directory /aztec-packages-host/.git - mkdir -p /root/aztec-packages && cd /root/aztec-packages + cd /root # Ensure we get a clean clone of the repo. - git init &>/dev/null - git remote add origin https://github.com/aztecprotocol/aztec-packages - git fetch --depth 1 origin $current_commit 2>/dev/null || (echo 'The commit was not pushed, run aborted.' && exit 1) - git checkout FETCH_HEAD &>/dev/null + git clone --depth 1 file:///aztec-packages-host aztec-packages + cd aztec-packages $cmd " diff --git a/ci3/cache_content_hash b/ci3/cache_content_hash index ad0d8e7e2cd..833a1907441 100755 --- a/ci3/cache_content_hash +++ b/ci3/cache_content_hash @@ -2,24 +2,29 @@ set -euo pipefail [ "${BUILD_SYSTEM_DEBUG:-}" = 1 ] && set -x -# Ensure REBUILD_PATTERNS or args are set -if [[ -z "${REBUILD_PATTERNS:-}" && "$#" = 0 ]]; then - echo "Error: No arguments provided and REBUILD_PATTERNS environment variable is not set." +# Ensure args are set +if [[ "$#" = 0 ]]; then + echo "Error: No arguments provided." exit 1 fi # If too many spurious cache misses: can be customized to pin artifacts to a specific version AZTEC_CACHE_COMMIT=${AZTEC_CACHE_COMMIT:-HEAD} PLATFORM_TAG="${PLATFORM_TAG:-${OSTYPE:-unknown}-$(uname -m)}" -# Literal patterns can be provided with REBUILD_PATTERNS -REBUILD_PATTERNS=${REBUILD_PATTERNS:-} -if [ "$#" != 0 ]; then - REBUILD_PATTERNS+=$'\n'$(cat "$@") -fi + +rebuild_patterns=() +for arg in "$@"; do + if [[ -f "$arg" ]]; then + rebuild_patterns+=$(cat "$arg") + rebuild_patterns+=$'\n' + else + rebuild_patterns+="$arg"$'\n' + fi +done # Concatenate patterns with '|' and double escape backslashes for AWK # filter empty lines -AWK_PATTERN=$(echo "$REBUILD_PATTERNS" | grep -v '^$' | sed 's/\\/\\\\/g' | tr '\n' '|' | sed 's/|$//') +AWK_PATTERN=$(echo "$rebuild_patterns" | grep -v '^$' | sed 's/\\/\\\\/g' | tr '\n' '|' | sed 's/|$//') # use git repo root because that is where our patterns are focused cd $(git rev-parse --show-toplevel) @@ -30,7 +35,7 @@ CONTENT_HASH=$(git ls-tree -r $AZTEC_CACHE_COMMIT | awk -v pattern="($AWK_PATTER # Check if file list was empty by comparing against the result of 'echo '' | git hash-object --stdin | cut -c1-16' ECHO_BLANK_HASH="8b137891791fe969" if [ "$CONTENT_HASH" = "$ECHO_BLANK_HASH" ]; then - echo "No files matched the rebuild patterns $REBUILD_PATTERNS." + echo "No files matched the rebuild patterns $rebuild_patterns." echo "Awk pattern expanded: $AWK_PATTERN." exit 1 fi diff --git a/noir-projects/noir-contracts/bootstrap.sh b/noir-projects/noir-contracts/bootstrap.sh index 180b9fe0e04..1c8257a58f5 100755 --- a/noir-projects/noir-contracts/bootstrap.sh +++ b/noir-projects/noir-contracts/bootstrap.sh @@ -97,8 +97,12 @@ function compile { contract_name=$(cat contracts/$1/src/main.nr | awk '/^contract / { print $2 }') local filename="$contract-$contract_name.json" local json_path="./target/$filename" - export REBUILD_PATTERNS="^noir-projects/noir-contracts/contracts/$contract/ ^noir-projects/aztec-nr/" - contract_hash="$(cache_content_hash ../../noir/.rebuild_patterns ../../avm-transpiler/.rebuild_patterns)" + contract_hash="$(cache_content_hash \ + ../../noir/.rebuild_patterns \ + ../../avm-transpiler/.rebuild_patterns \ + "^noir-projects/noir-contracts/contracts/$contract/" \ + "^noir-projects/aztec-nr/" \ + )" if ! cache_download contract-$contract_hash.tar.gz &> /dev/null; then $NARGO compile --package $contract --silence-warnings --inliner-aggressiveness 0 $TRANSPILER $json_path $json_path diff --git a/noir-projects/noir-protocol-circuits/bootstrap.sh b/noir-projects/noir-protocol-circuits/bootstrap.sh index 4038f9c9b05..de3b1796937 100755 --- a/noir-projects/noir-protocol-circuits/bootstrap.sh +++ b/noir-projects/noir-protocol-circuits/bootstrap.sh @@ -111,7 +111,7 @@ function test { set -eu # Whether we run the tests or not is coarse grained. name=$(basename "$PWD") - CIRCUITS_HASH=$(REBUILD_PATTERNS="^noir-projects/$name" cache_content_hash ../../noir/.rebuild_patterns) + CIRCUITS_HASH=$(cache_content_hash ../../noir/.rebuild_patterns "^noir-projects/$name") if ! test_should_run $name-tests-$CIRCUITS_HASH; then return fi From 3155db7d8c8eb5edc42c01e32734a519039bc9fe Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Thu, 19 Dec 2024 16:27:41 +0000 Subject: [PATCH 4/5] things --- .../acir_tests/browser-test-app/yarn.lock | 6 +++--- ci3/bootstrap_local | 2 +- ci3/dump_fail | 2 +- ci3/source_bootstrap | 2 +- .../noir-protocol-circuits/bootstrap.sh | 20 ++++++++++++------- 5 files changed, 19 insertions(+), 13 deletions(-) diff --git a/barretenberg/acir_tests/browser-test-app/yarn.lock b/barretenberg/acir_tests/browser-test-app/yarn.lock index d209e4296d3..37157884379 100644 --- a/barretenberg/acir_tests/browser-test-app/yarn.lock +++ b/barretenberg/acir_tests/browser-test-app/yarn.lock @@ -6,8 +6,8 @@ __metadata: cacheKey: 10c0 "@aztec/bb.js@file:../../ts::locator=browser-test-app%40workspace%3A.": - version: 0.67.0 - resolution: "@aztec/bb.js@file:../../ts#../../ts::hash=e236e3&locator=browser-test-app%40workspace%3A." + version: 0.67.1 + resolution: "@aztec/bb.js@file:../../ts#../../ts::hash=29e47a&locator=browser-test-app%40workspace%3A." dependencies: comlink: "npm:^4.4.1" commander: "npm:^12.1.0" @@ -17,7 +17,7 @@ __metadata: tslib: "npm:^2.4.0" bin: bb.js: ./dest/node/main.js - checksum: 10c0/4ca2234f9d1a4b59a60733979f97c5ca09f508e4d3091bf33755ee6a80f03f733e393ea848b812bc73bc1f0673a3d92f96462a1cbd5aa01eeca4e9d5a3be367f + checksum: 10c0/c01128ff74f29b6bbc5c46362792525ef5612c5fc8787341551bcf457ba9816a971e24a74292ab230c47b0b9efe8d7e0d1cabd44247e1b6e718727d0b6372400 languageName: node linkType: hard diff --git a/ci3/bootstrap_local b/ci3/bootstrap_local index 7462eaf0a2a..28a141afe13 100755 --- a/ci3/bootstrap_local +++ b/ci3/bootstrap_local @@ -9,7 +9,7 @@ root=$(git rev-parse --show-toplevel) source $root/ci3/source -cmd=${1:-"CI=1 ./bootstrap.sh fast || exec bash"} +cmd=${1:-"./bootstrap.sh ci || exec bash"} if ! git diff --quiet; then echo -e "${yellow}WARNING: There are local changes to tracked files.${reset}" diff --git a/ci3/dump_fail b/ci3/dump_fail index c40ddf0cbf2..5e933822055 100755 --- a/ci3/dump_fail +++ b/ci3/dump_fail @@ -30,7 +30,7 @@ if [ "$status" -ne 0 ]; then cat "$stderr" } >&2 else - echo "$output" + echo -n "$output" fi exit $status diff --git a/ci3/source_bootstrap b/ci3/source_bootstrap index 6bc88a65f32..2a3cc2ba030 100644 --- a/ci3/source_bootstrap +++ b/ci3/source_bootstrap @@ -10,7 +10,7 @@ case "${1:-}" in export DENOISE=${DENOISE:-1} ;; ""|"fast") - export USE_CACHE=1 + export USE_CACHE=${USE_CACHE:-1} ;; "test") export TEST=1 diff --git a/noir-projects/noir-protocol-circuits/bootstrap.sh b/noir-projects/noir-protocol-circuits/bootstrap.sh index de3b1796937..45184372fcf 100755 --- a/noir-projects/noir-protocol-circuits/bootstrap.sh +++ b/noir-projects/noir-protocol-circuits/bootstrap.sh @@ -33,7 +33,7 @@ function on_exit() { } trap on_exit EXIT -[ -f package.json ] && yarn && node ./scripts/generate_variants.js +[ -f package.json ] && denoise "yarn && node ./scripts/generate_variants.js" mkdir -p $tmp_dir mkdir -p $key_dir @@ -48,12 +48,18 @@ function compile { local filename="$name.json" local json_path="./target/$filename" local program_hash hash bytecode_hash vk vk_fields - program_hash=$($NARGO check --package $name --silence-warnings --show-program-hash | cut -d' ' -f2) - hash=$(echo "$NARGO_HASH-$program_hash" | sha256sum | tr -d ' -') - if ! cache_download circuit-$hash.tar.gz &> /dev/null; then + local program_hash_cmd="$NARGO check --package $name --silence-warnings --show-program-hash | cut -d' ' -f2" + # echo $program_hash_cmd >&2 + program_hash=$(dump_fail "$program_hash_cmd") + echo "Hash preimage: $NARGO_HASH-$program_hash" + hash=$(hash_str "$NARGO_HASH-$program_hash") + if ! cache_download circuit-$hash.tar.gz 1>&2; then SECONDS=0 + rm -f $json_path # TODO: --skip-brillig-constraints-check added temporarily for blobs build time. - $NARGO compile --package $name --silence-warnings --skip-brillig-constraints-check + local compile_cmd="$NARGO compile --package $name --silence-warnings --skip-brillig-constraints-check" + echo "$compile_cmd" + dump_fail "$compile_cmd" echo "Compilation complete for: $name (${SECONDS}s)" cache_upload circuit-$hash.tar.gz $json_path &> /dev/null fi @@ -74,8 +80,8 @@ function compile { # Change this to add verification_key to original json, like contracts does. # Will require changing TS code downstream. bytecode_hash=$(jq -r '.bytecode' $json_path | sha256sum | tr -d ' -') - hash=$(echo "$BB_HASH-$bytecode_hash-$proto" | sha256sum | tr -d ' -') - if ! cache_download vk-$hash.tar.gz &> /dev/null; then + hash=$(hash_str "$BB_HASH-$bytecode_hash-$proto") + if ! cache_download vk-$hash.tar.gz 1>&2; then local key_path="$key_dir/$name.vk.data.json" echo "Generating vk for function: $name..." >&2 SECONDS=0 From f5cf7e10d186d5a3a3c81aa554e2a85c54271f2b Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Thu, 19 Dec 2024 16:55:53 +0000 Subject: [PATCH 5/5] echo_stderr --- ci3/source | 7 ++++++- noir-projects/noir-contracts/bootstrap.sh | 17 +++++++---------- .../noir-protocol-circuits/bootstrap.sh | 18 +++++++++--------- 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/ci3/source b/ci3/source index b54c70f663e..063437456d9 100644 --- a/ci3/source +++ b/ci3/source @@ -18,6 +18,11 @@ function hash_str { set -eu echo $1 | git hash-object --stdin | cut -c1-16 | tr -d '\n' } -export -f hash_str + +function echo_stderr { + echo $@ >&2 +} + +export -f hash_str echo_stderr source $ci3/source_color \ No newline at end of file diff --git a/noir-projects/noir-contracts/bootstrap.sh b/noir-projects/noir-contracts/bootstrap.sh index 1c8257a58f5..aa43652e2d7 100755 --- a/noir-projects/noir-contracts/bootstrap.sh +++ b/noir-projects/noir-contracts/bootstrap.sh @@ -7,7 +7,7 @@ # - The exported functions called by parallel must enable their own flags at the start e.g. set -euo pipefail # - The exported functions are using stdin/stdout, so be very careful about what's printed where. # - The exported functions need to have external variables they require, to have been exported first. -# - If you want to echo something, send it to stderr e.g. echo "My debug" >&2 +# - If you want to echo something, send it to stderr e.g. echo_stderr "My debug" # - If you call another script, be sure it also doesn't output something you don't want. # - Note calls to cache scripts swallow everything with &> /dev/null. # - Local assignments with subshells don't propagate errors e.g. local capture=$(false). Declare locals separately. @@ -58,7 +58,7 @@ function process_function() { set +e make_vk=$(echo "$func" | jq -e '(.custom_attributes | index("public") == null) and (.is_unconstrained == false)') if [ $? -ne 0 ] && [ "$make_vk" != "false" ]; then - echo "Failed to check function $name is neither public nor unconstrained." >&2 + echo_stderr "Failed to check function $name is neither public nor unconstrained." exit 1 fi set -e @@ -70,7 +70,7 @@ function process_function() { hash=$((echo "$BB_HASH"; echo "$bytecode_b64") | sha256sum | tr -d ' -') if ! cache_download vk-$hash.tar.gz &> /dev/null; then # It's not in the cache. Generate the vk file and upload it to the cache. - echo "Generating vk for function: $name..." >&2 + echo_stderr "Generating vk for function: $name..." echo "$bytecode_b64" | base64 -d | gunzip | $BB write_vk_for_ivc -h -b - -o $tmp_dir/$hash 2>/dev/null cache_upload vk-$hash.tar.gz $tmp_dir/$hash &> /dev/null fi @@ -129,7 +129,7 @@ function build { compile $1 else set +e - echo "Compiling contracts (bb-hash: $BB_HASH)..." + echo_stderr "Compiling contracts (bb-hash: $BB_HASH)..." grep -oP '(?<=contracts/)[^"]+' Nargo.toml | \ parallel -j16 --joblog joblog.txt -v --line-buffer --tag --halt now,fail=1 compile {} code=$? @@ -147,15 +147,12 @@ case "$cmd" in ;; "clean-keys") for artifact in target/*.json; do - echo "Scrubbing vk from $artifact..." + echo_stderr "Scrubbing vk from $artifact..." jq '.functions |= map(del(.verification_key))' "$artifact" > "${artifact}.tmp" mv "${artifact}.tmp" "$artifact" done ;; - ""|"fast"|"ci") - USE_CACHE=1 build - ;; - "full") + ""|"fast"|"full"|"ci") build ;; "compile") @@ -167,6 +164,6 @@ case "$cmd" in exit 0 ;; *) - echo "Unknown command: $cmd" + echo_stderr "Unknown command: $cmd" exit 1 esac \ No newline at end of file diff --git a/noir-projects/noir-protocol-circuits/bootstrap.sh b/noir-projects/noir-protocol-circuits/bootstrap.sh index 45184372fcf..1abd3b83255 100755 --- a/noir-projects/noir-protocol-circuits/bootstrap.sh +++ b/noir-projects/noir-protocol-circuits/bootstrap.sh @@ -49,18 +49,18 @@ function compile { local json_path="./target/$filename" local program_hash hash bytecode_hash vk vk_fields local program_hash_cmd="$NARGO check --package $name --silence-warnings --show-program-hash | cut -d' ' -f2" - # echo $program_hash_cmd >&2 + # echo_stderr $program_hash_cmd program_hash=$(dump_fail "$program_hash_cmd") - echo "Hash preimage: $NARGO_HASH-$program_hash" + echo_stderr "Hash preimage: $NARGO_HASH-$program_hash" hash=$(hash_str "$NARGO_HASH-$program_hash") if ! cache_download circuit-$hash.tar.gz 1>&2; then SECONDS=0 rm -f $json_path # TODO: --skip-brillig-constraints-check added temporarily for blobs build time. local compile_cmd="$NARGO compile --package $name --silence-warnings --skip-brillig-constraints-check" - echo "$compile_cmd" + echo_stderr "$compile_cmd" dump_fail "$compile_cmd" - echo "Compilation complete for: $name (${SECONDS}s)" + echo_stderr "Compilation complete for: $name (${SECONDS}s)" cache_upload circuit-$hash.tar.gz $json_path &> /dev/null fi @@ -83,16 +83,16 @@ function compile { hash=$(hash_str "$BB_HASH-$bytecode_hash-$proto") if ! cache_download vk-$hash.tar.gz 1>&2; then local key_path="$key_dir/$name.vk.data.json" - echo "Generating vk for function: $name..." >&2 + echo_stderr "Generating vk for function: $name..." SECONDS=0 local vk_cmd="jq -r '.bytecode' $json_path | base64 -d | gunzip | $BB $write_vk_cmd -h -b - -o - --recursive | xxd -p -c 0" - echo $vk_cmd >&2 + echo_stderr $vk_cmd vk=$(dump_fail "$vk_cmd") local vkf_cmd="echo '$vk' | xxd -r -p | $BB $vk_as_fields_cmd -k - -o -" - # echo $vkf_cmd >&2 + # echo_stderrr $vkf_cmd vk_fields=$(dump_fail "$vkf_cmd") jq -n --arg vk "$vk" --argjson vkf "$vk_fields" '{keyAsBytes: $vk, keyAsFields: $vkf}' > $key_path - echo "Key output at: $key_path (${SECONDS}s)" + echo_stderr "Key output at: $key_path (${SECONDS}s)" cache_upload vk-$hash.tar.gz $key_path &> /dev/null fi } @@ -150,6 +150,6 @@ case "$CMD" in parallel --line-buffered bash -c {} ::: build test ;; *) - echo "Unknown command: $CMD" + echo_stderr "Unknown command: $CMD" exit 1 esac