diff --git a/.circleci/config.yml b/.circleci/config.yml index 794d5a090b7..fb0ea6e4b9a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -358,15 +358,15 @@ jobs: command: build yarn-project-prod | add_timestamps yarn-project-formatting: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: Check Formatting - command: cond_run_container yarn-project formatting | add_timestamps + command: cond_spot_run_container yarn-project 8 formatting | add_timestamps yarn-project-tests: docker: diff --git a/aztec-up/bin/.aztec-run b/aztec-up/bin/.aztec-run index c338bede2e1..82b12deb89a 100755 --- a/aztec-up/bin/.aztec-run +++ b/aztec-up/bin/.aztec-run @@ -13,16 +13,24 @@ VERSION=${VERSION:-"latest"} DOCKER_HOST_BINDS="" # Volumes to pass to the container. -DOCKER_VOLUME="" +DOCKER_VOLUME="-v $HOME:/root" + +# Colors. +y="\033[33m" +r="\033[0m" + +function warn { + echo -e "${y}$1${r}" +} if ! command -v docker &> /dev/null; then - echo "No docker found." + warn "No docker found." exit 1 fi -# Colors. -yellow="\033[33m" -reset="\033[0m" +if [[ $PWD != ${HOME}* ]]; then + warn "Due to how we containerize our applications, we require your working directory to be somewhere within $HOME." +fi # Set up host.docker.internal alias on Linux, just like it is on mac. UNAME=$(uname -s) @@ -30,51 +38,28 @@ if [ "$UNAME" == "Linux" ]; then if docker info 2>/dev/null | grep -q rootless; then # We're in rootless docker. Probe for the host ip and use that. ip=$(hostname -I | head | tr -d ' ') - echo -e "${yellow}WARNING: Running within rootless docker. Using $ip as host ip. Ensure listening services are listening on this interface.${reset}" + warn "WARNING: Running within rootless docker. Using $ip as host ip. Ensure listening services are listening on this interface." DOCKER_HOST_BINDS="$DOCKER_HOST_BINDS --add-host host.docker.internal:$ip" else DOCKER_HOST_BINDS="$DOCKER_HOST_BINDS --add-host host.docker.internal:host-gateway" fi fi -# Build a list of mount points -function add_mount() { - DIR="${1:-}" - - # Grab its dirname if its a file. - if [ -f "$DIR" ]; then - DIR=$(dirname "$DIR") - fi - - if [ ! -d "$DIR" ]; then - return - fi - - # Check if it's already been added. - REALDIR=$(realpath $DIR) - if [[ "$DOCKER_VOLUME" =~ "$REALDIR:" ]]; then - return - fi - - DOCKER_VOLUME="$DOCKER_VOLUME -v $REALDIR:$REALDIR" -} - -# Always mount the CWD into the container. -add_mount "$PWD" - # Substitute any references to localhost with our host gateway. +# TODO: In node, we can hook the resolve override for localhost with host.docker.internal. +# Consider if we should just do that, but that wouldn't help e.g. nargo. args=("$@") for i in "${!args[@]}"; do args[$i]=${args[$i]//localhost/host.docker.internal} done -# Check if it's either a filename or a directory that exists outside the CWD. -# If it is then mount inside the container. -# NOTE: This won't work with assignement-style flags, e.g. --outdir=/foo +# Check if it's either a filename or a directory that exists outside the HOME. +# If so, warn and exit. for i in "${!args[@]}"; do arg=${args[$i]} - if [[ -f "$arg" || -d "$arg" && $(realpath $arg) != ${PWD}* ]]; then - add_mount "$arg" + if [[ -f "$arg" || -d "$arg" && $(realpath $arg) != ${HOME}* ]]; then + warn "Due to how we containerize our applications, paths outside of $HOME cannot be referenced." + exit 1 fi done @@ -91,8 +76,7 @@ DOCKER_VOLUME="$DOCKER_VOLUME -v cache:/cache" docker run \ -ti \ --rm \ - --user $(id -u):$(id -g) \ - --workdir "$PWD" \ + --workdir "${PWD/$HOME/\/root}" \ $DOCKER_HOST_BINDS \ $DOCKER_ENV \ $DOCKER_VOLUME \ diff --git a/aztec-up/bin/aztec-install b/aztec-up/bin/aztec-install index 653e80e9886..e6c7ec5ebf4 100755 --- a/aztec-up/bin/aztec-install +++ b/aztec-up/bin/aztec-install @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -euo pipefail # Colors @@ -29,7 +29,16 @@ function title() { echo -e "Welcome to the ${bold}${b}Aztec${r} installer! Your journey into blockchain privacy begins... ${bold}${p}now${r}." echo -e "We presently leverage docker to simplify releases of our complex project." echo -e "Please ensure it's installed for your platform: https://docs.docker.com/engine/install" - echo + if [ "$(uname -s)" == "Darwin" ]; then + echo + echo -e "${y}WARNING: For best performance we recommend adjusting your default docker settings:" + echo -e " - Under general, enable VirtioFS." + echo -e " - Under resources, set CPUs to ~80-100% your maximum." + echo -e " - Under resources, set Memory to ~80% your maximum." + echo -e "You may receive a warning about your home directory being mounted into a container." + echo -e "This is requested so we can read and write project files, that is all." + echo -e "${r}" + fi echo -e "This will install the following scripts and update your PATH if necessary:" echo -e " ${bold}${g}aztec${r} - launches various infrastructure subsystems (sequencer, prover, pxe, etc)." echo -e " ${bold}${g}aztec-cli${r} - a command line tool for interfacing and experimenting with infrastructure." @@ -91,10 +100,12 @@ function pull_container { fi } -info "Pulling aztec version $VERSION..." -pull_container aztec-sandbox -pull_container cli -pull_container noir +if [ -z "${SKIP_PULL:-}" ]; then + info "Pulling aztec version $VERSION..." + pull_container aztec-sandbox + pull_container cli + pull_container noir +fi # Download the Docker Compose file. Used by aztec-start. curl -fsSL http://$INSTALL_HOST/docker-compose.yml -o $BIN_PATH/docker-compose.yml diff --git a/aztec-up/bin/aztec-sandbox b/aztec-up/bin/aztec-sandbox index ccbe9747045..1558bc49cb2 100755 --- a/aztec-up/bin/aztec-sandbox +++ b/aztec-up/bin/aztec-sandbox @@ -1,11 +1,21 @@ -#!/bin/bash +#!/usr/bin/env bash set -euo pipefail -# Change working dir, so relative volume mounts are in the right place. -cd ~/.aztec - # Favour 'docker compose', falling back on docker-compose. CMD="docker compose" $CMD &>/dev/null || CMD="docker-compose" -$CMD -f ~/.aztec/bin/docker-compose.yml up \ No newline at end of file +ARGS="-f $HOME/.aztec/bin/docker-compose.yml -p sandbox" + +# Function to be executed when SIGINT is received. +cleanup() { + $CMD $ARGS down +} + +# Set trap to catch SIGINT and call the cleanup function. +trap cleanup SIGINT + +# Change working dir, so relative volume mounts are in the right place. +cd ~/.aztec + +$CMD $ARGS up --force-recreate --remove-orphans \ No newline at end of file diff --git a/aztec-up/bin/aztec-up b/aztec-up/bin/aztec-up index d3b88660090..736307e1827 100755 --- a/aztec-up/bin/aztec-up +++ b/aztec-up/bin/aztec-up @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -euo pipefail export SKIP_TITLE=1 diff --git a/aztec-up/bin/docker-compose.yml b/aztec-up/bin/docker-compose.yml index 735466e3904..b861d729d16 100644 --- a/aztec-up/bin/docker-compose.yml +++ b/aztec-up/bin/docker-compose.yml @@ -1,27 +1,28 @@ -version: '3' +version: "3" services: ethereum: image: ghcr.io/foundry-rs/foundry@sha256:29ba6e34379e79c342ec02d437beb7929c9e254261e8032b17e187be71a2609f - entrypoint: > - sh -c ' - if [ -n "$FORK_BLOCK_NUMBER" ] && [ -n "$FORK_URL" ]; then - exec anvil -p 8545 --host 0.0.0.0 --chain-id 31337 --silent --fork-url "$FORK_URL" --fork-block-number "$FORK_BLOCK_NUMBER" - elif [ -n "$FORK_URL" ]; then - exec anvil -p 8545 --host 0.0.0.0 --chain-id 31337 --silent --fork-url "$FORK_URL" - else - exec anvil -p 8545 --host 0.0.0.0 --chain-id 31337 --silent - fi' + command: > + ' + [ -n "$$FORK_URL" ] && ARGS="$$ARGS --fork-url $$FORK_URL"; + [ -n "$$FORK_BLOCK_NUMBER" ] && ARGS="$$ARGS --fork-block-number $$FORK_BLOCK_NUMBER"; + echo anvil -p 8545 --host 0.0.0.0 --chain-id 31337 --silent $$ARGS; + anvil -p 8545 --host 0.0.0.0 --chain-id 31337 --silent $$ARGS + ' ports: - - '${SANDBOX_ANVIL_PORT:-8545}:8545' + - "${SANDBOX_ANVIL_PORT:-8545}:8545" + environment: + FORK_URL: + FORK_BLOCK_NUMBER: aztec: - image: 'aztecprotocol/aztec-sandbox' + image: "aztecprotocol/aztec-sandbox" ports: - - '${SANDBOX_AZTEC_NODE_PORT:-8079}:8079' - - '${SANDBOX_PXE_PORT:-8080}:8080' + - "${SANDBOX_AZTEC_NODE_PORT:-8079}:8079" + - "${SANDBOX_PXE_PORT:-8080}:8080" environment: DEBUG: # Loaded from the user shell if explicitly set - HOST_WORKDIR: '${PWD}' # Loaded from the user shell to show log files absolute path in host + HOST_WORKDIR: "${PWD}" # Loaded from the user shell to show log files absolute path in host ETHEREUM_HOST: http://ethereum:8545 CHAIN_ID: 31337 ARCHIVER_POLLING_INTERVAL_MS: 50 diff --git a/barretenberg/acir_tests/Dockerfile.noir_acir_tests b/barretenberg/acir_tests/Dockerfile.noir_acir_tests index 87fdd8604a4..5131e94e279 100644 --- a/barretenberg/acir_tests/Dockerfile.noir_acir_tests +++ b/barretenberg/acir_tests/Dockerfile.noir_acir_tests @@ -3,7 +3,7 @@ # So, it lives here. # This chains off the nargo build, and creates a container with a compiled set of acir tests. FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir -RUN apk add bash jq +RUN apt update && apt install -y jq && rm -rf /var/lib/apt/lists/* && apt-get clean ENV PATH="/usr/src/noir/target/release:${PATH}" WORKDIR /usr/src/noir/test_programs COPY . . diff --git a/barretenberg/acir_tests/bash_helpers/catch.sh b/barretenberg/acir_tests/bash_helpers/catch.sh index 888af3cbb44..bc2025d4da5 100644 --- a/barretenberg/acir_tests/bash_helpers/catch.sh +++ b/barretenberg/acir_tests/bash_helpers/catch.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Handler for SIGCHLD, cleanup if child exit with error handle_sigchild() { diff --git a/barretenberg/acir_tests/bench_acir_tests.sh b/barretenberg/acir_tests/bench_acir_tests.sh index b338e82e30b..a41b261d1cc 100755 --- a/barretenberg/acir_tests/bench_acir_tests.sh +++ b/barretenberg/acir_tests/bench_acir_tests.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash TEST_NAMES=("$@") THREADS=(1 4 16 32 64) diff --git a/barretenberg/acir_tests/clone_test_vectors.sh b/barretenberg/acir_tests/clone_test_vectors.sh index c6fd6ef1bf1..b15814ab013 100755 --- a/barretenberg/acir_tests/clone_test_vectors.sh +++ b/barretenberg/acir_tests/clone_test_vectors.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eu TEST_SRC=${TEST_SRC:-../../noir/test_programs/acir_artifacts} diff --git a/barretenberg/acir_tests/gen_inner_proof_inputs.sh b/barretenberg/acir_tests/gen_inner_proof_inputs.sh index 64b87fe19db..36137bde82e 100755 --- a/barretenberg/acir_tests/gen_inner_proof_inputs.sh +++ b/barretenberg/acir_tests/gen_inner_proof_inputs.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Env var overrides: # BIN: to specify a different binary to test with (e.g. bb.js or bb.js-dev). set -eu diff --git a/barretenberg/acir_tests/run_acir_tests.sh b/barretenberg/acir_tests/run_acir_tests.sh index 84953d01c55..ee28c975113 100755 --- a/barretenberg/acir_tests/run_acir_tests.sh +++ b/barretenberg/acir_tests/run_acir_tests.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Env var overrides: # BIN: to specify a different binary to test with (e.g. bb.js or bb.js-dev). # VERBOSE: to enable logging for each test. @@ -80,8 +80,8 @@ else # If parallel flag is set, run in parallel if [ -n "${PARALLEL:-}" ]; then test $TEST_NAME & - else - test $TEST_NAME + else + test $TEST_NAME fi done fi diff --git a/barretenberg/acir_tests/run_acir_tests_browser.sh b/barretenberg/acir_tests/run_acir_tests_browser.sh index 22830656250..1c1f2ce0e08 100755 --- a/barretenberg/acir_tests/run_acir_tests_browser.sh +++ b/barretenberg/acir_tests/run_acir_tests_browser.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -em cleanup() { diff --git a/barretenberg/bootstrap.sh b/barretenberg/bootstrap.sh index 504f1980982..c16fd294b6c 100755 --- a/barretenberg/bootstrap.sh +++ b/barretenberg/bootstrap.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eu cd "$(dirname "$0")" diff --git a/barretenberg/cpp/bootstrap.sh b/barretenberg/cpp/bootstrap.sh index 25350c4fcce..0b9295bafbd 100755 --- a/barretenberg/cpp/bootstrap.sh +++ b/barretenberg/cpp/bootstrap.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eu # Navigate to script folder diff --git a/barretenberg/cpp/format.sh b/barretenberg/cpp/format.sh index 0bf8bca805c..ae314e96a6f 100755 --- a/barretenberg/cpp/format.sh +++ b/barretenberg/cpp/format.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -e if [ "$1" == "staged" ]; then diff --git a/barretenberg/cpp/scripts/bb-tests.sh b/barretenberg/cpp/scripts/bb-tests.sh index 459eb756593..a3f322b2afa 100755 --- a/barretenberg/cpp/scripts/bb-tests.sh +++ b/barretenberg/cpp/scripts/bb-tests.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # This script runs all test suites that have not been broken out into their own jobs for parallelisation. # Might be better to list exclusions here rather than inclusions as risky to maintain. set -eu diff --git a/barretenberg/cpp/scripts/benchmarks.sh b/barretenberg/cpp/scripts/benchmarks.sh index a7ec1dcb1d0..8a547c29aba 100755 --- a/barretenberg/cpp/scripts/benchmarks.sh +++ b/barretenberg/cpp/scripts/benchmarks.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eu # Move above script dir. diff --git a/barretenberg/cpp/scripts/collect_coverage_information.sh b/barretenberg/cpp/scripts/collect_coverage_information.sh index b018e251483..0a3231a304c 100755 --- a/barretenberg/cpp/scripts/collect_coverage_information.sh +++ b/barretenberg/cpp/scripts/collect_coverage_information.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash if [ $# -ne 2 ]; then echo "Usage: $0 " diff --git a/barretenberg/cpp/scripts/collect_heap_information.sh b/barretenberg/cpp/scripts/collect_heap_information.sh index 1d25c5a791c..a1b6bee078e 100755 --- a/barretenberg/cpp/scripts/collect_heap_information.sh +++ b/barretenberg/cpp/scripts/collect_heap_information.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eu PRESET=gperftools diff --git a/barretenberg/cpp/scripts/collect_profile_information.sh b/barretenberg/cpp/scripts/collect_profile_information.sh index df932c086bc..ebc0249392e 100755 --- a/barretenberg/cpp/scripts/collect_profile_information.sh +++ b/barretenberg/cpp/scripts/collect_profile_information.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eu # can also be 'xray-1thread' diff --git a/barretenberg/cpp/scripts/install-wasi-sdk.sh b/barretenberg/cpp/scripts/install-wasi-sdk.sh index 1da258d880b..9d27d012a61 100755 --- a/barretenberg/cpp/scripts/install-wasi-sdk.sh +++ b/barretenberg/cpp/scripts/install-wasi-sdk.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eu if [[ -d ./src/wasi-sdk-20.0 && -d ./src/wasi-sdk-20.0+threads ]]; then diff --git a/barretenberg/cpp/scripts/run_tests b/barretenberg/cpp/scripts/run_tests index 28d9f8cc4f8..a67f03ee992 100755 --- a/barretenberg/cpp/scripts/run_tests +++ b/barretenberg/cpp/scripts/run_tests @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # This is the default test runner which takes the as arguments: # 1. The number of ignition transcripts to download. # 2. The set of gtest binary names to run. diff --git a/barretenberg/cpp/src/barretenberg/benchmark/compare_branch_vs_baseline.sh b/barretenberg/cpp/src/barretenberg/benchmark/compare_branch_vs_baseline.sh index 0ac6dce1157..34ee2ce171d 100755 --- a/barretenberg/cpp/src/barretenberg/benchmark/compare_branch_vs_baseline.sh +++ b/barretenberg/cpp/src/barretenberg/benchmark/compare_branch_vs_baseline.sh @@ -1,7 +1,7 @@ -#!/bin/bash +#!/usr/bin/env bash # This script is used to compare a suite of benchmarks between baseline (default: master) and -# the branch from which the script is run. Simply check out the branch of interest, ensure +# the branch from which the script is run. Simply check out the branch of interest, ensure # it is up to date with local master, and run the script. # Specify the benchmark suite and the "baseline" branch against which to compare @@ -43,7 +43,7 @@ BASELINE_RESULTS="$BENCH_RESULTS_DIR/results_baseline.json" echo -e "\nRunning $BENCH_TARGET in master.." bin/$BENCH_TARGET --benchmark_format=json > $BASELINE_RESULTS -# Call compare.py on the results (json) to get high level statistics. +# Call compare.py on the results (json) to get high level statistics. # See docs at https://github.com/google/benchmark/blob/main/docs/tools.md for more details. $BENCH_TOOLS_DIR/compare.py benchmarks $BASELINE_RESULTS $BRANCH_RESULTS diff --git a/barretenberg/cpp/src/barretenberg/benchmark/honk_bench/compare_honk_to_plonk_ultra.sh b/barretenberg/cpp/src/barretenberg/benchmark/honk_bench/compare_honk_to_plonk_ultra.sh index 1863327ae4e..f97dbc0eaa7 100755 --- a/barretenberg/cpp/src/barretenberg/benchmark/honk_bench/compare_honk_to_plonk_ultra.sh +++ b/barretenberg/cpp/src/barretenberg/benchmark/honk_bench/compare_honk_to_plonk_ultra.sh @@ -1,7 +1,7 @@ -#!/bin/bash +#!/usr/bin/env bash # This script is used to compare the results of honk_bench between baseline (master) and -# the branch from which the script is run. Simply check out the branch of interest, ensure +# the branch from which the script is run. Simply check out the branch of interest, ensure # it is up to date with local master, and run the script. echo -e '\nComparing Ultra Plonk/Honk benchmarks.' @@ -19,21 +19,21 @@ pip3 install --user -r $BUILD_DIR/_deps/benchmark-src/requirements.txt cd $BASE_DIR mkdir $BENCH_RESULTS_DIR -# +# echo -e '\nBuilding and running Standard benchmarks..' # rm -rf $BUILD_DIR -cmake --preset bench > /dev/null && cmake --build --preset bench --target ultra_plonk_bench +cmake --preset bench > /dev/null && cmake --build --preset bench --target ultra_plonk_bench cd build-bench PLONK_BENCH_RESULTS="$BENCH_RESULTS_DIR/plonk_bench.json" ./bin/ultra_plonk_bench --benchmark_format=json > $PLONK_BENCH_RESULTS cd .. -cmake --preset bench > /dev/null && cmake --build --preset bench --target ultra_honk_bench +cmake --preset bench > /dev/null && cmake --build --preset bench --target ultra_honk_bench cd build-bench HONK_BENCH_RESULTS="$BENCH_RESULTS_DIR/honk_bench.json" ./bin/ultra_honk_bench --benchmark_format=json > $HONK_BENCH_RESULTS -# Call compare.py on the results (json) to get high level statistics. +# Call compare.py on the results (json) to get high level statistics. # See docs at https://github.com/google/benchmark/blob/main/docs/tools.md for more details. $BENCH_TOOLS_DIR/compare.py benchmarks $PLONK_BENCH_RESULTS $HONK_BENCH_RESULTS diff --git a/barretenberg/scripts/bindgen.sh b/barretenberg/scripts/bindgen.sh index 1a2034a213a..e3080a84824 100755 --- a/barretenberg/scripts/bindgen.sh +++ b/barretenberg/scripts/bindgen.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eu #find ./cpp/src -type f -name "c_bind*.hpp" | ./scripts/decls_json.py > exports.json diff --git a/barretenberg/sol/scripts/init.sh b/barretenberg/sol/scripts/init.sh index 147adc9ae21..70868e37808 100755 --- a/barretenberg/sol/scripts/init.sh +++ b/barretenberg/sol/scripts/init.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash PLONK_FLAVOUR="ultra" diff --git a/barretenberg/sol/scripts/run_fuzzer.sh b/barretenberg/sol/scripts/run_fuzzer.sh index e6e235a85e5..2c76ad1de43 100755 --- a/barretenberg/sol/scripts/run_fuzzer.sh +++ b/barretenberg/sol/scripts/run_fuzzer.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash PLONK_FLAVOUR=${1:-"ultra"} CIRCUIT_FLAVOUR=${2:-"blake"} diff --git a/barretenberg/ts/bootstrap.sh b/barretenberg/ts/bootstrap.sh index 8b2d5b76c66..2f0fa19bb81 100755 --- a/barretenberg/ts/bootstrap.sh +++ b/barretenberg/ts/bootstrap.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eu cd "$(dirname "$0")" diff --git a/barretenberg/ts/scripts/run_tests b/barretenberg/ts/scripts/run_tests index ed93c9a8091..0f0d2895cec 100755 --- a/barretenberg/ts/scripts/run_tests +++ b/barretenberg/ts/scripts/run_tests @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -xe $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null diff --git a/bootstrap.sh b/bootstrap.sh index 5a953eee6b5..ac402f07baa 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Usage: # Bootstraps the repo. End to end tests should be runnable after a bootstrap: # ./bootstrap.sh diff --git a/bootstrap/bootstrap_test.sh b/bootstrap/bootstrap_test.sh index 805f0d0ef0e..8e05b2650ef 100755 --- a/bootstrap/bootstrap_test.sh +++ b/bootstrap/bootstrap_test.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # This script takes the state of your current repository, and clones it inside of a docker container. # You likely don't have a fresh clone, and it's paramount that to test bootstrapping, we don't have any # intermediate build state in the context. diff --git a/bootstrap_docker.sh b/bootstrap_docker.sh index 6fc48b2b118..6d086d1d331 100755 --- a/bootstrap_docker.sh +++ b/bootstrap_docker.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # This script builds the projects listed in build_manifest.sh, terminating when it reaches PROJECT_NAME. # If run from within a project, it will build only that project, unless env var ONLY_TARGET=false. # diff --git a/build-system/remote/bootstrap_build_instance.sh b/build-system/remote/bootstrap_build_instance.sh index 535bcdd7ce7..c911eb113b3 100644 --- a/build-system/remote/bootstrap_build_instance.sh +++ b/build-system/remote/bootstrap_build_instance.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null sudo apt update diff --git a/build-system/scripts/add_timestamps b/build-system/scripts/add_timestamps index 8b658925ba5..230d51fb626 100755 --- a/build-system/scripts/add_timestamps +++ b/build-system/scripts/add_timestamps @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash while IFS= read -r line; do printf '%(%Y-%m-%d %H:%M:%S)T %s\n' -1 "$line" done diff --git a/build-system/scripts/build b/build-system/scripts/build index 6906a275149..c82e6bde0a6 100755 --- a/build-system/scripts/build +++ b/build-system/scripts/build @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # Builds a docker image and pushes it to it's repository. Leverages caches where possible. # Cached images include previous successfully built images (including multi-stages) built on this branch. diff --git a/build-system/scripts/build_local b/build-system/scripts/build_local index cdb9f3a60f3..0bc99f17d4a 100755 --- a/build-system/scripts/build_local +++ b/build-system/scripts/build_local @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Builds the PROJECTS in the given order. # Will terminate build at TARGET_PROJECT (if given). # Will only build TARGET_PROJECT if ONLY_TARGET given. diff --git a/build-system/scripts/calculate_content_hash b/build-system/scripts/calculate_content_hash index 28ae8ff2e62..7b7b68ade71 100755 --- a/build-system/scripts/calculate_content_hash +++ b/build-system/scripts/calculate_content_hash @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -euo pipefail diff --git a/build-system/scripts/calculate_image_tag b/build-system/scripts/calculate_image_tag index c273648287c..f2a3cea871f 100755 --- a/build-system/scripts/calculate_image_tag +++ b/build-system/scripts/calculate_image_tag @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Return a repositories build cache image tag based on content hash. # If the second argument is set: # It's used to suffix the tag with the given unique arch descriptor. diff --git a/build-system/scripts/calculate_image_uri b/build-system/scripts/calculate_image_uri index 8efd7ab4cdc..c107647434c 100755 --- a/build-system/scripts/calculate_image_uri +++ b/build-system/scripts/calculate_image_uri @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -eu diff --git a/build-system/scripts/calculate_rebuild_files b/build-system/scripts/calculate_rebuild_files index 9be1afd4912..49e07634955 100755 --- a/build-system/scripts/calculate_rebuild_files +++ b/build-system/scripts/calculate_rebuild_files @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -eu diff --git a/build-system/scripts/check_rebuild b/build-system/scripts/check_rebuild index 76f99c6540e..df2291065e4 100755 --- a/build-system/scripts/check_rebuild +++ b/build-system/scripts/check_rebuild @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # If this script fails (nonzero exit), then the caller should rebuild. [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -euo pipefail diff --git a/build-system/scripts/clean_image_tags b/build-system/scripts/clean_image_tags index 832fdf4cda6..9c4557e68c6 100755 --- a/build-system/scripts/clean_image_tags +++ b/build-system/scripts/clean_image_tags @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -e @@ -20,7 +20,7 @@ fi # Collect all the commits ids in the repository and remove the remote for faster lookups # See warning in https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/ -git config fetch.recurseSubmodules false +git config fetch.recurseSubmodules false git fetch --filter=tree:0 origin ORIGIN_URL=$(git remote get-url origin) git remote remove origin @@ -29,7 +29,7 @@ git remote remove origin # This happens for all commits tagged for PRs that then get squashed and merged IFS=$'\n' for TAG in $IMAGE_TAGS; do - if [[ $TAG =~ ^cache-[0-9a-fA-F]+-builder$ ]]; then + if [[ $TAG =~ ^cache-[0-9a-fA-F]+-builder$ ]]; then TAG_COMMIT=$(echo "$TAG" | cut -d '-' -f 2) if git cat-file -e $TAG_COMMIT; then echo "Commit for $TAG found" diff --git a/build-system/scripts/cond_run_compose b/build-system/scripts/cond_run_compose index ee412325a6a..d1d14c6baca 100755 --- a/build-system/scripts/cond_run_compose +++ b/build-system/scripts/cond_run_compose @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -eu diff --git a/build-system/scripts/cond_run_container b/build-system/scripts/cond_run_container index ca8d67fc0bd..12badea45af 100755 --- a/build-system/scripts/cond_run_container +++ b/build-system/scripts/cond_run_container @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Conditionally runs a script if the REPOSITORY content hash has changed and we haven't had a successful run. # # Arguments are: diff --git a/build-system/scripts/cond_run_script b/build-system/scripts/cond_run_script index c1ada5f8cf2..1eb6b3695d9 100755 --- a/build-system/scripts/cond_run_script +++ b/build-system/scripts/cond_run_script @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Conditionally runs a script if the REPOSITORY content hash has changed and we haven't had a successful run. # # Arguments are: diff --git a/build-system/scripts/cond_spot_run_build b/build-system/scripts/cond_spot_run_build index 3333e5dec7c..a2e2663de89 100755 --- a/build-system/scripts/cond_spot_run_build +++ b/build-system/scripts/cond_spot_run_build @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -euo pipefail diff --git a/build-system/scripts/cond_spot_run_compose b/build-system/scripts/cond_spot_run_compose index 51a4529471a..56da3e3da4e 100755 --- a/build-system/scripts/cond_spot_run_compose +++ b/build-system/scripts/cond_spot_run_compose @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -euo pipefail diff --git a/build-system/scripts/cond_spot_run_container b/build-system/scripts/cond_spot_run_container index 48876a65f3b..5b2ebc71f32 100755 --- a/build-system/scripts/cond_spot_run_container +++ b/build-system/scripts/cond_spot_run_container @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -eu diff --git a/build-system/scripts/cond_spot_run_script b/build-system/scripts/cond_spot_run_script index 4b1832127ac..23a678fcd32 100755 --- a/build-system/scripts/cond_spot_run_script +++ b/build-system/scripts/cond_spot_run_script @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Conditionally runs a script on a remote spot instance if the REPOSITORY content hash has changed and we haven't had a # successful run. # The TAG_POSTFIX is used by cond_spot_run_test whereby we use an image tag postfixed with JOB_NAME to identifify if @@ -19,7 +19,12 @@ CPUS=$2 ARCH=$3 shift 3 -BASE_TAG=$(calculate_image_tag $REPOSITORY) +MULTIARCH=$(query_manifest multiarch $REPOSITORY) +if [ "$MULTIARCH" == "host" ]; then + BASE_TAG=$(calculate_image_tag $REPOSITORY $ARCH) +else + BASE_TAG=$(calculate_image_tag $REPOSITORY) +fi SUCCESS_TAG=$BASE_TAG if [ -n "${TAG_POSTFIX:-}" ]; then diff --git a/build-system/scripts/cond_spot_run_test b/build-system/scripts/cond_spot_run_test index e8b8f71358d..cc48ffc742e 100755 --- a/build-system/scripts/cond_spot_run_test +++ b/build-system/scripts/cond_spot_run_test @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -euo pipefail diff --git a/build-system/scripts/create_ecr_manifest b/build-system/scripts/create_ecr_manifest index 5bda420039d..fcc96d7651e 100755 --- a/build-system/scripts/create_ecr_manifest +++ b/build-system/scripts/create_ecr_manifest @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # This script: # 1. Logs into ECR and ensures we have the given repository # 2. Computes the image uri of the cached images for the given repository given the list of architectures diff --git a/build-system/scripts/deploy b/build-system/scripts/deploy index ca43d5b9cfb..a158175c414 100755 --- a/build-system/scripts/deploy +++ b/build-system/scripts/deploy @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -eu diff --git a/build-system/scripts/deploy_dockerhub b/build-system/scripts/deploy_dockerhub index 606e0952b5d..b8d925b0711 100755 --- a/build-system/scripts/deploy_dockerhub +++ b/build-system/scripts/deploy_dockerhub @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -eu diff --git a/build-system/scripts/deploy_ecr b/build-system/scripts/deploy_ecr index 3be534694c7..c156f99e71e 100755 --- a/build-system/scripts/deploy_ecr +++ b/build-system/scripts/deploy_ecr @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -eu diff --git a/build-system/scripts/deploy_global b/build-system/scripts/deploy_global index 28b70a28619..2267474706f 100755 --- a/build-system/scripts/deploy_global +++ b/build-system/scripts/deploy_global @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Deployment script for global service (e.g. company website and metrics). [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -eu diff --git a/build-system/scripts/deploy_npm b/build-system/scripts/deploy_npm index a35d68c4650..16df5a156d0 100755 --- a/build-system/scripts/deploy_npm +++ b/build-system/scripts/deploy_npm @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -eu diff --git a/build-system/scripts/deploy_s3 b/build-system/scripts/deploy_s3 index caaa4d00bd9..87acbf004d9 100755 --- a/build-system/scripts/deploy_s3 +++ b/build-system/scripts/deploy_s3 @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -eu diff --git a/build-system/scripts/deploy_service b/build-system/scripts/deploy_service index b014ca0d823..83c45128141 100755 --- a/build-system/scripts/deploy_service +++ b/build-system/scripts/deploy_service @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -eu diff --git a/build-system/scripts/deploy_terraform b/build-system/scripts/deploy_terraform index ce2231853aa..b81c4d3f4b2 100755 --- a/build-system/scripts/deploy_terraform +++ b/build-system/scripts/deploy_terraform @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -eu diff --git a/build-system/scripts/dockerhub_login b/build-system/scripts/dockerhub_login index dae523fb43b..50a2068377b 100755 --- a/build-system/scripts/dockerhub_login +++ b/build-system/scripts/dockerhub_login @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eu # Retries up to 3 times with 10 second intervals for i in $(seq 1 3); do diff --git a/build-system/scripts/ecr_login b/build-system/scripts/ecr_login index 54a8e6d36da..331b49d4024 100755 --- a/build-system/scripts/ecr_login +++ b/build-system/scripts/ecr_login @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eu REGION=${1:-$ECR_REGION} # Retries up to 3 times with 10 second intervals diff --git a/build-system/scripts/ensure_repo b/build-system/scripts/ensure_repo index b767976354f..b71c0fabb2f 100755 --- a/build-system/scripts/ensure_repo +++ b/build-system/scripts/ensure_repo @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Logs the shell into the ECR instance at the given region, establishes if the given repository exists, creates it if it # doesn't, and re-applies thie lifecycle policy (determines when images should be automatically deleted) if it does. [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace diff --git a/build-system/scripts/ensure_terraform b/build-system/scripts/ensure_terraform index d7444aa4e4c..ce53dba1b31 100755 --- a/build-system/scripts/ensure_terraform +++ b/build-system/scripts/ensure_terraform @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Downloads and installs `terraform` if it's not installed. [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -eu diff --git a/build-system/scripts/extract_repo b/build-system/scripts/extract_repo index c2579b5a46f..6bdae1027f9 100755 --- a/build-system/scripts/extract_repo +++ b/build-system/scripts/extract_repo @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Given a repository, extracts the builds entire /usr/src dir to the given path. [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -eu diff --git a/build-system/scripts/extract_tag_version b/build-system/scripts/extract_tag_version index d142458104d..c0f7c3fd165 100755 --- a/build-system/scripts/extract_tag_version +++ b/build-system/scripts/extract_tag_version @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # This script takes a repository name as variable, # then checks if the commit tag variable (if any) # is a valid semver & echoes that valid semver. @@ -28,7 +28,7 @@ if [[ "$COMMIT_TAG" == *"/"* ]]; then COMMIT_TAG_VERSION="${COMMIT_TAG#*/}" echo "Tag was made for: $REPO_NAME" >&2 echo "Version: $COMMIT_TAG_VERSION" >&2 - + # Check if REPO_NAME is equal to REPOSITORY if [[ "$REPO_NAME" != "$REPOSITORY" ]]; then echo "REPO_NAME ($REPO_NAME) does not match REPOSITORY ($REPOSITORY). Exiting..." >&2 diff --git a/build-system/scripts/image_exists b/build-system/scripts/image_exists index 0a4bee54d7a..0ad9d90ae69 100755 --- a/build-system/scripts/image_exists +++ b/build-system/scripts/image_exists @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eu # Returns true if the given image exists in the current ECR. aws ecr describe-images --region=$ECR_REGION --repository-name=$1 --image-ids=imageTag=$2 > /dev/null 2>&1 diff --git a/build-system/scripts/init_submodules b/build-system/scripts/init_submodules index a4a9cd62db6..0b89b089866 100755 --- a/build-system/scripts/init_submodules +++ b/build-system/scripts/init_submodules @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # For a given repository, init any required submodules. [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -euo pipefail diff --git a/build-system/scripts/query_manifest b/build-system/scripts/query_manifest index 1fee0ee5759..a6e84650331 100755 --- a/build-system/scripts/query_manifest +++ b/build-system/scripts/query_manifest @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -eu CMD=$1 diff --git a/build-system/scripts/remote_run_script b/build-system/scripts/remote_run_script index e9ef84d723d..8e1d8adf38c 100755 --- a/build-system/scripts/remote_run_script +++ b/build-system/scripts/remote_run_script @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Copies the runner script to the remote instance, runs it giving it script and args to run. # The runner script checks out the repository first and runs setup-env. # diff --git a/build-system/scripts/remote_runner b/build-system/scripts/remote_runner index ce1a567a198..6283050283c 100755 --- a/build-system/scripts/remote_runner +++ b/build-system/scripts/remote_runner @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -eu diff --git a/build-system/scripts/request_spot b/build-system/scripts/request_spot index 35c4d650bac..3d669fed1df 100755 --- a/build-system/scripts/request_spot +++ b/build-system/scripts/request_spot @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -eu diff --git a/build-system/scripts/setup_env b/build-system/scripts/setup_env index 8e4058a1c53..3486e5f8653 100755 --- a/build-system/scripts/setup_env +++ b/build-system/scripts/setup_env @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # This script sets up the global build environment. This should be called before any other build scripts, # as the other build scripts assume these global variables are set. The global variables are written to # the file in $BASH_ENV, which means that any new bash shells launched within the lifetime of the machine diff --git a/build-system/scripts/should_deploy b/build-system/scripts/should_deploy index 53f295b191f..283191f51f3 100755 --- a/build-system/scripts/should_deploy +++ b/build-system/scripts/should_deploy @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Returns success if we are expected to do a deployment. # Right now, that's only if we're master. set -eu diff --git a/build-system/scripts/should_release b/build-system/scripts/should_release index 5473fc6da76..701b8a375db 100755 --- a/build-system/scripts/should_release +++ b/build-system/scripts/should_release @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Returns true if we are expected to proceed with a release job. # Specifically if we have a commit tag, are master, or are being forced to release. # This script should be used at the start of all release steps to early out PR runs. diff --git a/build-system/scripts/spot_run_script b/build-system/scripts/spot_run_script index 7a2377973d5..d5e77e35712 100755 --- a/build-system/scripts/spot_run_script +++ b/build-system/scripts/spot_run_script @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Runs a test script on a remote spot instance. Arguments are: # 1. NAME: Used to identify spot jobs. # 2. CPUS: Number of cpus on spot request. diff --git a/build-system/scripts/tag_remote_image b/build-system/scripts/tag_remote_image index bc7675b237c..0bb9e2a390f 100755 --- a/build-system/scripts/tag_remote_image +++ b/build-system/scripts/tag_remote_image @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -eu @@ -34,7 +34,7 @@ if [ "$EXISTING_TAG_MANIFEST" != "$NEW_TAG_MANIFEST" ]; then --image-tag $NEW_TAG \ --image-manifest "$EXISTING_TAG_MANIFEST" 2>&1) TAG_EXIT_CODE=$? - + # If we failed to tag due to too many tags on this image, then clean some of them up and try again if [ $TAG_EXIT_CODE -ne 0 ] && $(echo $TAG_RESULT | grep -q LimitExceededException); then echo "Failed to tag due to limit exceeded. Starting tag cleanup." diff --git a/build-system/scripts/untag_remote_image b/build-system/scripts/untag_remote_image index f79fee729bc..aa913c1200b 100755 --- a/build-system/scripts/untag_remote_image +++ b/build-system/scripts/untag_remote_image @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash REPOSITORY=$1 TAG=$2 aws ecr batch-delete-image --region=$ECR_REGION --repository-name $REPOSITORY --image-ids imageTag=$2 > /dev/null \ No newline at end of file diff --git a/build-system/scripts/upload_logs_to_s3 b/build-system/scripts/upload_logs_to_s3 index ede934edae5..25c7fe2e50a 100755 --- a/build-system/scripts/upload_logs_to_s3 +++ b/build-system/scripts/upload_logs_to_s3 @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Uploads to S3 the contents of the log file mounted on the end-to-end container, # which contains log entries with an associated event and metrics for it. diff --git a/build-system/start_interactive b/build-system/start_interactive index 2ed1f2df13a..64e24499782 100755 --- a/build-system/start_interactive +++ b/build-system/start_interactive @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Starts an interactive shell with the build system initialised. # Good for playing around with build system on development machines. diff --git a/build_manifest.yml b/build_manifest.yml index 588c3c70a93..43bce1a4071 100644 --- a/build_manifest.yml +++ b/build_manifest.yml @@ -216,4 +216,4 @@ docs: yellow-paper: buildDir: yellow-paper rebuildPatterns: - - ^yellow-paper/ \ No newline at end of file + - ^yellow-paper/ diff --git a/circuits/cpp/bootstrap.sh b/circuits/cpp/bootstrap.sh index 6a64d541b7b..440bf1e44c6 100755 --- a/circuits/cpp/bootstrap.sh +++ b/circuits/cpp/bootstrap.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eu cd "$(dirname "$0")" diff --git a/circuits/cpp/format.sh b/circuits/cpp/format.sh index e12e84ab27a..54047bd6def 100755 --- a/circuits/cpp/format.sh +++ b/circuits/cpp/format.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eu if [ "${1:-}" == "staged" ]; then diff --git a/circuits/cpp/scripts/build_run_tests_docker_local b/circuits/cpp/scripts/build_run_tests_docker_local index 19eb1784f71..f861fecbfa8 100755 --- a/circuits/cpp/scripts/build_run_tests_docker_local +++ b/circuits/cpp/scripts/build_run_tests_docker_local @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -eu diff --git a/circuits/cpp/scripts/collect_coverage_information.sh b/circuits/cpp/scripts/collect_coverage_information.sh index cac9393a9b6..45ad2764a22 100755 --- a/circuits/cpp/scripts/collect_coverage_information.sh +++ b/circuits/cpp/scripts/collect_coverage_information.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Check that the correct number of args have been provided if [ $# -ne 2 ]; then @@ -67,7 +67,7 @@ if [ ${#non_empty_profiles[@]} -gt 1 ]; then additional_objects+="-object $WORKING_DIRECTORY/bin/${non_empty_profile_base}_tests " done object_string=${additional_objects#"-object"} - + # Output the coverage report into `all_tests_coverage_report` folder rm -rf "$WORKING_DIRECTORY/all_tests_coverage_report" mkdir "$WORKING_DIRECTORY/all_tests_coverage_report" diff --git a/circuits/cpp/scripts/run_tests b/circuits/cpp/scripts/run_tests index dd7d7e21388..62fbd90153b 100755 --- a/circuits/cpp/scripts/run_tests +++ b/circuits/cpp/scripts/run_tests @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -eu diff --git a/circuits/cpp/scripts/run_tests_local b/circuits/cpp/scripts/run_tests_local index d65ff5f358f..4c12807414e 100755 --- a/circuits/cpp/scripts/run_tests_local +++ b/circuits/cpp/scripts/run_tests_local @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -eu diff --git a/circuits/cpp/scripts/tidy.sh b/circuits/cpp/scripts/tidy.sh index 3283fb89b62..7faab2658c6 100755 --- a/circuits/cpp/scripts/tidy.sh +++ b/circuits/cpp/scripts/tidy.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eu # Run clang-tidy on all C++ source files diff --git a/docs/deploy_netlify.sh b/docs/deploy_netlify.sh index 82b447119f6..24c7c1f3845 100755 --- a/docs/deploy_netlify.sh +++ b/docs/deploy_netlify.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -eu diff --git a/docs/scripts/build.sh b/docs/scripts/build.sh index e06ae5f2aa9..eb63a05887d 100755 --- a/docs/scripts/build.sh +++ b/docs/scripts/build.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eo pipefail # Helper function for building packages in yarn project @@ -10,7 +10,7 @@ build_package() { (cd "yarn-project/$package_name" && $build_command) } -# Build script. If run on Netlify, first it needs to compile all yarn-projects +# Build script. If run on Netlify, first it needs to compile all yarn-projects # that are involved in typedoc in order to generate their type information. if [ -n "$NETLIFY" ]; then # Move to project root diff --git a/iac/mainnet-fork/scripts/run_nginx_anvil.sh b/iac/mainnet-fork/scripts/run_nginx_anvil.sh index 4b701f13256..38788424ed7 100755 --- a/iac/mainnet-fork/scripts/run_nginx_anvil.sh +++ b/iac/mainnet-fork/scripts/run_nginx_anvil.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eum pipefail diff --git a/iac/mainnet-fork/scripts/wait_for_fork b/iac/mainnet-fork/scripts/wait_for_fork index 326582c25af..831e06723e2 100755 --- a/iac/mainnet-fork/scripts/wait_for_fork +++ b/iac/mainnet-fork/scripts/wait_for_fork @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -e # When destroying and applying mainnet fork terraform, it may not be diff --git a/l1-contracts/bootstrap.sh b/l1-contracts/bootstrap.sh index db2a71b8c75..f776b6072dc 100755 --- a/l1-contracts/bootstrap.sh +++ b/l1-contracts/bootstrap.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eu cd "$(dirname "$0")" diff --git a/l1-contracts/scripts/ci_deploy_contracts.sh b/l1-contracts/scripts/ci_deploy_contracts.sh index 128a1702f31..aaedb6f2f7d 100755 --- a/l1-contracts/scripts/ci_deploy_contracts.sh +++ b/l1-contracts/scripts/ci_deploy_contracts.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash export ETHEREUM_HOST=https://$DEPLOY_TAG-mainnet-fork.aztec.network:8545/$FORK_API_KEY diff --git a/l1-contracts/scripts/deploy_contracts.sh b/l1-contracts/scripts/deploy_contracts.sh index 6223a0719a9..0bd743e1567 100755 --- a/l1-contracts/scripts/deploy_contracts.sh +++ b/l1-contracts/scripts/deploy_contracts.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eu # Sets up defaults then runs the E2E Setup script to perform contract deployments diff --git a/noir/.github/actions/docs/build-status/script.sh b/noir/.github/actions/docs/build-status/script.sh index 0b282557cf2..2e86de6c173 100755 --- a/noir/.github/actions/docs/build-status/script.sh +++ b/noir/.github/actions/docs/build-status/script.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash BRANCH_NAME=$(echo "$BRANCH_NAME" | sed -e "s#refs/[^/]*/##") DEPLOY_STATUS=$(curl -X GET "https://api.netlify.com/api/v1/sites/$SITE_ID/deploys?branch=$BRANCH_NAME" | jq -r '.[] | select(.created_at != null) | .state' | head -1) diff --git a/noir/Dockerfile b/noir/Dockerfile index ac818cb8bd2..de63983db3e 100644 --- a/noir/Dockerfile +++ b/noir/Dockerfile @@ -1,15 +1,10 @@ -FROM rust:alpine3.17 -RUN apk update \ - && apk upgrade \ - && apk add --no-cache \ - build-base \ - bash +FROM rust:bookworm WORKDIR /usr/src/noir COPY . . RUN ./scripts/bootstrap_native.sh -# When running the container, mount the current working directory to /project. -FROM alpine:3.17 +# When running the container, mount the users home directory to /root +FROM ubuntu:lunar COPY --from=0 /usr/src/noir/target/release/nargo /usr/src/noir/target/release/nargo -WORKDIR /project +WORKDIR /root ENTRYPOINT ["/usr/src/noir/target/release/nargo"] \ No newline at end of file diff --git a/noir/bootstrap.sh b/noir/bootstrap.sh index bf672ac0ad2..5ebe7ade090 100755 --- a/noir/bootstrap.sh +++ b/noir/bootstrap.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eu cd $(dirname "$0") diff --git a/noir/compiler/integration-tests/scripts/codegen-verifiers.sh b/noir/compiler/integration-tests/scripts/codegen-verifiers.sh index 13667038728..4518141fc13 100644 --- a/noir/compiler/integration-tests/scripts/codegen-verifiers.sh +++ b/noir/compiler/integration-tests/scripts/codegen-verifiers.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash self_path=$(dirname "$(readlink -f "$0")") diff --git a/noir/scripts/bootstrap_native.sh b/noir/scripts/bootstrap_native.sh index 693a9d9678e..b5e004106ff 100755 --- a/noir/scripts/bootstrap_native.sh +++ b/noir/scripts/bootstrap_native.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eu cd $(dirname "$0")/.. diff --git a/noir/scripts/bootstrap_packages.sh b/noir/scripts/bootstrap_packages.sh index 5fce2675037..1363acf51a6 100755 --- a/noir/scripts/bootstrap_packages.sh +++ b/noir/scripts/bootstrap_packages.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eu cd $(dirname "$0")/.. diff --git a/noir/scripts/install_wasm-bindgen.sh b/noir/scripts/install_wasm-bindgen.sh index 5e9f9127506..c6e85bac50b 100755 --- a/noir/scripts/install_wasm-bindgen.sh +++ b/noir/scripts/install_wasm-bindgen.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eu cd $(dirname "$0")/.. diff --git a/noir/test_programs/gates_report.sh b/noir/test_programs/gates_report.sh index e06e6812e9d..4192c581376 100755 --- a/noir/test_programs/gates_report.sh +++ b/noir/test_programs/gates_report.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -e # These tests are incompatible with gas reporting diff --git a/noir/test_programs/rebuild.sh b/noir/test_programs/rebuild.sh index dfc3dc5c967..fd3e4478d62 100755 --- a/noir/test_programs/rebuild.sh +++ b/noir/test_programs/rebuild.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -e process_dir() { diff --git a/scripts/ci/assemble_e2e_benchmark.sh b/scripts/ci/assemble_e2e_benchmark.sh index 7456d4e1cbc..4a7b1bbe015 100755 --- a/scripts/ci/assemble_e2e_benchmark.sh +++ b/scripts/ci/assemble_e2e_benchmark.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Grabs the log files uploaded in yarn-project/end-to-end/scripts/upload_logs_to_s3.sh # that contain representative benchmarks, extracts whatever metrics are interesting, # and assembles a single file that shows the current state of the repository. @@ -70,7 +70,7 @@ if [ -n "${BENCHMARK_LATEST_FILE:-}" ]; then aws s3 cp $BENCHMARK_FILE_JSON "s3://${BUCKET_NAME}/${BENCHMARK_LATEST_FILE}" fi -# If on a pull request, get the data from the most recent commit on master where it's available, +# If on a pull request, get the data from the most recent commit on master where it's available, # generate a markdown comment, and post it on the pull request if [ -n "${CIRCLE_PULL_REQUEST:-}" ]; then MASTER_COMMIT_HASH=$(curl -s "https://api.github.com/repos/AztecProtocol/aztec-packages/pulls/${CIRCLE_PULL_REQUEST##*/}" | jq -r '.base.sha') @@ -88,11 +88,9 @@ if [ -n "${CIRCLE_PULL_REQUEST:-}" ]; then done set -e - if [ -z "${BASE_COMMIT_HASH:-}" ]; then + if [ -z "${BASE_COMMIT_HASH:-}" ]; then echo "No base commit data found" fi (yarn-project/scripts/run_script.sh workspace @aztec/scripts bench-comment && echo "commented on pr $CIRCLE_PULL_REQUEST") || echo "failed commenting on pr" fi - - diff --git a/scripts/ci/store_test_benchmark_logs b/scripts/ci/store_test_benchmark_logs index 3554aa746d2..c9ae07fc96a 100755 --- a/scripts/ci/store_test_benchmark_logs +++ b/scripts/ci/store_test_benchmark_logs @@ -1,5 +1,5 @@ -#!/bin/bash -# Script for storing barretenberg benchmark results. +#!/usr/bin/env bash +# Script for storing barretenberg benchmark results. # Uploads results to the AztecProtocol/benchmark-archive repository. [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace @@ -36,7 +36,7 @@ cat /tmp/csv/new.csv \ # If there actually were any logs, update the information in the benchmark repository if [ -s /tmp/csv/trimmed.csv ]; then cd /tmp - + git clone --depth 1 https://$AZTEC_GITHUB_TOKEN:@github.com/AztecProtocol/benchmark-archive cd benchmark-archive diff --git a/scripts/git-subrepo/note/init-test b/scripts/git-subrepo/note/init-test index a20854de88a..810dae14a4d 100755 --- a/scripts/git-subrepo/note/init-test +++ b/scripts/git-subrepo/note/init-test @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -ex cat $0 # Show this script in the output diff --git a/scripts/git-subrepo/note/recreate-rebase-conflict.sh b/scripts/git-subrepo/note/recreate-rebase-conflict.sh index 8e1103e575d..6dc0a51e300 100644 --- a/scripts/git-subrepo/note/recreate-rebase-conflict.sh +++ b/scripts/git-subrepo/note/recreate-rebase-conflict.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eu set -x diff --git a/scripts/git-subrepo/note/subtree-rebase-fail-example/test.bash b/scripts/git-subrepo/note/subtree-rebase-fail-example/test.bash index df8b818cee6..9118236848a 100755 --- a/scripts/git-subrepo/note/subtree-rebase-fail-example/test.bash +++ b/scripts/git-subrepo/note/subtree-rebase-fail-example/test.bash @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -ex @@ -24,6 +24,6 @@ git clone repo1 repo3 git subrepo clone ../repo2 subrepo bash git rebase -i HEAD^ - git log -p + git log -p ls ) diff --git a/scripts/git-subrepo/note/test-subrepo-push.sh b/scripts/git-subrepo/note/test-subrepo-push.sh index afceb5efa92..13b76e47fc8 100644 --- a/scripts/git-subrepo/note/test-subrepo-push.sh +++ b/scripts/git-subrepo/note/test-subrepo-push.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -ex diff --git a/scripts/git-subrepo/note/test.sh b/scripts/git-subrepo/note/test.sh index fae278795d1..d27548f0937 100755 --- a/scripts/git-subrepo/note/test.sh +++ b/scripts/git-subrepo/note/test.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -x diff --git a/scripts/git_subrepo.sh b/scripts/git_subrepo.sh index df9743a2bd5..969789cdcc8 100755 --- a/scripts/git_subrepo.sh +++ b/scripts/git_subrepo.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eu SCRIPT_DIR=$(dirname "$(realpath "$0")") @@ -11,10 +11,10 @@ fi # git subrepo is quite nice, but has one flaw in our workflow: # We frequently squash commits in PRs, and we might update the .gitrepo file -# with a parent commit that later does not exist. +# with a parent commit that later does not exist. # A backup heuristic is used to later find the squashed commit's parent -# using the .gitrepo file's git history. This might be brittle -# in the face of e.g. a .gitrepo whitespace change, but it's a fallback, +# using the .gitrepo file's git history. This might be brittle +# in the face of e.g. a .gitrepo whitespace change, but it's a fallback, # we only have this issue in master, and the file should only be edited # generally by subrepo commands. SUBREPO_PATH="${2:-}" diff --git a/scripts/migrate_barretenberg_branch.sh b/scripts/migrate_barretenberg_branch.sh index 8cf396867c3..36c128a6232 100755 --- a/scripts/migrate_barretenberg_branch.sh +++ b/scripts/migrate_barretenberg_branch.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eu # Usage: ./this.sh diff --git a/scripts/tmux-splits b/scripts/tmux-splits index 296cdb36632..6b1e28f8559 100755 --- a/scripts/tmux-splits +++ b/scripts/tmux-splits @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Launches tmux with 1 window that has 2 panes running, from top to bottom: # - aztec.js # - a shell in end-to-end. If a test is given as an argument, the test will be run here automatically. diff --git a/yarn-project/Dockerfile b/yarn-project/Dockerfile index 257d93af911..696c60fbe1c 100644 --- a/yarn-project/Dockerfile +++ b/yarn-project/Dockerfile @@ -6,8 +6,6 @@ # Any subsequent build steps needed to support downstream containers should be done in those containers build files. FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project-base as builder -RUN apk add bash perl - # Copy in the entire workspace. COPY . . diff --git a/yarn-project/Dockerfile.prod b/yarn-project/Dockerfile.prod index 95e4479137b..16f65db4740 100644 --- a/yarn-project/Dockerfile.prod +++ b/yarn-project/Dockerfile.prod @@ -3,8 +3,8 @@ FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project AS yarn-project # Need new arch specific image. -FROM node:18.19.0-alpine AS builder -RUN apk add bash jq --no-cache +FROM node:18.19.0 AS builder +RUN apt update && apt install -y jq && rm -rf /var/lib/apt/lists/* && apt-get clean COPY --from=yarn-project /usr/src /usr/src WORKDIR /usr/src/yarn-project ARG COMMIT_TAG="" @@ -18,5 +18,32 @@ RUN yarn workspaces focus @aztec/cli @aztec/aztec-sandbox @aztec/aztec-faucet -- RUN rm -rf /usr/src/noir/target # Create fresh minimal size image. -FROM node:18.19.0-alpine -COPY --from=builder /usr/src /usr/src \ No newline at end of file +# Installs our specific version of node, stripping out the unnecessary. +# We could probably just apt install nodejs, but it's both a different version, and seemingly a bit slower. +# We could also use distroless, to get us about 20mb off, but meh. It's actually useful to shell into containers. +#FROM gcr.io/distroless/nodejs18-debian12 +FROM ubuntu:lunar +# RUN apt update && apt install -y nodejs && rm -rf /var/lib/apt/lists/* && apt-get clean +RUN apt update && apt install -y curl && rm -rf /var/lib/apt/lists/* && apt-get clean +ENV NODE_VERSION=18.19.0 +RUN ARCH= && \ + dpkgArch="$(dpkg --print-architecture)" && \ + case "${dpkgArch##*-}" in \ + amd64) ARCH='x64';; \ + arm64) ARCH='arm64';; \ + *) echo "unsupported architecture"; exit 1 ;; \ + esac && \ + curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-$ARCH.tar.gz" && \ + tar zxf "node-v$NODE_VERSION-linux-$ARCH.tar.gz" -C /usr --strip-components=1 --no-same-owner \ + --exclude "*/share/*" \ + --exclude "*/bin/corepack" \ + --exclude "*/bin/npx" \ + --exclude "*/bin/npm" \ + --exclude "*/corepack/*" \ + --exclude "*/npm/man/*" \ + --exclude "*/npm/docs/*" \ + --exclude "*/include/*" && \ + rm "node-v$NODE_VERSION-linux-$ARCH.tar.gz" && \ + node --version +COPY --from=builder /usr/src /usr/src +ENTRYPOINT ["/usr/bin/node"] \ No newline at end of file diff --git a/yarn-project/aztec-node/terraform/servicediscovery-drain.sh b/yarn-project/aztec-node/terraform/servicediscovery-drain.sh index 1fa02e92d91..b8d6c301519 100755 --- a/yarn-project/aztec-node/terraform/servicediscovery-drain.sh +++ b/yarn-project/aztec-node/terraform/servicediscovery-drain.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash [ $# -ne 1 ] && echo "Usage: $0 " && exit 1 diff --git a/yarn-project/aztec-sandbox/Dockerfile b/yarn-project/aztec-sandbox/Dockerfile index 2f9644563e0..93378a67503 100644 --- a/yarn-project/aztec-sandbox/Dockerfile +++ b/yarn-project/aztec-sandbox/Dockerfile @@ -1,8 +1,7 @@ FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project-prod AS yarn-project-prod +CMD ["--no-warnings", "/usr/src/yarn-project/aztec-sandbox/dest/bin/index.js"] +EXPOSE 8079 8080 # The version has been updated in yarn-project-prod. # Adding COMMIT_TAG here to rebuild versioned image. -ARG COMMIT_TAG="" - -ENTRYPOINT ["node", "--no-warnings", "/usr/src/yarn-project/aztec-sandbox/dest/bin/index.js"] -EXPOSE 8079 8080 +ARG COMMIT_TAG="" \ No newline at end of file diff --git a/yarn-project/bootstrap.sh b/yarn-project/bootstrap.sh index 6f9acecd7db..ff01dcb5e80 100755 --- a/yarn-project/bootstrap.sh +++ b/yarn-project/bootstrap.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eu cd "$(dirname "$0")" diff --git a/yarn-project/boxes/bootstrap.sh b/yarn-project/boxes/bootstrap.sh index 340bc0e8972..9d1b562be36 100755 --- a/yarn-project/boxes/bootstrap.sh +++ b/yarn-project/boxes/bootstrap.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # should be run from yarn-project/boxes original_path=$(pwd) @@ -7,16 +7,16 @@ original_path=$(pwd) for dir in *; do # Check if it's a directory if [ -d "${dir}" ]; then - + cd "${dir}" # Run the compile command echo "Running compile command inside ${dir}..." # Runs ts-node command from "../cli" to use latest "compile" code. this uses the yarn command to use the subpackage ts-node dep yarn compile:local - + # Change back to the original directory cd "${original_path}" - + fi done diff --git a/yarn-project/boxes/run_tests b/yarn-project/boxes/run_tests index 24a089b203c..c358a9366ee 100755 --- a/yarn-project/boxes/run_tests +++ b/yarn-project/boxes/run_tests @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # This script is used to run an e2e type test in CI (see .circleci/config.yml). # It pulls images and runs docker-compose, which has the test as the entrypoint. [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace diff --git a/yarn-project/canary/scripts/cond_run_script b/yarn-project/canary/scripts/cond_run_script index 32834cef2f1..84c03bd4574 100755 --- a/yarn-project/canary/scripts/cond_run_script +++ b/yarn-project/canary/scripts/cond_run_script @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Conditionally runs a script if any dependent code has changed between # the last successful run and the present commit. # diff --git a/yarn-project/canary/scripts/extract_packages.sh b/yarn-project/canary/scripts/extract_packages.sh index 24499e23375..78255e01b24 100755 --- a/yarn-project/canary/scripts/extract_packages.sh +++ b/yarn-project/canary/scripts/extract_packages.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash FILE=$1 diff --git a/yarn-project/canary/scripts/run_tests b/yarn-project/canary/scripts/run_tests index 21894dbedb0..6bde0b451a4 100755 --- a/yarn-project/canary/scripts/run_tests +++ b/yarn-project/canary/scripts/run_tests @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # This script is used to run an e2e test in CI (see config.yml and cond_spot_run_tests). # It sets a few environment variables used inside the docker-compose.yml, pulls images, and runs docker-compose. [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace diff --git a/yarn-project/canary/scripts/update_packages.sh b/yarn-project/canary/scripts/update_packages.sh index c8fe7740bb7..085dfd217bd 100755 --- a/yarn-project/canary/scripts/update_packages.sh +++ b/yarn-project/canary/scripts/update_packages.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eu DIST_TAG=$1 diff --git a/yarn-project/cli/Dockerfile b/yarn-project/cli/Dockerfile index dfffe9b1843..96cc8fd045d 100644 --- a/yarn-project/cli/Dockerfile +++ b/yarn-project/cli/Dockerfile @@ -1,17 +1,6 @@ FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project-prod AS yarn-project-prod - -ENTRYPOINT ["node", "--no-warnings", "/usr/src/yarn-project/cli/dest/bin/index.js"] +CMD ["--no-warnings", "/usr/src/yarn-project/cli/dest/bin/index.js"] # The version has been updated in yarn-project-prod. # Adding COMMIT_TAG here to rebuild versioned image. -ARG COMMIT_TAG="" - -# Setup cache volume. -ENV XDG_CACHE_HOME /cache -RUN mkdir /cache && chmod 777 /cache -VOLUME [ "/cache" ] - -# Run as non-root user. -RUN corepack enable -RUN addgroup -S aztec && adduser -S aztec -G aztec -USER aztec +ARG COMMIT_TAG="" \ No newline at end of file diff --git a/yarn-project/deploy_npm.sh b/yarn-project/deploy_npm.sh index 04f6ab46841..f092e6fe106 100755 --- a/yarn-project/deploy_npm.sh +++ b/yarn-project/deploy_npm.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -eu diff --git a/yarn-project/end-to-end/scripts/setup_canary.sh b/yarn-project/end-to-end/scripts/setup_canary.sh index 2d4cd7fa2ef..14d7e176652 100755 --- a/yarn-project/end-to-end/scripts/setup_canary.sh +++ b/yarn-project/end-to-end/scripts/setup_canary.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eu TARGET_PKGS_FILE=$1 diff --git a/yarn-project/l1-artifacts/scripts/generate-artifacts.sh b/yarn-project/l1-artifacts/scripts/generate-artifacts.sh index 1c4bd5e2ba9..37a07c35e17 100755 --- a/yarn-project/l1-artifacts/scripts/generate-artifacts.sh +++ b/yarn-project/l1-artifacts/scripts/generate-artifacts.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -euo pipefail; target_dir=./generated diff --git a/yarn-project/noir-compiler/src/cli/add_noir_compiler_commander_actions.ts b/yarn-project/noir-compiler/src/cli/add_noir_compiler_commander_actions.ts index f651eecd996..7dff5ecb7b1 100644 --- a/yarn-project/noir-compiler/src/cli/add_noir_compiler_commander_actions.ts +++ b/yarn-project/noir-compiler/src/cli/add_noir_compiler_commander_actions.ts @@ -26,7 +26,7 @@ export function addNoirCompilerCommanderActions(program: Command, log: LogFn = ( .option('-o, --outdir ', 'Output folder for the binary artifacts, relative to the project path', 'target') .option('-ts, --typescript ', 'Optional output folder for generating typescript wrappers', undefined) .option('-i, --interface ', 'Optional output folder for generating an Aztec.nr contract interface', undefined) - .option('-c --compiler ', 'Which compiler to use. Either nargo or wasm. Defaults to nargo', 'wasm') + .option('-c --compiler ', 'Which compiler to use. Either nargo or wasm.', 'wasm') .description('Compiles the Noir Source in the target project') .action(async (projectPath: string, options: Options) => { diff --git a/yarn-project/noir-contracts/scripts/catch.sh b/yarn-project/noir-contracts/scripts/catch.sh index 8a0a894b93f..87b485eb3f3 100644 --- a/yarn-project/noir-contracts/scripts/catch.sh +++ b/yarn-project/noir-contracts/scripts/catch.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Handler for SIGCHLD, cleanup if child exit with error, used by nargo_test.sh and compile.sh handle_sigchld() { diff --git a/yarn-project/noir-contracts/scripts/compile.sh b/yarn-project/noir-contracts/scripts/compile.sh index bc217a6156b..055485220e0 100755 --- a/yarn-project/noir-contracts/scripts/compile.sh +++ b/yarn-project/noir-contracts/scripts/compile.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -euo pipefail; diff --git a/yarn-project/noir-contracts/scripts/compile_all.sh b/yarn-project/noir-contracts/scripts/compile_all.sh index 32de820a630..7445dbbc854 100755 --- a/yarn-project/noir-contracts/scripts/compile_all.sh +++ b/yarn-project/noir-contracts/scripts/compile_all.sh @@ -1,3 +1,3 @@ -#!/bin/bash +#!/usr/bin/env bash echo "Compiling all contracts" ./scripts/compile.sh $(./scripts/get_all_contracts.sh) diff --git a/yarn-project/noir-contracts/scripts/get_all_contracts.sh b/yarn-project/noir-contracts/scripts/get_all_contracts.sh index caaf81c1a13..1311a4862a4 100755 --- a/yarn-project/noir-contracts/scripts/get_all_contracts.sh +++ b/yarn-project/noir-contracts/scripts/get_all_contracts.sh @@ -1,3 +1,3 @@ -#!/bin/bash +#!/usr/bin/env bash # Utility to get the names of all contracts echo $(ls -d src/contracts/*_contract/Nargo.toml | sed -r "s/src\\/contracts\\/(.+)_contract\\/Nargo.toml/\\1/") \ No newline at end of file diff --git a/yarn-project/noir-contracts/scripts/get_all_libraries.sh b/yarn-project/noir-contracts/scripts/get_all_libraries.sh index f1913a46caf..8fbe7bb1b19 100755 --- a/yarn-project/noir-contracts/scripts/get_all_libraries.sh +++ b/yarn-project/noir-contracts/scripts/get_all_libraries.sh @@ -1,3 +1,3 @@ -#!/bin/bash +#!/usr/bin/env bash # Utility to get the names of all noir libraries located in ../aztec-nr echo $(ls -d ../aztec-nr/*/Nargo.toml | sed -r "s/..\\/aztec-nr\\/(.+)\\/Nargo.toml/\\1/") \ No newline at end of file diff --git a/yarn-project/noir-contracts/scripts/install_noir.sh b/yarn-project/noir-contracts/scripts/install_noir.sh index 325e9a56620..b1105d6ad46 100755 --- a/yarn-project/noir-contracts/scripts/install_noir.sh +++ b/yarn-project/noir-contracts/scripts/install_noir.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Script to install noirup and the latest aztec nargo set -eu diff --git a/yarn-project/noir-contracts/scripts/install_noirup.sh b/yarn-project/noir-contracts/scripts/install_noirup.sh index 11ba9b15d31..025237333cd 100755 --- a/yarn-project/noir-contracts/scripts/install_noirup.sh +++ b/yarn-project/noir-contracts/scripts/install_noirup.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Script to install noirup and the latest nargo set -eu diff --git a/yarn-project/noir-contracts/scripts/nargo_check.sh b/yarn-project/noir-contracts/scripts/nargo_check.sh index 10d9d14c2c1..45209d4ee43 100644 --- a/yarn-project/noir-contracts/scripts/nargo_check.sh +++ b/yarn-project/noir-contracts/scripts/nargo_check.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Check nargo version matches the expected one nargo_check() { diff --git a/yarn-project/noir-contracts/scripts/nargo_test.sh b/yarn-project/noir-contracts/scripts/nargo_test.sh index 8468c19f492..0666714388d 100755 --- a/yarn-project/noir-contracts/scripts/nargo_test.sh +++ b/yarn-project/noir-contracts/scripts/nargo_test.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Tests noir contracts, if multiple are provided, then they are testing in parallel, bubbling any testing errors # diff --git a/yarn-project/noir-contracts/scripts/nargo_test_ci.sh b/yarn-project/noir-contracts/scripts/nargo_test_ci.sh index 5a4458d4da5..d835f972324 100755 --- a/yarn-project/noir-contracts/scripts/nargo_test_ci.sh +++ b/yarn-project/noir-contracts/scripts/nargo_test_ci.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Runs tests scripts for all contracts, then for all libraries. ./scripts/nargo_test.sh CONTRACT $(./scripts/get_all_contracts.sh) diff --git a/yarn-project/noir-contracts/scripts/types.sh b/yarn-project/noir-contracts/scripts/types.sh index ebbc24bdc5f..3e2410e4004 100755 --- a/yarn-project/noir-contracts/scripts/types.sh +++ b/yarn-project/noir-contracts/scripts/types.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Example: # - this script will automatically be run when running `yarn noir:build` diff --git a/yarn-project/noir-contracts/scripts/types_all.sh b/yarn-project/noir-contracts/scripts/types_all.sh index 5cbb3ce9d91..38081dc6efe 100755 --- a/yarn-project/noir-contracts/scripts/types_all.sh +++ b/yarn-project/noir-contracts/scripts/types_all.sh @@ -1,3 +1,3 @@ -#!/bin/bash +#!/usr/bin/env bash # Run the types script for all files ./scripts/types.sh $(./scripts/get_all_contracts.sh) diff --git a/yarn-project/noir-contracts/src/scripts/compile.sh b/yarn-project/noir-contracts/src/scripts/compile.sh index 551f7a74869..adedfaeba4f 100755 --- a/yarn-project/noir-contracts/src/scripts/compile.sh +++ b/yarn-project/noir-contracts/src/scripts/compile.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Example: diff --git a/yarn-project/p2p-bootstrap/terraform/servicediscovery-drain.sh b/yarn-project/p2p-bootstrap/terraform/servicediscovery-drain.sh index 1fa02e92d91..b8d6c301519 100755 --- a/yarn-project/p2p-bootstrap/terraform/servicediscovery-drain.sh +++ b/yarn-project/p2p-bootstrap/terraform/servicediscovery-drain.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash [ $# -ne 1 ] && echo "Usage: $0 " && exit 1 diff --git a/yarn-project/scripts/run_script.sh b/yarn-project/scripts/run_script.sh index a3347411e85..a06cf4eb42e 100755 --- a/yarn-project/scripts/run_script.sh +++ b/yarn-project/scripts/run_script.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Downloads the image that contains the built scripts package and executes the given command in it. [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -eu diff --git a/yarn-project/scripts/version_packages.sh b/yarn-project/scripts/version_packages.sh index a708cb7a4ed..bf631bd8ac1 100755 --- a/yarn-project/scripts/version_packages.sh +++ b/yarn-project/scripts/version_packages.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eu if [ -n "$COMMIT_TAG" ]; then diff --git a/yarn-project/yarn-project-base/Dockerfile b/yarn-project/yarn-project-base/Dockerfile index ee9c4f4f6b7..96be5e824c3 100644 --- a/yarn-project/yarn-project-base/Dockerfile +++ b/yarn-project/yarn-project-base/Dockerfile @@ -47,8 +47,8 @@ FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/bb.js as bb.js FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir as noir FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir-packages as noir-packages -FROM node:18.19.0-alpine -RUN apk update && apk add --no-cache bash jq curl +FROM node:18.19.0 +RUN apt update && apt install -y jq curl perl && rm -rf /var/lib/apt/lists/* && apt-get clean # Copy L1 contracts. COPY --from=contracts /usr/src/l1-contracts /usr/src/l1-contracts