diff --git a/.circleci/config.yml b/.circleci/config.yml index 7cdf64b5bff..5e939431ab5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -68,14 +68,42 @@ setup_env: &setup_env name: "Setup environment" command: ./build-system/scripts/setup_env "$CIRCLE_SHA1" "$CIRCLE_TAG" "$CIRCLE_JOB" "$CIRCLE_REPOSITORY_URL" "$CIRCLE_BRANCH" "$CIRCLE_PULL_REQUEST" -# This step is used to save logs from various barretenberg test to the workspace so that they can be used later to parse benchmark values out of them -save_logs: &save_logs - persist_to_workspace: - root: /tmp/test-logs - paths: - - ./* - jobs: + # Noir + noir-x86_64: + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small + steps: + - *checkout + - *setup_env + - run: + name: "Build" + command: cond_spot_run_build noir 32 + + noir-arm64: + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small + steps: + - *checkout + - *setup_env + - run: + name: "Build" + command: cond_spot_run_build noir 32 arm64 + + noir-ecr-manifest: + machine: + image: ubuntu-2204:2023.07.2 + resource_class: medium + steps: + - *checkout + - *setup_env + - run: + name: "Create ECR manifest" + command: create_ecr_manifest noir x86_64,arm64 + + # Barretenberg barretenberg-wasm-linux-clang: docker: - image: aztecprotocol/alpine-build-image @@ -152,7 +180,6 @@ jobs: - run: name: "Test" command: cond_spot_run_test barretenberg-x86_64-linux-clang-assert 32 ./scripts/run_tests 1 stdlib-tests - - *save_logs barretenberg-dsl-tests: docker: @@ -164,7 +191,6 @@ jobs: - run: name: "Test" command: cond_spot_run_test barretenberg-x86_64-linux-clang-assert 32 ./scripts/run_tests 1 dsl_tests - - *save_logs barretenberg-tests: docker: @@ -176,7 +202,6 @@ jobs: - run: name: "Test" command: cond_spot_run_test barretenberg-x86_64-linux-clang-assert 32 ./scripts/bb-tests.sh - - *save_logs barretenberg-honk-tests: docker: @@ -188,7 +213,6 @@ jobs: - run: name: "Test" command: cond_spot_run_test barretenberg-x86_64-linux-clang-assert 32 ./scripts/run_tests 1 honk_tests - - *save_logs barretenberg-proof-system-tests: docker: @@ -200,7 +224,6 @@ jobs: - run: name: "Test" command: cond_spot_run_test barretenberg-x86_64-linux-clang-assert 32 ./scripts/run_tests 1 proof_system_tests - - *save_logs barretenberg-stdlib-recursion-ultra-tests: docker: @@ -212,7 +235,6 @@ jobs: - run: name: "Test" command: cond_spot_run_test barretenberg-x86_64-linux-clang-assert 32 ./scripts/run_tests 3 stdlib_recursion_tests --gtest_filter=-*turbo* - - *save_logs barretenberg-join-split-tests: docker: @@ -224,20 +246,6 @@ jobs: - run: name: "Test" command: cond_spot_run_test barretenberg-x86_64-linux-clang-assert 32 ./scripts/run_tests 3 join_split_example_proofs_join_split_tests --gtest_filter=-*full_proof* - - *save_logs - - # barretenberg-benchmark-aggregator: - # docker: - # - image: aztecprotocol/alpine-build-image - # resource_class: small - # steps: - # - attach_workspace: - # at: /tmp/test-logs - # - *checkout - # - *setup_env - # - run: - # name: "Test" - # command: ./scripts/ci/store_test_benchmark_logs $AZTEC_GITHUB_TOKEN barretenberg-acir-tests-bb: docker: @@ -315,7 +323,6 @@ jobs: - run: name: "Test" command: cond_spot_run_test circuits-x86_64-linux-clang-assert 32 ./scripts/run_tests 1 x86_64 scripts/a3-tests -*.skip* - - *save_logs l1-contracts: machine: @@ -381,7 +388,7 @@ jobs: - *setup_env - run: name: "Build and test" - command: build aztec-sandbox false x86_64 + command: build aztec-sandbox aztec-sandbox-arm64: machine: @@ -396,7 +403,7 @@ jobs: # out to eu-west2 despite the image being locally tagged, resulting in unauthorized 401. Weird docker bug? command: | echo "export DOCKER_BUILDKIT=" > $BASH_ENV - build aztec-sandbox false arm64 + build aztec-sandbox cli-x86_64: machine: @@ -407,7 +414,7 @@ jobs: - *setup_env - run: name: "Build and test" - command: build cli false x86_64 + command: build cli cli-arm64: machine: @@ -422,7 +429,7 @@ jobs: # out to eu-west2 despite the image being locally tagged, resulting in unauthorized 401. Weird docker bug? command: | echo "export DOCKER_BUILDKIT=" > $BASH_ENV - build cli false arm64 + build cli mainnet-fork: machine: @@ -435,17 +442,6 @@ jobs: name: "Build" command: build mainnet-fork | add_timestamps - deploy-mainnet-fork: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: medium - steps: - - *checkout - - *setup_env - - run: - name: "Deploy mainnet fork" - command: deploy_ecr mainnet-fork - aztec-faucet: machine: image: ubuntu-2204:2023.07.2 @@ -503,17 +499,6 @@ jobs: name: "Test via adhoc script" command: ./yarn-project/boxes/run_tests token - canary: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large - steps: - - *checkout - - *setup_env - - run: - name: "Build" - command: build canary-build - end-to-end: machine: image: ubuntu-2204:2023.07.2 @@ -920,7 +905,7 @@ jobs: name: "Noop" command: echo Noop - e2e-end: + end: docker: - image: cimg/base:2023.09 resource_class: small @@ -929,15 +914,6 @@ jobs: name: "Noop" command: echo Noop - # canary-end: - # docker: - # - image: cimg/base:2023.09 - # resource_class: small - # steps: - # - run: - # name: "Noop" - # command: echo Noop - bench-summary: machine: image: ubuntu-2204:2023.07.2 @@ -948,18 +924,19 @@ jobs: name: "Assemble benchmark summary from uploaded logs" command: ./scripts/ci/assemble_e2e_benchmark.sh - # deploy-npm-canary: - # # Deploys next version under 'canary' dist tag. - # machine: - # image: ubuntu-2204:2023.07.2 - # resource_class: medium - # steps: - # - *checkout - # - *setup_env - # # Aztec.js and dependencies - # - run: - # name: "yarn-project" - # command: yarn-project/deploy_npm.sh canary + # Deploy jobs. + deploy-mainnet-fork: + machine: + image: ubuntu-2204:2023.07.2 + resource_class: medium + steps: + - *checkout + - *setup_env + - run: + name: "Deploy mainnet fork" + command: | + should_deploy || exit 0 + deploy_ecr mainnet-fork deploy-npm: machine: @@ -968,21 +945,11 @@ jobs: steps: - *checkout - *setup_env - # Aztec.js and dependencies - run: name: "yarn-project" - command: yarn-project/deploy_npm.sh latest - - # deploy-dockerhub-canary: - # machine: - # image: ubuntu-2204:2023.07.2 - # resource_class: medium - # steps: - # - *checkout - # - *setup_env - # - run: - # name: "Deploy to dockerhub" - # command: yarn-project/deploy_dockerhub.sh canary + command: | + should_deploy || exit 0 + yarn-project/deploy_npm.sh latest deploy-dockerhub: machine: @@ -993,71 +960,17 @@ jobs: - *setup_env - run: name: "Deploy to dockerhub" - command: yarn-project/deploy_dockerhub.sh - - # build-deployment-canary: - # machine: - # image: ubuntu-2204:2023.07.2 - # resource_class: large - # steps: - # - *checkout - # - *setup_env - # - run: - # name: "Build" - # command: build canary true - - # canary-uniswap-test: - # machine: - # image: ubuntu-2204:2023.07.2 - # resource_class: large - # steps: - # - *checkout - # - *setup_env - # - run: - # name: "Test" - # command: run_script canary ./scripts/run_tests ./src/uniswap_trade_on_l1_from_l2.test.ts canary ./scripts/docker-compose.yml - - # canary-browser-test: - # machine: - # image: ubuntu-2204:2023.07.2 - # resource_class: large - # steps: - # - *checkout - # - *setup_env - # - run: - # name: "Test" - # command: run_script canary ./scripts/run_tests ./src/aztec_js_browser.test.ts canary ./scripts/docker-compose-browser.yml - - # canary-cli-test: - # machine: - # image: ubuntu-2204:2023.07.2 - # resource_class: large - # steps: - # - *checkout - # - *setup_env - # - run: - # name: "Test" - # command: run_script canary ./scripts/run_tests ./src/cli.test.ts canary ./scripts/docker-compose.yml + command: | + should_deploy || exit 0 + deploy_dockerhub noir x86_64,arm64 + deploy_dockerhub aztec-sandbox x86_64,arm64 + deploy_dockerhub cli x86_64,arm64 # Repeatable config for defining the workflow below. -tag_regex: &tag_regex /^aztec-packages-v.*/ defaults: &defaults filters: tags: - only: *tag_regex - context: - - build - - slack - post-steps: - - slack/notify: - event: fail - branch_pattern: "master" -release_defaults: &release_defaults - filters: - tags: - only: *tag_regex - branches: - ignore: /.*/ + only: /^aztec-packages-v.*/ context: - build - slack @@ -1065,20 +978,42 @@ release_defaults: &release_defaults - slack/notify: event: fail branch_pattern: "master" + +defaults_yarn_project: &defaults_yarn_project + requires: + - yarn-project + <<: *defaults + +defaults_deploy: &defaults_deploy + requires: + - end + <<: *defaults + e2e_test: &e2e_test requires: - e2e-join <<: *defaults + bb_test: &bb_test requires: - barretenberg-x86_64-linux-clang-assert <<: *defaults +# Workflows. workflows: system: when: equal: [system, << pipeline.parameters.workflow >>] jobs: + # Noir + - noir-x86_64: *defaults + - noir-arm64: *defaults + - noir-ecr-manifest: + requires: + - noir-x86_64 + - noir-arm64 + <<: *defaults + # Barretenberg - barretenberg-x86_64-linux-gcc: *defaults - barretenberg-x86_64-linux-clang: *defaults @@ -1093,17 +1028,6 @@ workflows: - barretenberg-stdlib-tests: *bb_test - barretenberg-stdlib-recursion-ultra-tests: *bb_test - barretenberg-join-split-tests: *bb_test - # - barretenberg-benchmark-aggregator: - # requires: - # - barretenberg-tests - # - barretenberg-stdlib-tests - # - barretenberg-stdlib-recursion-ultra-tests - # - barretenberg-join-split-tests - # filters: - # branches: - # only: - # - master - # <<: *defaults - barretenberg-acir-tests-bb: requires: - barretenberg-x86_64-linux-clang-assert @@ -1138,10 +1062,6 @@ workflows: - l1-contracts: *defaults - mainnet-fork: *defaults - - deploy-mainnet-fork: - requires: - - mainnet-fork - <<: *release_defaults # Yarn Project - yarn-project-base: @@ -1154,51 +1074,15 @@ workflows: requires: - yarn-project-base <<: *defaults - - yarn-project-formatting: - requires: - - yarn-project - <<: *defaults - - yarn-project-tests: - requires: - - yarn-project - <<: *defaults - - - end-to-end: - requires: - - yarn-project - <<: *defaults - - canary: - requires: - - yarn-project - <<: *defaults - - build-docs: - requires: - - yarn-project - <<: *defaults - - - aztec-sandbox-x86_64: - requires: - - yarn-project - <<: *defaults - - aztec-sandbox-arm64: - requires: - - yarn-project - <<: *defaults - - - cli-x86_64: - requires: - - yarn-project - <<: *defaults - - cli-arm64: - requires: - - yarn-project - <<: *defaults - - - aztec-faucet: - requires: - - yarn-project - <<: *defaults - + - yarn-project-formatting: *defaults_yarn_project + - yarn-project-tests: *defaults_yarn_project + - end-to-end: *defaults_yarn_project + - build-docs: *defaults_yarn_project + - aztec-sandbox-x86_64: *defaults_yarn_project + - aztec-sandbox-arm64: *defaults_yarn_project + - cli-x86_64: *defaults_yarn_project + - cli-arm64: *defaults_yarn_project + - aztec-faucet: *defaults_yarn_project - ecr-manifest: requires: - aztec-sandbox-x86_64 @@ -1207,28 +1091,26 @@ workflows: - cli-arm64 <<: *defaults + # Boxes. - boxes-blank-react: requires: - aztec-sandbox-x86_64 <<: *defaults - - boxes-blank: requires: - aztec-sandbox-x86_64 <<: *defaults - - boxes-token: requires: - aztec-sandbox-x86_64 <<: *defaults + # End to end tests. - e2e-join: requires: - end-to-end - ecr-manifest - - canary <<: *defaults - - e2e-2-pxes: *e2e_test - e2e-deploy-contract: *e2e_test - e2e-lending-contract: *e2e_test @@ -1260,11 +1142,12 @@ workflows: - guides-dapp-testing: *e2e_test - guides-sample-dapp: *e2e_test - guides-up-quick-start: *e2e_test - - bench-publish-rollup: *e2e_test - - bench-process-history: *e2e_test - - e2e-end: + # Everything that must complete before deployment. + - end: requires: + - mainnet-fork + - noir-ecr-manifest - e2e-2-pxes - e2e-deploy-contract - e2e-lending-contract @@ -1298,64 +1181,16 @@ workflows: - guides-up-quick-start <<: *defaults + # Benchmark jobs. + - bench-publish-rollup: *e2e_test + - bench-process-history: *e2e_test - bench-summary: requires: - - e2e-end - bench-publish-rollup - bench-process-history <<: *defaults - # Deploy under canary tag - # - deploy-npm-canary: - # requires: - # - e2e-end - # <<: *release_defaults - # - deploy-dockerhub-canary: - # requires: - # - e2e-end - # <<: *release_defaults - # - deploy-ecr: - # requires: - # - e2e-end - # <<: *release_defaults - - # - build-deployment-canary: - # requires: - # - deploy-npm-canary - # - deploy-dockerhub-canary - # <<: *release_defaults - - # Run canary tests - # - canary-uniswap-test: - # requires: - # - build-deployment-canary - # <<: *release_defaults - - # - canary-browser-test: - # requires: - # - build-deployment-canary - # <<: *release_defaults - - # - canary-cli-test: - # requires: - # - build-deployment-canary - # <<: *release_defaults - - # - canary-end: - # requires: - # - canary-uniswap-test - # - canary-browser-test - # - canary-cli-test - # <<: *release_defaults - # Production deployment - - deploy-dockerhub: - requires: - # - canary-end - - e2e-end - <<: *release_defaults - - deploy-npm: - requires: - # - canary-end - - e2e-end - <<: *release_defaults + - deploy-dockerhub: *defaults_deploy + - deploy-npm: *defaults_deploy + - deploy-mainnet-fork: *defaults_deploy diff --git a/PROJECT b/PROJECT index bc836bf5f94..5f8dfee5219 100644 --- a/PROJECT +++ b/PROJECT @@ -1 +1 @@ -aztec3-packages \ No newline at end of file +aztec \ No newline at end of file diff --git a/VERSION b/VERSION deleted file mode 100644 index a0dcf85366c..00000000000 --- a/VERSION +++ /dev/null @@ -1 +0,0 @@ -v0.15.1 x-release-please-version diff --git a/barretenberg/VERSION b/barretenberg/VERSION deleted file mode 100644 index a0dcf85366c..00000000000 --- a/barretenberg/VERSION +++ /dev/null @@ -1 +0,0 @@ -v0.15.1 x-release-please-version diff --git a/build-system/remote/32core.json b/build-system/remote/32core.json deleted file mode 100644 index 79f0f396d8f..00000000000 --- a/build-system/remote/32core.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "ImageId": "ami-0e5df77ac318c7a18", - "KeyName": "build-instance", - "SecurityGroupIds": ["sg-0ccd4e5df0dcca0c9"], - "InstanceType": "r5.8xlarge", - "BlockDeviceMappings": [ - { - "DeviceName": "/dev/sda1", - "Ebs": { - "VolumeSize": 16 - } - } - ] -} diff --git a/build-system/remote/64core.json b/build-system/remote/64core.json deleted file mode 100644 index 13df8fd3539..00000000000 --- a/build-system/remote/64core.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "ImageId": "ami-0e5df77ac318c7a18", - "KeyName": "build-instance", - "SecurityGroupIds": ["sg-0ccd4e5df0dcca0c9"], - "InstanceType": "r5.16xlarge", - "BlockDeviceMappings": [ - { - "DeviceName": "/dev/sda1", - "Ebs": { - "VolumeSize": 16 - } - } - ] -} diff --git a/build-system/remote/bootstrap_build_instance.sh b/build-system/remote/bootstrap_build_instance.sh new file mode 100644 index 00000000000..535bcdd7ce7 --- /dev/null +++ b/build-system/remote/bootstrap_build_instance.sh @@ -0,0 +1,8 @@ +#!/bin/bash +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg +echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null +sudo apt update +sudo apt install -y apt-transport-https ca-certificates curl software-properties-common awscli docker-ce +sudo usermod -aG docker ${USER} +mkdir .aws +echo "Add build-instance credentials to ~/.aws/credentials \ No newline at end of file diff --git a/build-system/scripts/build b/build-system/scripts/build index 904ed2b3107..71826e74dbf 100755 --- a/build-system/scripts/build +++ b/build-system/scripts/build @@ -8,26 +8,20 @@ # usage: ./build # example: ./build aztec-connect-cpp-x86_64-linux-clang # output image: -# 278380418400.dkr.ecr.us-east-2.amazonaws.com/aztec-connect-cpp-x86_64-linux-clang:cache-deadbeefcafebabe1337c0d3 +# 278380418400.dkr.ecr.us-east-2.amazonaws.com/aztec-connect-cpp-x86_64-linux-clang:cache-deadbeefcafebabe1337c0d3-x86_64 # # In more detail: # - Init all submodules required to build this project. # - Log into cache ECR, and ensures repository exists. -# - Checks if current project needs to be rebuilt, if not, retag previous image with current commit hash and early out. # - Validate any terraform that may exist. -# - Pull down dependent images that we do not control (e.g. alpine etc). -# - For images we do control, pull the image we've built (or retagged) as part of this build. -# - For each "named stage" (usually intermittent builders before creating final image), pull previous to prime the cache, build and push the results. -# - Pull previous project image to use it as a layer cache if it exists. -# - Perform the build of the image itself. With the cache primed we should only have to rebuild the necessary layers. -# - Push the image tagged with the commit hash to the cache. +# - For images built previously in pipeline, pull the image we've built. +# - Perform the build of the image itself. +# - Push the image tagged with the content hash to the cache. [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -euo pipefail REPOSITORY=$1 -FORCE_BUILD=${2:-"false"} -ARCH=${3:-""} DOCKERFILE=$(query_manifest dockerfile $REPOSITORY) PROJECT_DIR=$(query_manifest projectDir $REPOSITORY) BUILD_DIR=$(query_manifest buildDir $REPOSITORY) @@ -36,7 +30,6 @@ echo "Repository: $REPOSITORY" echo "Working directory: $PWD" echo "Dockerfile: $DOCKERFILE" echo "Build directory: $BUILD_DIR" -echo "Arch: $ARCH" # Fetch images with retries function fetch_image() { @@ -67,7 +60,8 @@ echo "Content hash: $CONTENT_HASH" cd $BUILD_DIR # If we have previously successful commit, we can early out if nothing relevant has changed since. -if [[ $FORCE_BUILD == 'false' ]] && check_rebuild cache-"$CONTENT_HASH" $REPOSITORY; then +IMAGE_COMMIT_TAG=$(calculate_image_tag $REPOSITORY) +if check_rebuild $IMAGE_COMMIT_TAG $REPOSITORY; then echo "No rebuild necessary." exit 0 fi @@ -94,15 +88,17 @@ fi # For each dependency, pull in the latest image and give it correct tag. for PARENT_REPO in $(query_manifest dependencies $REPOSITORY); do - PARENT_CONTENT_HASH=$(calculate_content_hash $PARENT_REPO) - # There must be a parent image to continue. - if [ -z "$PARENT_CONTENT_HASH" ]; then - echo "No parent image found for $PARENT_REPO" - exit 1 + PARENT_IMAGE_URI=$(calculate_image_uri $PARENT_REPO) + echo "Pulling dependency $PARENT_IMAGE_URI..." + if ! fetch_image $PARENT_IMAGE_URI; then + # This is a *bit* of a hack maybe. Some of our arm images can be built from x86 dependents. + # e.g. node projects are architecture independent. + # This may not hold true if we start introducing npm modules that are backed by native code. + # But for now, to avoid building some projects twice, we can fallback onto x86 variant. + PARENT_IMAGE_URI=$(calculate_image_uri $PARENT_REPO x86_64) + echo "Falling back onto x86 build. Pulling dependency $PARENT_IMAGE_URI..." + fetch_image $PARENT_IMAGE_URI fi - PARENT_IMAGE_URI=$ECR_URL/$PARENT_REPO:cache-$PARENT_CONTENT_HASH - echo "Pulling dependency $PARENT_REPO..." - fetch_image $PARENT_IMAGE_URI # Tag it to look like an official release as that's what we use in Dockerfiles. TAG=$ECR_DEPLOY_URL/$PARENT_REPO docker tag $PARENT_IMAGE_URI $TAG @@ -112,10 +108,7 @@ COMMIT_TAG_VERSION=$(extract_tag_version $REPOSITORY false) echo "Commit tag version: $COMMIT_TAG_VERSION" # Build the actual image and give it a commit tag. -IMAGE_COMMIT_URI=$ECR_URL/$REPOSITORY:cache-$CONTENT_HASH -if [[ -n "$ARCH" ]]; then - IMAGE_COMMIT_URI=$IMAGE_COMMIT_URI-$ARCH -fi +IMAGE_COMMIT_URI=$(calculate_image_uri $REPOSITORY) echo "Building image: $IMAGE_COMMIT_URI" docker build -t $IMAGE_COMMIT_URI -f $DOCKERFILE --build-arg COMMIT_TAG=$COMMIT_TAG_VERSION --build-arg ARG_CONTENT_HASH=$CONTENT_HASH . echo "Pushing image: $IMAGE_COMMIT_URI" diff --git a/build-system/scripts/calculate_image_tag b/build-system/scripts/calculate_image_tag new file mode 100755 index 00000000000..2a7786ecdc1 --- /dev/null +++ b/build-system/scripts/calculate_image_tag @@ -0,0 +1,9 @@ +#!/bin/bash +[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace +set -eu + +REPOSITORY=$1 +ARCH=${2:-$(uname -m)} +[ "$ARCH" == "aarch64" ] && ARCH=arm64 +CONTENT_HASH=$(calculate_content_hash $REPOSITORY) +echo "cache-$CONTENT_HASH-$ARCH" diff --git a/build-system/scripts/calculate_image_uri b/build-system/scripts/calculate_image_uri index bb276ed6187..fe60a9c05ab 100755 --- a/build-system/scripts/calculate_image_uri +++ b/build-system/scripts/calculate_image_uri @@ -3,5 +3,7 @@ set -eu REPOSITORY=$1 +ARCH=${2:-$(uname -m)} +[ "$ARCH" == "aarch64" ] && ARCH=arm64 CONTENT_HASH=$(calculate_content_hash $REPOSITORY) -echo "$ECR_URL/$REPOSITORY:cache-$CONTENT_HASH" +echo "$ECR_URL/$REPOSITORY:cache-$CONTENT_HASH-$ARCH" diff --git a/build-system/scripts/check_npm_version b/build-system/scripts/check_npm_version deleted file mode 100755 index 35173d7e74d..00000000000 --- a/build-system/scripts/check_npm_version +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace -set -eu - -readonly LOCAL_VERSION=$(node -pe "require('./package.json').version") -readonly PACKAGE_NAME=${1:-./} -readonly PUBLISHED_VERSION=$(npm show ${PACKAGE_NAME} version 2> /dev/null) -readonly HIGHER_VERSION=$(echo $(npx semver ${LOCAL_VERSION} ${PUBLISHED_VERSION}) | cut -d ' ' -f 2) - -if [ "$(npx semver $LOCAL_VERSION)" == "" ]; then - echo "'$LOCAL_VERSION' is not a semantic version." - exit 1 -elif [ "$LOCAL_VERSION" != "$PUBLISHED_VERSION" ] && [ "$LOCAL_VERSION" == "$HIGHER_VERSION" ]; then - exit 0 -else - echo "Expect npm version number to be higher than '$PUBLISHED_VERSION'. Current local version is '$LOCAL_VERSION'." - exit 1 -fi diff --git a/build-system/scripts/cond_run_compose b/build-system/scripts/cond_run_compose index 025b66f7abc..ee412325a6a 100755 --- a/build-system/scripts/cond_run_compose +++ b/build-system/scripts/cond_run_compose @@ -6,11 +6,10 @@ REPOSITORY=$1 COMPOSE_FILE=$2 shift 2 -CONTENT_HASH=$(calculate_content_hash $REPOSITORY) -BASE_TAG=cache-$CONTENT_HASH +BASE_TAG=$(calculate_image_tag $REPOSITORY) SUCCESS_TAG=$BASE_TAG-$JOB_NAME -echo "Content hash: $CONTENT_HASH" +echo "Success tag: $SUCCESS_TAG" if ! check_rebuild $SUCCESS_TAG $REPOSITORY; then # Login to pull our ecr images with docker. @@ -27,7 +26,12 @@ if ! check_rebuild $SUCCESS_TAG $REPOSITORY; then cd $(query_manifest projectDir $REPOSITORY) export $@ - docker-compose -f $COMPOSE_FILE up --exit-code-from $REPOSITORY --force-recreate + if docker compose > /dev/null 2>&1; then + CMD="docker compose" + else + CMD="docker-compose" + fi + $CMD -f $COMPOSE_FILE up --exit-code-from $REPOSITORY --force-recreate upload_logs_to_s3 log diff --git a/build-system/scripts/cond_run_container b/build-system/scripts/cond_run_container index 98562f1c379..ca8d67fc0bd 100755 --- a/build-system/scripts/cond_run_container +++ b/build-system/scripts/cond_run_container @@ -10,11 +10,10 @@ set -eu REPOSITORY=$1 shift -CONTENT_HASH=$(calculate_content_hash $REPOSITORY) -BASE_TAG=cache-$CONTENT_HASH +BASE_TAG=$(calculate_image_tag $REPOSITORY) SUCCESS_TAG=$BASE_TAG-$JOB_NAME -echo "Content hash: $CONTENT_HASH" +echo "Success tag: $SUCCESS_TAG" if ! check_rebuild $SUCCESS_TAG $REPOSITORY; then IMAGE_URI=$(calculate_image_uri $REPOSITORY) diff --git a/build-system/scripts/cond_run_script b/build-system/scripts/cond_run_script index c14214ed94d..c1ada5f8cf2 100755 --- a/build-system/scripts/cond_run_script +++ b/build-system/scripts/cond_run_script @@ -10,11 +10,10 @@ set -eu REPOSITORY=$1 shift -CONTENT_HASH=$(calculate_content_hash $REPOSITORY) -BASE_TAG=cache-$CONTENT_HASH +BASE_TAG=$(calculate_image_tag $REPOSITORY) SUCCESS_TAG=$BASE_TAG-$JOB_NAME -echo "Content hash: $CONTENT_HASH" +echo "Success tag: $SUCCESS_TAG" if ! check_rebuild $SUCCESS_TAG $REPOSITORY; then init_submodules $REPOSITORY diff --git a/build-system/scripts/cond_spot_run_build b/build-system/scripts/cond_spot_run_build index 9ed6596d052..6e970387cce 100755 --- a/build-system/scripts/cond_spot_run_build +++ b/build-system/scripts/cond_spot_run_build @@ -1,8 +1,9 @@ #!/bin/bash [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace -set -eu +set -euo pipefail REPOSITORY=$1 CPUS=$2 +ARCH=${3:-x86_64} -cond_spot_run_script $REPOSITORY $CPUS build $REPOSITORY +cond_spot_run_script $REPOSITORY $CPUS $ARCH build $REPOSITORY $ARCH | add_timestamps diff --git a/build-system/scripts/cond_spot_run_compose b/build-system/scripts/cond_spot_run_compose index e210bd94648..51a4529471a 100755 --- a/build-system/scripts/cond_spot_run_compose +++ b/build-system/scripts/cond_spot_run_compose @@ -7,4 +7,4 @@ CPUS=$2 shift 2 export TAG_POSTFIX=$JOB_NAME -cond_spot_run_script $REPOSITORY $CPUS cond_run_compose $REPOSITORY $@ 2>&1 | add_timestamps +cond_spot_run_script $REPOSITORY $CPUS x86_64 cond_run_compose $REPOSITORY $@ 2>&1 | add_timestamps diff --git a/build-system/scripts/cond_spot_run_container b/build-system/scripts/cond_spot_run_container index 98ccf7d55e6..48876a65f3b 100755 --- a/build-system/scripts/cond_spot_run_container +++ b/build-system/scripts/cond_spot_run_container @@ -7,4 +7,4 @@ CPUS=$2 shift 2 export TAG_POSTFIX=$JOB_NAME -cond_spot_run_script $REPOSITORY $CPUS cond_run_container $REPOSITORY $@ +cond_spot_run_script $REPOSITORY $CPUS x86_64 cond_run_container $REPOSITORY $@ diff --git a/build-system/scripts/cond_spot_run_script b/build-system/scripts/cond_spot_run_script index 9474ff6923c..89d41c9af0e 100755 --- a/build-system/scripts/cond_spot_run_script +++ b/build-system/scripts/cond_spot_run_script @@ -16,20 +16,21 @@ set -eu REPOSITORY=$1 CPUS=$2 -shift 2 +ARCH=$3 +shift 3 -CONTENT_HASH=$(calculate_content_hash $REPOSITORY) -BASE_TAG=cache-$CONTENT_HASH +# If the CPUS have a specific architecture assigned, we need to use that to build the success tag. +BASE_TAG=$(calculate_image_tag $REPOSITORY $ARCH) SUCCESS_TAG=$BASE_TAG if [ -n "${TAG_POSTFIX:-}" ]; then SUCCESS_TAG=$BASE_TAG-$TAG_POSTFIX fi -echo "Content hash: $CONTENT_HASH" +echo "Success tag: $SUCCESS_TAG" if ! check_rebuild $SUCCESS_TAG $REPOSITORY; then init_submodules $REPOSITORY - spot_run_script $CONTENT_HASH $CPUS $@ + spot_run_script $SUCCESS_TAG $CPUS $ARCH $@ retry tag_remote_image $REPOSITORY $BASE_TAG $SUCCESS_TAG fi diff --git a/build-system/scripts/cond_spot_run_test b/build-system/scripts/cond_spot_run_test index ca3249f1b7f..e8b8f71358d 100755 --- a/build-system/scripts/cond_spot_run_test +++ b/build-system/scripts/cond_spot_run_test @@ -13,5 +13,5 @@ SCRIPT=$(query_manifest relativeProjectDir $REPOSITORY)/$SCRIPT # Specify a TAG_POSTFIX as the JOB_NAME mkdir -p /tmp/test-logs -export TAG_POSTFIX=$JOB_NAME -cond_spot_run_script $REPOSITORY $CPUS $SCRIPT $@ | tee "/tmp/test-logs/$JOB_NAME.log" +export TAG_POSTFIX=$JOB_NAME +cond_spot_run_script $REPOSITORY $CPUS x86_64 $SCRIPT $@ | tee "/tmp/test-logs/$JOB_NAME.log" diff --git a/build-system/scripts/create_dockerhub_manifest b/build-system/scripts/create_dockerhub_manifest deleted file mode 100755 index 26461d419f8..00000000000 --- a/build-system/scripts/create_dockerhub_manifest +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -# This script: -# 1. Logs into DockerHub -# 2. Creates 2 manifest lists, the first is version tagged, the second is 'latest' tagged -# 3. Adds the arch specific tagged image to each list -# 4. Pushes the 2 lists - -[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace -set -eu - -if [ -z "$COMMIT_TAG" ]; then - echo "Will only push tagged builds to dockerhub. Skipping." - exit 0 -fi - -REPOSITORY=$1 -ARCH_LIST=$2 -DIST_TAG=${3:-"latest"} - -echo "Repo: $REPOSITORY" -echo "Arch list: $ARCH_LIST" - -# Extract the version, if valid; exit otherwise. -IMAGE_TAG=$(extract_tag_version $REPOSITORY true) - -MANIFEST_DEPLOY_URI=$DOCKERHUB_ACCOUNT/$REPOSITORY:$IMAGE_TAG -MANIFEST_DIST_URI=$DOCKERHUB_ACCOUNT/$REPOSITORY:$DIST_TAG - -# Login to dockerhub. -dockerhub_login - -export DOCKER_CLI_EXPERIMENTAL=enabled - -OLD_IFS=$IFS -IFS=',' - -# For each arch, add the tagged image to 2 manifest lists. One tagged with the version, the other with 'latest' -for A in $ARCH_LIST; do - IMAGE_DEPLOY_URI=$DOCKERHUB_ACCOUNT/$REPOSITORY:$IMAGE_TAG-$A - echo "Adding image $IMAGE_DEPLOY_URI to manifest list $MANIFEST_DEPLOY_URI" - docker_or_dryrun manifest create $MANIFEST_DEPLOY_URI \ - --amend $IMAGE_DEPLOY_URI - - echo "Adding image $IMAGE_DEPLOY_URI to manifest list $MANIFEST_DIST_URI" - docker_or_dryrun manifest create $MANIFEST_DIST_URI \ - --amend $IMAGE_DEPLOY_URI -done - -IFS=$OLD_IFS -unset OLD_IFS - -echo "Pushing manifest list $MANIFEST_DEPLOY_URI..." -# Push the version tagged list -docker_or_dryrun manifest push --purge $MANIFEST_DEPLOY_URI - -echo "Pushing manifest list $MANIFEST_DIST_URI..." -# Push the latest tagged list -docker_or_dryrun manifest push --purge $MANIFEST_DIST_URI diff --git a/build-system/scripts/create_ecr_manifest b/build-system/scripts/create_ecr_manifest index 2e9fad6ae68..5bda420039d 100755 --- a/build-system/scripts/create_ecr_manifest +++ b/build-system/scripts/create_ecr_manifest @@ -11,25 +11,20 @@ set -eu REPOSITORY=$1 ARCH_LIST=$2 -# Ensure ECR repository exists. -retry ensure_repo $REPOSITORY $ECR_REGION refresh_lifecycle +ecr_login -IMAGE_URI=$(calculate_image_uri $REPOSITORY) -echo "Image URI: $IMAGE_URI" +CONTENT_HASH=$(calculate_content_hash $REPOSITORY) +MULTIARCH_IMAGE_URI=$ECR_URL/$REPOSITORY:cache-$CONTENT_HASH -echo "Creating manifest list..." +echo "Multi-arch Image URI: $MULTIARCH_IMAGE_URI" export DOCKER_CLI_EXPERIMENTAL=enabled -OLD_IFS=$IFS IFS=',' -for A in $ARCH_LIST -do - ARCH_IMAGE=$IMAGE_URI-$A - echo "Adding image $ARCH_IMAGE to manifest list." - retry docker manifest create $IMAGE_URI --amend $ARCH_IMAGE +for A in $ARCH_LIST; do + IMAGE_URI=$(calculate_image_uri $REPOSITORY $A) + echo "Adding image $IMAGE_URI to manifest list $MULTIARCH_IMAGE_URI..." + docker manifest create $MULTIARCH_IMAGE_URI --amend $IMAGE_URI done -IFS=$OLD_IFS -unset OLD_IFS -retry docker manifest push --purge $IMAGE_URI +retry docker manifest push --purge $MULTIARCH_IMAGE_URI diff --git a/build-system/scripts/deploy b/build-system/scripts/deploy index a404733ad91..06410f5f031 100755 --- a/build-system/scripts/deploy +++ b/build-system/scripts/deploy @@ -19,7 +19,7 @@ if check_rebuild cache-$CONTENT_HASH-$DEPLOY_TAG-deployed $REPOSITORY; then exit 0 fi -deploy_terraform $REPOSITORY ./terraform/$VERSION_TAG "$TO_TAINT" +deploy_terraform $REPOSITORY ./terraform/$DEPLOY_ENV "$TO_TAINT" # Restart services. for SERVICE in $SERVICES; do diff --git a/build-system/scripts/deploy_dockerhub b/build-system/scripts/deploy_dockerhub index ebc87de79f1..06d0933d430 100755 --- a/build-system/scripts/deploy_dockerhub +++ b/build-system/scripts/deploy_dockerhub @@ -2,56 +2,44 @@ [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -eu -if [ -z "$COMMIT_TAG" ]; then - echo "Will only push tagged builds to dockerhub. Skipping." - exit 0 -fi - REPOSITORY=$1 -ARCH=${2:-""} -DIST_TAG=${3:-"latest"} - -echo "Repo: $REPOSITORY" -echo "Arch: $ARCH" - -IMAGE_COMMIT_URI=$(calculate_image_uri $REPOSITORY) +# TODO: Why even provide this? We can just figure it out by probing. +ARCH_LIST=${2:-} -IMAGE_DIST_URI=$ACCOUNT/$REPOSITORY:$DIST_TAG -if [[ -n "$ARCH" ]]; then - IMAGE_COMMIT_URI=$IMAGE_COMMIT_URI-$ARCH - IMAGE_DIST_URI=$IMAGE_DIST_URI-$ARCH -fi +function docker_or_dryrun { + if [ "$DRY_DEPLOY" -eq 1 ] ; then + echo DRY RUN: docker $@ + else + retry docker $@ + fi +} -COMMIT_TAG_VERSION=$(extract_tag_version $REPOSITORY true) - -IMAGE_TAG=$COMMIT_TAG_VERSION -IMAGE_DEPLOY_URI=$ACCOUNT/$REPOSITORY:$IMAGE_TAG -if [[ -n "$ARCH" ]]; then - IMAGE_DEPLOY_URI=$IMAGE_DEPLOY_URI-$ARCH -fi - -echo "Deploying to dockerhub: $IMAGE_DEPLOY_URI" +echo "Repo: $REPOSITORY" +echo "Arch List: $ARCH_LIST" -# Login. -retry ensure_repo $REPOSITORY $ECR_DEPLOY_REGION +VERSION_TAG=$(extract_tag_version $REPOSITORY true) +MANIFEST_DEPLOY_URI=aztecprotocol/$REPOSITORY:$VERSION_TAG # Login to dockerhub and ecr dockerhub_login ecr_login -echo "Pulling $IMAGE_COMMIT_URI" -# Pull image. -retry docker pull $IMAGE_COMMIT_URI +IFS=',' +for ARCH in $ARCH_LIST; do + IMAGE_COMMIT_URI=$(calculate_image_uri $REPOSITORY $ARCH) + retry docker pull $IMAGE_COMMIT_URI + + # Retag and push image. + IMAGE_DEPLOY_URI=aztecprotocol/$REPOSITORY:$VERSION_TAG-$ARCH + docker tag $IMAGE_COMMIT_URI $IMAGE_DEPLOY_URI + docker_or_dryrun push $IMAGE_DEPLOY_URI -echo "Tagging $IMAGE_COMMIT_URI as $IMAGE_DEPLOY_URI" -# Tag with commit tag -docker_or_dryrun tag $IMAGE_COMMIT_URI $IMAGE_DEPLOY_URI + echo "Adding image $IMAGE_DEPLOY_URI to manifest list $MANIFEST_DEPLOY_URI..." + docker_or_dryrun manifest create $MANIFEST_DEPLOY_URI --amend $IMAGE_DEPLOY_URI +done -echo "Tagging $IMAGE_COMMIT_URI as $IMAGE_DIST_URI" -# Tag with :$DIST_TAG (latest or canary) -docker_or_dryrun tag $IMAGE_COMMIT_URI $IMAGE_DIST_URI +docker_or_dryrun push $MANIFEST_DEPLOY_URI -# Push tagged image to dockerhub. -docker_or_dryrun push $IMAGE_DEPLOY_URI -# Push :latest or :canary image to dockerhub -docker_or_dryrun push $IMAGE_DIST_URI +# Retag versioned as latest. +docker_or_dryrun tag $MANIFEST_DEPLOY_URI aztecprotocol/$REPOSITORY:latest +docker_or_dryrun push aztecprotocol/$REPOSITORY:latest \ No newline at end of file diff --git a/build-system/scripts/deploy_npm b/build-system/scripts/deploy_npm index 59d3cc034b4..a35d68c4650 100755 --- a/build-system/scripts/deploy_npm +++ b/build-system/scripts/deploy_npm @@ -50,8 +50,8 @@ if [ -z "$STANDALONE" ]; then fi # Publish if we have a commit tag -if [ -n "$COMMIT_TAG" ] ; then - npm publish $TAG_ARG --access public -else +if [ "$DRY_DEPLOY" -eq 1 ] ; then npm publish --dry-run $TAG_ARG --access public +else + npm publish $TAG_ARG --access public fi diff --git a/build-system/scripts/deploy_s3 b/build-system/scripts/deploy_s3 index b4180957121..caaa4d00bd9 100755 --- a/build-system/scripts/deploy_s3 +++ b/build-system/scripts/deploy_s3 @@ -22,7 +22,7 @@ if [ -n "$IS_GLOBAL" ]; then TF_DIR=$ROOT_PATH/$PROJECT_DIR/terraform deploy_terraform "" $TF_DIR else - TF_DIR=$ROOT_PATH/$PROJECT_DIR/terraform/$VERSION_TAG + TF_DIR=$ROOT_PATH/$PROJECT_DIR/terraform/$DEPLOY_ENV deploy_terraform $REPOSITORY $TF_DIR fi diff --git a/build-system/scripts/docker_or_dryrun b/build-system/scripts/docker_or_dryrun deleted file mode 100755 index 004359417b1..00000000000 --- a/build-system/scripts/docker_or_dryrun +++ /dev/null @@ -1,5 +0,0 @@ -if [ -n "$COMMIT_TAG" ] ; then - retry docker $@ -else - echo DRY RUN: docker $@ -fi diff --git a/build-system/scripts/ensure_apt_package b/build-system/scripts/ensure_apt_package deleted file mode 100755 index 08d9269d308..00000000000 --- a/build-system/scripts/ensure_apt_package +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace -set -eu - -if dpkg -s $1 &> /dev/null; then - exit 0 -fi - ->&2 echo "Installing $1..." -sudo apt update &> /dev/null && sudo apt install --force-yes $1 &> /dev/null diff --git a/build-system/scripts/erase_image_tags b/build-system/scripts/erase_image_tags deleted file mode 100755 index 6c171e9f0ad..00000000000 --- a/build-system/scripts/erase_image_tags +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -# Erase the image tag associated with the last commit for the given repository. -# If TILL_COMMIT_HASH is given, erase tags going back in time until we reach TILL_COMMIT_HASH. -[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace -set -eu - -REPOSITORY=$1 -TILL_COMMIT_HASH=$2 - -if [ -n "${3:-}" ]; then - TAG_POSTFIX=-$3 -fi - -for COMMIT_HASH in $(git log -n 50 --pretty=format:"%H"); do - CONTENT_HASH=$(calculate_content_hash $REPOSITORY $COMMIT_HASH) - TAG=cache-${CONTENT_HASH}$TAG_POSTFIX - if image_exists $REPOSITORY $TAG; then - echo "Erasing image tag: $REPOSITORY:$TAG" - aws ecr batch-delete-image --region=$ECR_REGION --repository-name $1 --image-ids imageTag=$TAG > /dev/null - if [ -z "$TILL_COMMIT_HASH" -o "$COMMIT_HASH" = "$TILL_COMMIT_HASH" ]; then - break - fi - fi -done diff --git a/build-system/scripts/request_spot b/build-system/scripts/request_spot index 3eef009d8ab..211ffef56b8 100755 --- a/build-system/scripts/request_spot +++ b/build-system/scripts/request_spot @@ -4,6 +4,7 @@ set -eu NAME=$1 CPUS=$2 +ARCH=$3 # Declare an associative array to map CPU counts to instance type suffixes. declare -A cpu_map @@ -29,7 +30,17 @@ if [ -z "$INSTANCE_TYPE_SUFFIX" ]; then fi # Construct the full instance type. We use r6a currently. -INSTANCE_TYPE="r6a.$INSTANCE_TYPE_SUFFIX" +if [ "$ARCH" == "x86_64" ]; then + MACHINE_TYPE="r6a" + AMI="ami-0e5df77ac318c7a18" +elif [ "$ARCH" == "arm64" ]; then + MACHINE_TYPE="r7g" + AMI="ami-02c5a5064fe72b22e" +else + echo "Unknown arch: $ARCH" + exit 1 +fi +INSTANCE_TYPE="$MACHINE_TYPE.$INSTANCE_TYPE_SUFFIX" export AWS_DEFAULT_REGION=us-east-2 @@ -38,7 +49,7 @@ PRICE=$(jq -n "$BID_PER_CPU_HOUR*$CPUS*100000 | round / 100000") launch_spec=$(cat <&2 echo "Waiting for SSH at $IP..." SECONDS=0 -while ! nc -z -w1 $IP 22; do +SSH_CONFIG_PATH=${SSH_CONFIG_PATH:-$BUILD_SYSTEM_PATH/remote/ssh_config} +while ! ssh -F $SSH_CONFIG_PATH -o ConnectTimeout=1 $IP true > /dev/null 2>&1; do if (( SECONDS >= 60 )); then - >&2 echo "Timeout: SSH port 22 at $IP did not become available after 60 seconds." + >&2 echo "Timeout: SSH could not login to $IP within 60 seconds." exit 1 fi + sleep 1 done echo $IP \ No newline at end of file diff --git a/build-system/scripts/run_script b/build-system/scripts/run_script deleted file mode 100755 index 97ae0163fb4..00000000000 --- a/build-system/scripts/run_script +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -# Runs a script from within the provided repository project directory -# -# Arguments are: -# 1. REPOSITORY: The project repository name in ECR. -# 2... ARGS: Script and arguments to run. -[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace -set -eu - -REPOSITORY=$1 -shift -cd $(query_manifest projectDir $REPOSITORY) -$@ diff --git a/build-system/scripts/setup_env b/build-system/scripts/setup_env index dbc58ac5a45..d227545fd30 100755 --- a/build-system/scripts/setup_env +++ b/build-system/scripts/setup_env @@ -18,8 +18,6 @@ PULL_REQUEST=${6:-} BASH_ENV=${BASH_ENV:-} BUILD_SYSTEM_PATH=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) PROJECT=$(cat PROJECT) -VERSION=$(cat VERSION) -VERSION=${VERSION%% x-release-please-version*} COMMIT_MESSAGE=$(git log -n 1 --pretty=format:"%s" $COMMIT_HASH) echo "COMMIT_HASH=$COMMIT_HASH" @@ -27,41 +25,33 @@ echo "COMMIT_TAG=$COMMIT_TAG" echo "JOB_NAME=$JOB_NAME" echo "GIT_REPOSITORY_URL=$GIT_REPOSITORY_URL" echo "BRANCH=$BRANCH" +echo "PULL_REQUEST=$PULL_REQUEST" -if [ -n "${COMMIT_TAG:-}" ]; then - # We're tagged e.g. v2.1.123 or v2.1.123-testnet.0. - # First we sanity check that the tag matches the VERSION file. - echo "Extracting commit tag version..." - # Strips the e.g. '-testnet.XX' from the COMMIT_TAG - WITHOUT_VERSION_TAG=${COMMIT_TAG%%-*} - # Strips the subversion, leaving just the 'vMAJ.MIN' e.g. v2.1 - COMMIT_TAG_VERSION=${WITHOUT_VERSION_TAG%.*} - echo "WITHOUT_VERSION_TAG=$WITHOUT_VERSION_TAG" - echo "COMMIT_TAG_VERSION=$COMMIT_TAG_VERSION" - # COMMIT_TAG_VERSION=$(echo "$COMMIT_TAG" | grep -oE "v\d+.\d+") - if [ "$COMMIT_TAG" != "$VERSION" ]; then - echo "Commit tag $COMMIT_TAG does not match repo version $VERSION." - exit 1 - fi +# If the user has requested to perform a "dry deploy", we set the commit tag to fake version, and set DRY_DEPLOY to 1. +if [[ "$COMMIT_MESSAGE" == *"[ci dry-deploy]"* ]]; then + COMMIT_TAG=v999.999.999 + DRY_DEPLOY=1 +fi - # Extract the version tag from the commit tag, if it has one, e.g. testnet. +if [ -n "${COMMIT_TAG:-}" ]; then + # Extract the deploy env from the commit tag, if it has one, e.g. testnet. # If we have one, we look something like v2.1.123-testnet.0. This is a "non production" release. if [[ "$COMMIT_TAG" == *"-"* ]]; then # Strips the trailing '.XX' from the end of the commit tag TEMP=${COMMIT_TAG%.*} # Strips the 'vX.Y.ZZZ-' from the front of the commit tag, leaving the e.g. 'testnet' - VERSION_TAG=${TEMP##*-} + DEPLOY_ENV=${TEMP##*-} else - VERSION_TAG=prod + DEPLOY_ENV=prod fi else - # We have no commit tag. Check this is our staging branch, otherwise use dev. - [[ $BRANCH = "stage" ]] && VERSION_TAG="stage" || VERSION_TAG="dev" + # We have no commit tag, this is our devnet. + DEPLOY_ENV="dev" fi -# - The deploy tag (used in api paths, subdomains), is a concatenation of the project name and version tag, -# e.g. aztec-connect-dev, aztec-connect-testnet, or aztec-connect-prod -DEPLOY_TAG=$PROJECT-$VERSION_TAG +# - The deploy tag (used in api paths, subdomains), is a concatenation of the project name and deploy environment, +# e.g. aztec-dev, aztec-testnet, or aztec-prod +DEPLOY_TAG=$PROJECT-$DEPLOY_ENV if [ -z "$BASH_ENV" ]; then BASH_ENV=$(mktemp) @@ -70,6 +60,7 @@ fi echo export ROOT_PATH=$PWD >> $BASH_ENV echo export BUILD_SYSTEM_PATH=$BUILD_SYSTEM_PATH >> $BASH_ENV echo export DOCKER_BUILDKIT=${DOCKER_BUILDKIT:-1} >> $BASH_ENV +echo export BUILDKIT_PROGRESS=plain >> $BASH_ENV echo export SSH_CONFIG_PATH=$BUILD_SYSTEM_PATH/remote/ssh_config >> $BASH_ENV echo export PATH=\$PATH:$BUILD_SYSTEM_PATH/scripts >> $BASH_ENV echo export AWS_DEFAULT_REGION=eu-west-2 >> $BASH_ENV @@ -86,14 +77,22 @@ echo export COMMIT_TAG=$COMMIT_TAG >> $BASH_ENV echo "export COMMIT_MESSAGE='${COMMIT_MESSAGE//\'/\'\\\'\'}'" >> $BASH_ENV echo export JOB_NAME=$JOB_NAME >> $BASH_ENV echo export GIT_REPOSITORY_URL=$GIT_REPOSITORY_URL >> $BASH_ENV -echo export VERSION_TAG=$VERSION_TAG >> $BASH_ENV +echo export DEPLOY_ENV=$DEPLOY_ENV >> $BASH_ENV echo export DEPLOY_TAG=$DEPLOY_TAG >> $BASH_ENV echo export BRANCH=$BRANCH >> $BASH_ENV echo export PULL_REQUEST=$PULL_REQUEST >> $BASH_ENV +echo export DRY_DEPLOY=${DRY_DEPLOY:-} >> $BASH_ENV +# We want very strict failures on any failing command, undefined variable, or commands that pipe to other commands. +echo set -euo pipefail >> $BASH_ENV +# Enable logging if [ci debug] is in commit message. +if [[ "$COMMIT_MESSAGE" == *"[ci debug]"* ]]; then + echo export BUILD_SYSTEM_DEBUG=1 >> $BASH_ENV + echo set -x >> $BASH_ENV +fi # Only run the following if we're the result of a commit (i.e. not being run manually). if [ -n "$COMMIT_HASH" ]; then - # Our alpine build image has yq installed, ubuntu build instances download from github. + # Our alpine build image has yq installed, ubuntu build instances install from gzips in ../bin. if [ ! -f /etc/alpine-release ]; then BIN_PATH=$BUILD_SYSTEM_PATH/bin echo export PATH=$BIN_PATH:\$PATH >> $BASH_ENV diff --git a/build-system/scripts/should_deploy b/build-system/scripts/should_deploy new file mode 100755 index 00000000000..aac1504ab33 --- /dev/null +++ b/build-system/scripts/should_deploy @@ -0,0 +1,7 @@ +#!/bin/bash +# Retuns success if we are expected to do a deploy. +# At present this is only if we have a commit tag. +# Once we are doing master deployments to devnet, we'll want to check if BRANCH is master. +set -eu + +[ -n "$COMMIT_TAG" ] \ No newline at end of file diff --git a/build-system/scripts/spot_run_script b/build-system/scripts/spot_run_script index 33a0fc3dfb0..7a2377973d5 100755 --- a/build-system/scripts/spot_run_script +++ b/build-system/scripts/spot_run_script @@ -1,6 +1,6 @@ #!/bin/bash # Runs a test script on a remote spot instance. Arguments are: -# 1. CONTENT_HASH: Content hash for our repository contents. Used to identify spot jobs and image tags. +# 1. NAME: Used to identify spot jobs. # 2. CPUS: Number of cpus on spot request. # 3... ARGS: Script and arguments to run. # @@ -8,13 +8,15 @@ # JOB_NAME: Set within setup-env. The job name as per CI. [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -eu -CONTENT_HASH=$1 +NAME=$1 CPUS=$2 -shift 2 +ARCH=$3 +shift 3 IP= # On any sort of exit (error or not). function on_exit { + # We want to execute all this block, regardless of potential errors. set +e if [ -n "$IP" ]; then @@ -23,8 +25,8 @@ function on_exit { fi # Kill spot request so it doesn't count against quota. - if [ -f "sir-$CONTENT_HASH:$JOB_NAME.txt" ]; then - SIR=$(cat sir-$CONTENT_HASH:$JOB_NAME.txt) + if [ -f "sir-$NAME.txt" ]; then + SIR=$(cat sir-$NAME.txt) echo "Cancelling spot instance request $SIR..." aws ec2 cancel-spot-instance-requests --spot-instance-request-ids $SIR >/dev/null 2>&1 || true fi @@ -32,7 +34,7 @@ function on_exit { trap on_exit EXIT # Get spot instance. -IP=$(retry request_spot $CONTENT_HASH:$JOB_NAME $CPUS) +IP=$(retry request_spot $NAME $CPUS $ARCH) if [ -z "$IP" ]; then echo "Failed to get spot instance." diff --git a/build-system/scripts/spot_run_test_script b/build-system/scripts/spot_run_test_script deleted file mode 100755 index f3f05e0475c..00000000000 --- a/build-system/scripts/spot_run_test_script +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -set -eu -SCRIPT_PATH=$1 -REPOSITORY=$2 -shift -shift - -cd $(query_manifest projectDir $REPOSITORY) - -mkdir -p /tmp/test-logs - -[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace -set -o pipefail - -CONTENT_HASH=$(calculate_content_hash $REPOSITORY) -echo "Content hash: $CONTENT_HASH" -spot_run_script $CONTENT_HASH 32 $SCRIPT_PATH $@ | tee "/tmp/test-logs/$JOB_NAME.log" diff --git a/build-system/scripts/store_test_benchmark_logs b/build-system/scripts/store_test_benchmark_logs deleted file mode 100755 index 02ad0329614..00000000000 --- a/build-system/scripts/store_test_benchmark_logs +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash -[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace -set -eu - -REPOSITORY=$1 -shift - -cd $(query_manifest projectDir $REPOSITORY) - -mkdir -p /tmp/csv -export SAVED_TIMESTAMP=$(date +%s) -export HUMAN_READABLE_TIME=$(date -u -d @${SAVED_TIMESTAMP}) - -# Pick logs from test-logs and convert all information into CSV format including the current timestamp, branch, commit and tag information -for file in $(ls /tmp/test-logs); do - echo $file - cat /tmp/test-logs/$file | grep "##BENCHMARK_INFO_PREFIX##" | sed "s/.*##BENCHMARK_INFO_PREFIX##\(.*\)##BENCHMARK_INFO_SUFFIX##.*/\1/" | sed "s/#/,/g" | sed "s_\(.*\)_$SAVED_TIMESTAMP,$HUMAN_READABLE_TIME,$BRANCH,$COMMIT_HASH,$COMMIT_TAG,\1_" | tee -a /tmp/csv/new.csv 1>/dev/null -done -echo "Parsed from logs:" -cat /tmp/csv/new.csv - -# We have lots of repeated entries, no need to put them into repository. Unfortunately build times differ a bit and uniq only works with space as separator -cat /tmp/csv/new.csv | sort | sed "s_ _%_g" | sed "s_^\(.*\),\(.*\)\$_\2 \1_" | uniq -f 1 | sed "s_^\(.*\) \(.*\)\$_\2,\1_" | sed "s_%_ _g" >/tmp/csv/trimmed.csv - -# If there actually were any logs, update the information in the benchmark repository -if [ -s /tmp/csv/trimmed.csv ]; then - cd /tmp - git clone --depth 1 git@github.com:AztecProtocol/benchmark-archive.git - - cd benchmark-archive - git config user.email "circleci@bot" - git config user.name "CircleCi Bot" - cat /tmp/csv/trimmed.csv >>benchmarks.csv - git add benchmarks.csv - git commit -m "Added information from branch $BRANCH commit $COMMIT_HASH" - git push -fi diff --git a/build_manifest.yml b/build_manifest.yml index 99f5bad4072..53f1a8db4f1 100644 --- a/build_manifest.yml +++ b/build_manifest.yml @@ -162,3 +162,6 @@ docs: - ^.*.nr$ dependencies: - yarn-project + +noir: + buildDir: noir diff --git a/noir/Dockerfile b/noir/Dockerfile new file mode 100644 index 00000000000..746efbe64c5 --- /dev/null +++ b/noir/Dockerfile @@ -0,0 +1,15 @@ +FROM rust:alpine3.17 +RUN apk update \ + && apk upgrade \ + && apk add --no-cache \ + build-base +WORKDIR /usr/src/noir +COPY . . +RUN GIT_DIRTY=false \ + GIT_COMMIT=$(awk '/commit =/ {print $3}' .gitrepo) \ + cargo build --features="noirc_frontend/aztec" --release + +FROM alpine:3.17 +COPY --from=0 /usr/src/noir/target/release/nargo /usr/bin/nargo +WORKDIR /project +ENTRYPOINT ["/usr/bin/nargo"] \ No newline at end of file diff --git a/noir/Dockerfile.dockerignore b/noir/Dockerfile.dockerignore new file mode 100644 index 00000000000..521e73ea5bf --- /dev/null +++ b/noir/Dockerfile.dockerignore @@ -0,0 +1,2 @@ +target +Dockerfile* \ No newline at end of file diff --git a/release-please-config.json b/release-please-config.json index fa44df2381a..09f9e248f7e 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -19,8 +19,7 @@ ".": { "release-type": "simple", "component": "aztec-packages", - "package-name": "aztec-packages", - "extra-files": ["VERSION"] + "package-name": "aztec-packages" }, "barretenberg/ts": { "release-type": "node", @@ -31,12 +30,7 @@ "release-type": "simple", "component": "barretenberg", "package-name": "barretenberg", - "extra-files": [ - "cpp/CMakeLists.txt", - "VERSION", - "barretenberg.nix", - "barretenberg-wasm.nix" - ] + "extra-files": ["cpp/CMakeLists.txt"] } }, "plugins": [ diff --git a/yarn-project/boxes/run_tests b/yarn-project/boxes/run_tests index c645f888b67..24a089b203c 100755 --- a/yarn-project/boxes/run_tests +++ b/yarn-project/boxes/run_tests @@ -23,41 +23,39 @@ YARN_PROJECT=yarn-project SANDBOX_IMAGE_URI=$(calculate_image_uri $SANDBOX) YP_IMAGE_URI=$(calculate_image_uri $YARN_PROJECT) -################### CHECK REBUILD ################### +################### CHECK REBUILD ################### # Before doing any work we check if we need to rebuild # Each box is suffixed with the box name, such that rebuilding of each box is independent, if one fails we don't rebuild the others ensure_repo $SANDBOX $ECR_REGION refresh_lifecycle -CONTENT_HASH=$(calculate_content_hash $SANDBOX) -BASE_TAG=cache-$CONTENT_HASH-$BOX_NAME # Append the box name to separate the tag +BASE_TAG=$(calculate_image_tag $SANDBOX) SUCCESS_TAG=$BASE_TAG -echo "Content hash: $CONTENT_HASH" +echo "Success tag: $SUCCESS_TAG" if check_rebuild $SUCCESS_TAG $SANDBOX; then echo "No rebuild required." retry tag_remote_image $SANDBOX $BASE_TAG $SUCCESS_TAG exit 0 fi -################### PREAMBLE ################### +################### PREAMBLE ################### # Pull images from ecr and retag for the docker compose -SANDBOX_IMAGE=$SANDBOX_IMAGE_URI-x86_64 -echo "pulling docker image for $SANDBOX $SANDBOX_IMAGE" -retry docker pull $SANDBOX_IMAGE -retry docker tag $SANDBOX_IMAGE aztecprotocol/$SANDBOX:latest +echo "pulling docker image for $SANDBOX $SANDBOX_IMAGE_URI" +retry docker pull $SANDBOX_IMAGE_URI +retry docker tag $SANDBOX_IMAGE_URI aztecprotocol/$SANDBOX:latest echo "pulling docker image for $YARN_PROJECT $YP_IMAGE_URI" retry docker pull $YP_IMAGE_URI retry docker tag $YP_IMAGE_URI aztecprotocol/$YARN_PROJECT:latest -################### RUN TESTS ################### +################### RUN TESTS ################### docker-compose rm -f docker-compose -f $COMPOSE_FILE up --exit-code-from boxes-$BOX_NAME -################### POST TEST ################### +################### POST TEST ################### # Success - push a new tag for the commit hash with the box name appended IMAGE_COMMIT_URI=$SANDBOX_IMAGE_URI-$BOX_NAME -retry docker tag $SANDBOX_IMAGE $IMAGE_COMMIT_URI +retry docker tag $SANDBOX_IMAGE_URI $IMAGE_COMMIT_URI retry docker push $IMAGE_COMMIT_URI > /dev/null 2>&1 \ No newline at end of file diff --git a/yarn-project/deploy_dockerhub.sh b/yarn-project/deploy_dockerhub.sh deleted file mode 100755 index c8fe2ea0f77..00000000000 --- a/yarn-project/deploy_dockerhub.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -DIST_TAG=${1:-"latest"} - -extract_repo yarn-project /usr/src project -PROJECT_ROOT=$(pwd)/project/src/ - -for REPOSITORY in "aztec-sandbox" "cli"; do - echo "Deploying $REPOSITORY $DIST_TAG" - RELATIVE_PROJECT_DIR=$(query_manifest relativeProjectDir $REPOSITORY) - cd "$PROJECT_ROOT/$RELATIVE_PROJECT_DIR" - - deploy_dockerhub $REPOSITORY x86_64 $DIST_TAG - deploy_dockerhub $REPOSITORY arm64 $DIST_TAG - create_dockerhub_manifest $REPOSITORY x86_64,arm64 $DIST_TAG -done diff --git a/yarn-project/deploy_npm.sh b/yarn-project/deploy_npm.sh index a66c538e2c2..20575d12167 100755 --- a/yarn-project/deploy_npm.sh +++ b/yarn-project/deploy_npm.sh @@ -52,7 +52,9 @@ function deploy_package() { fi # Publish - if [ -n "${COMMIT_TAG:-}" ]; then + if [ "$DRY_DEPLOY" -eq 1 ]; then + npm publish --dry-run $TAG_ARG --access public + else # Check if version exists if npm view "$PACKAGE_NAME@$VERSION" version >/dev/null 2>&1; then # Tag the existing version @@ -61,8 +63,6 @@ function deploy_package() { # Publish new version npm publish $TAG_ARG --access public fi - else - npm publish --dry-run $TAG_ARG --access public fi # Back to root