diff --git a/.circleci/config.yml b/.circleci/config.yml index 72edbd995fa..51c2166c23d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -85,7 +85,7 @@ jobs: - *setup_env - run: name: "Build" - command: cond_spot_run_build barretenberg-wasm-linux-clang 64 + command: cond_spot_run_build barretenberg-wasm-linux-clang 128 barretenberg-x86_64-linux-gcc: docker: @@ -96,7 +96,7 @@ jobs: - *setup_env - run: name: "Build" - command: cond_spot_run_build barretenberg-x86_64-linux-gcc 64 + command: cond_spot_run_build barretenberg-x86_64-linux-gcc 128 barretenberg-x86_64-linux-clang: docker: @@ -107,7 +107,7 @@ jobs: - *setup_env - run: name: "Build" - command: cond_spot_run_build barretenberg-x86_64-linux-clang 64 + command: cond_spot_run_build barretenberg-x86_64-linux-clang 128 barretenberg-x86_64-linux-clang-fuzzing: docker: @@ -118,7 +118,7 @@ jobs: - *setup_env - run: name: "Build" - command: cond_spot_run_build barretenberg-x86_64-linux-clang-fuzzing 64 + command: cond_spot_run_build barretenberg-x86_64-linux-clang-fuzzing 128 barretenberg-x86_64-linux-clang-assert: docker: @@ -129,7 +129,7 @@ jobs: - *setup_env - run: name: "Build" - command: cond_spot_run_build barretenberg-x86_64-linux-clang-assert 64 + command: cond_spot_run_build barretenberg-x86_64-linux-clang-assert 128 barretenberg-stdlib-tests: docker: @@ -293,7 +293,7 @@ jobs: - *setup_env - run: name: "Build" - command: cond_spot_run_build circuits-wasm-linux-clang 64 + command: cond_spot_run_build circuits-wasm-linux-clang 32 circuits-wasm-linux-clang-assert: docker: @@ -304,7 +304,7 @@ jobs: - *setup_env - run: name: "Build" - command: cond_spot_run_build circuits-wasm-linux-clang-assert 64 + command: cond_spot_run_build circuits-wasm-linux-clang-assert 32 circuits-x86_64-linux-clang-tidy: docker: @@ -315,7 +315,7 @@ jobs: - *setup_env - run: name: "Build" - command: cond_spot_run_build circuits-x86_64-linux-clang-tidy 64 + command: cond_spot_run_build circuits-x86_64-linux-clang-tidy 32 circuits-x86_64-linux-clang: docker: @@ -326,7 +326,7 @@ jobs: - *setup_env - run: name: "Build" - command: cond_spot_run_build circuits-x86_64-linux-clang 64 + command: cond_spot_run_build circuits-x86_64-linux-clang 32 circuits-x86_64-linux-clang-assert: docker: @@ -337,7 +337,7 @@ jobs: - *setup_env - run: name: "Build" - command: cond_spot_run_build circuits-x86_64-linux-clang-assert 64 + command: cond_spot_run_build circuits-x86_64-linux-clang-assert 32 circuits-wasm-tests: docker: @@ -377,6 +377,7 @@ jobs: noir-contracts-build: machine: image: ubuntu-2204:2023.07.2 + resource_class: large steps: - *checkout - *setup_env @@ -387,7 +388,7 @@ jobs: yarn-project-base: machine: image: ubuntu-2204:2023.07.2 - resource_class: large + resource_class: xlarge steps: - *checkout - *setup_env @@ -396,6 +397,28 @@ jobs: command: build yarn-project-base | add_timestamps yarn-project: + machine: + image: ubuntu-2204:2023.07.2 + resource_class: large + steps: + - *checkout + - *setup_env + - run: + name: Build + command: build yarn-project | add_timestamps + + yarn-project-format: + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small + steps: + - *checkout + - *setup_env + - run: + name: Build + command: cond_spot_run_script yarn-project 32 cond_run_container yarn-project formatting | add_timestamps + + yarn-project-test: docker: - image: aztecprotocol/alpine-build-image resource_class: small @@ -404,7 +427,7 @@ jobs: - *setup_env - run: name: Build - command: cond_spot_run_build yarn-project 64 | add_timestamps + command: cond_spot_run_script yarn-project 64 cond_run_container yarn-project test | add_timestamps aztec-sandbox-base: machine: @@ -556,6 +579,7 @@ jobs: e2e-sandbox-example: machine: image: ubuntu-2204:2023.07.2 + resource_class: large steps: - *checkout - *setup_env @@ -566,6 +590,7 @@ jobs: e2e-multi-transfer-contract: machine: image: ubuntu-2204:2023.07.2 + resource_class: large steps: - *checkout - *setup_env @@ -653,6 +678,7 @@ jobs: e2e-escrow-contract: machine: image: ubuntu-2204:2023.07.2 + resource_class: large steps: - *checkout - *setup_env @@ -740,6 +766,7 @@ jobs: e2e-p2p: machine: image: ubuntu-2204:2023.07.2 + resource_class: large steps: - *checkout - *setup_env @@ -750,6 +777,7 @@ jobs: e2e-browser-sandbox: machine: image: ubuntu-2204:2023.07.2 + resource_class: large steps: - *checkout - *setup_env @@ -760,6 +788,7 @@ jobs: e2e-card-game: machine: image: ubuntu-2204:2023.07.2 + resource_class: large steps: - *checkout - *setup_env @@ -770,6 +799,7 @@ jobs: aztec-rpc-sandbox: machine: image: ubuntu-2204:2023.07.2 + resource_class: large steps: - *checkout - *setup_env @@ -780,6 +810,7 @@ jobs: cli-docs-sandbox: machine: image: ubuntu-2204:2023.07.2 + resource_class: large steps: - *checkout - *setup_env @@ -801,6 +832,7 @@ jobs: guides-dapp-testing: machine: image: ubuntu-2204:2023.07.2 + resource_class: large steps: - *checkout - *setup_env @@ -811,6 +843,7 @@ jobs: guides-sample-dapp: machine: image: ubuntu-2204:2023.07.2 + resource_class: large steps: - *checkout - *setup_env @@ -821,6 +854,7 @@ jobs: guides-up-quick-start: machine: image: ubuntu-2204:2023.07.2 + resource_class: large steps: - *checkout - *setup_env @@ -831,6 +865,7 @@ jobs: e2e-canary-test: machine: image: ubuntu-2204:2023.07.2 + resource_class: large steps: - *checkout - *setup_env @@ -1063,6 +1098,14 @@ workflows: - circuits-wasm-linux-clang - l1-contracts <<: *defaults + - yarn-project-format: + requires: + - yarn-project + <<: *defaults + - yarn-project-test: + requires: + - yarn-project + <<: *defaults - yarn-project: requires: - yarn-project-base diff --git a/.gitmodules b/.gitmodules index e355139eb3d..f643e34f4ca 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ -[submodule "legacy-barretenberg-build-system"] - path = barretenberg/build-system - url = https://github.com/AztecProtocol/build-system [submodule "l1-contracts/lib/openzeppelin-contracts"] path = l1-contracts/lib/openzeppelin-contracts url = https://github.com/openzeppelin/openzeppelin-contracts diff --git a/barretenberg/.circleci/config.yml b/barretenberg/.circleci/config.yml deleted file mode 100644 index 0436b49b804..00000000000 --- a/barretenberg/.circleci/config.yml +++ /dev/null @@ -1,438 +0,0 @@ -# This file uses YAML anchors and aliases to prevent repetition of blocks of config: -# https://support.atlassian.com/bitbucket-cloud/docs/yaml-anchors/ -# -# Two primary anchors are checkout and setup_env, called as the first step of almost all jobs: -# - checkout: A custom checkout step to reduce the amount of data downloaded to improve speed. -# - setup_env: Sets up the common environment used by all build steps. -# -# Two CCI executors are used: -# - docker (small): Used only to launch external EC2 instances for big workloads. It's the cheapest option. -# - machine (large): Used for building in CCI itself. 4cpus, 15GB has the optimal power/cost ratio. -# -# The docker executor uses a custom image build in `build_image`. It's specifically streamlined for fast download -# with just enough tools to execute the build system, and launch EC2 instances etc. - -version: 2.1 - -orbs: - slack: circleci/slack@4.12.1 - -parameters: - workflow: - type: string - default: "system" - -# This build step checks out the code from the repository. It has a hardcoded readonly key to allow the checkout. -# Initially it just fetches the repo metadata for the current commit hash to a depth of 50 commits. -# We need historical commit hashes to calculate diffs between previous and current commits. -# It then checks out the fetched head to actually download the data. -checkout: &checkout - run: - name: "Checkout code" - command: | - cd $HOME - mkdir -p .ssh - chmod 0700 .ssh - ssh-keyscan -t rsa github.com >> .ssh/known_hosts - - # IF YOU'RE CHANGING THIS, YOU ALSO WANT TO CHANGE: build-system/remote_build/remote_build - # Shallow checkout this commit. - mkdir -p project - cd project - git init - git remote add origin $CIRCLE_REPOSITORY_URL - # Only download metadata when fetching. - git fetch --depth 50 --filter=blob:none origin $CIRCLE_SHA1 - git checkout FETCH_HEAD - # Initialize submodules recursively - git submodule update --init --recursive - -# Called setup_env to setup a bunch of global variables used throughout the rest of the build process. -# It takes the required CCI environment variables as inputs, and gives them normalised names for the rest of -# the build process. This enables easy running of the build system external to CCI, as used for powerful EC2 builds. -setup_env: &setup_env - run: - name: "Setup environment" - command: ./build-system/scripts/setup_env "$CIRCLE_SHA1" "$CIRCLE_TAG" "$CIRCLE_JOB" "$CIRCLE_REPOSITORY_URL" "$CIRCLE_BRANCH" - -# setup_aztec_commit: &setup_aztec_commit -# run: -# name: "Setup environment for Aztec Integration Testing" -# command: | -# # Load the aztec commit into env for use in integration tests -# # Aztec commit will be chosen according to the following priorities: -# # 1. if present, grab commit from cpp/.aztec-packages-commit file -# # 2. if this is a branch with an identically named branch in aztec-pacakages, run against that branch -# # 3. default to aztec-packages 'master' - -# # default to master -# AZTEC_COMMIT=master -# echo "Checking if a branch/commit is specified in .aztec-packages-commit" -# echo "or if this is a branch with a sibling branch in aztec-packages." -# echo "Otherwise defaulting to test against circuits in aztec-packages 'master'." - -# COMMIT_FROM_FILE=$(cat cpp/.aztec-packages-commit 2>/dev/null || true) -# if [ -n "$COMMIT_FROM_FILE" ] && [[ "$COMMIT_FROM_FILE" = *[![:space:]]* ]]; then -# # if a commit is present in .aztec-packages-commit, use that -# # check that there is text in the commit file that is not just whitespace -# echo "Using aztec-packages commit ($COMMIT_FROM_FILE) from .aztec-packages-commit file" -# AZTEC_COMMIT=$COMMIT_FROM_FILE -# elif [ -n "$CIRCLE_BRANCH" ] && [ "$CIRCLE_BRANCH" != master ]; then -# echo "Checking if a sibling branch ($CIRCLE_BRANCH) exists in aztec-packages" -# if git ls-remote --heads git@github.com:AztecProtocol/aztec-packages.git 2>/dev/null | grep "refs/heads/$CIRCLE_BRANCH" &>/dev/null; then -# echo "Sibling branch ($CIRCLE_BRANCH) exists in aztec-packages! Circuits tests will run against that branch." -# AZTEC_COMMIT=$CIRCLE_BRANCH -# else -# echo "No sibling branch exists in aztec-packages. Defaulting to 'master'." -# fi -# else -# echo "Defaulting to 'master'." -# fi - -# echo "export AZTEC_COMMIT=$AZTEC_COMMIT" >> "$BASH_ENV" -# source "$BASH_ENV" - -# # This step is used to save logs from various barretenberg test to the workspace so that they can be used later to parse benchmark values out of them -# # NOTE(AD): This is now obsolete as benchmarking has been moved to aztec-packages repo. Another pass may simplify this. -# save_logs: &save_logs -# persist_to_workspace: -# root: /tmp/test-logs -# paths: -# - ./* - -jobs: - # wasm-linux-clang: - # docker: - # - image: aztecprotocol/alpine-build-image - # resource_class: small - # steps: - # - *checkout - # - *setup_env - # - run: - # name: "Build" - # command: cond_spot_run_build barretenberg-wasm-linux-clang 64 - - # x86_64-linux-gcc: - # docker: - # - image: aztecprotocol/alpine-build-image - # resource_class: small - # steps: - # - *checkout - # - *setup_env - # - run: - # name: "Build" - # command: cond_spot_run_build barretenberg-x86_64-linux-gcc 64 - - x86_64-linux-clang: - docker: - - image: aztecprotocol/alpine-build-image - resource_class: small - steps: - - *checkout - - *setup_env - - run: - name: "Build" - command: cond_spot_run_build barretenberg-x86_64-linux-clang 64 - - # x86_64-linux-clang-assert: - # docker: - # - image: aztecprotocol/alpine-build-image - # resource_class: small - # steps: - # - *checkout - # - *setup_env - # - run: - # name: "Build" - # command: cond_spot_run_build barretenberg-x86_64-linux-clang-assert 64 - - x86_64-linux-clang-fuzzing: - docker: - - image: aztecprotocol/alpine-build-image - resource_class: small - steps: - - *checkout - - *setup_env - - run: - name: "Build" - command: cond_spot_run_build barretenberg-x86_64-linux-clang-fuzzing 64 - - # stdlib-tests: - # docker: - # - image: aztecprotocol/alpine-build-image - # resource_class: small - # steps: - # - *checkout - # - *setup_env - # - run: - # name: "Test" - # command: cond_spot_run_tests barretenberg-x86_64-linux-clang-assert 1 stdlib-tests - # - *save_logs - - # dsl-tests: - # docker: - # - image: aztecprotocol/alpine-build-image - # resource_class: small - # steps: - # - *checkout - # - *setup_env - # - run: - # name: "Test" - # command: cond_spot_run_tests barretenberg-x86_64-linux-clang-assert 1 dsl_tests - # - *save_logs - - # barretenberg-tests: - # docker: - # - image: aztecprotocol/alpine-build-image - # resource_class: small - # steps: - # - *checkout - # - *setup_env - # - run: - # name: "Test" - # command: cond_spot_run_test_script ./scripts/bb-tests.sh barretenberg-x86_64-linux-clang-assert - # - *save_logs - - # honk-tests: - # docker: - # - image: aztecprotocol/alpine-build-image - # resource_class: small - # steps: - # - *checkout - # - *setup_env - # - run: - # name: "Test" - # command: cond_spot_run_tests barretenberg-x86_64-linux-clang-assert 1 honk_tests - # - *save_logs - - # proof-system-tests: - # docker: - # - image: aztecprotocol/alpine-build-image - # resource_class: small - # steps: - # - *checkout - # - *setup_env - # - run: - # name: "Test" - # command: cond_spot_run_tests barretenberg-x86_64-linux-clang-assert 1 proof_system_tests - # - *save_logs - - # stdlib-recursion-turbo-tests: - # docker: - # - image: aztecprotocol/alpine-build-image - # resource_class: small - # steps: - # - *checkout - # - *setup_env - # - run: - # name: "Test" - # command: cond_spot_run_tests barretenberg-x86_64-linux-clang-assert 1 stdlib_recursion_tests --gtest_filter=*turbo* - # - *save_logs - - # stdlib-recursion-ultra-tests: - # docker: - # - image: aztecprotocol/alpine-build-image - # resource_class: small - # steps: - # - *checkout - # - *setup_env - # - run: - # name: "Test" - # command: cond_spot_run_tests barretenberg-x86_64-linux-clang-assert 3 stdlib_recursion_tests --gtest_filter=-*turbo* - # - *save_logs - - # join-split-tests: - # docker: - # - image: aztecprotocol/alpine-build-image - # resource_class: small - # steps: - # - *checkout - # - *setup_env - # - run: - # name: "Test" - # command: cond_spot_run_tests barretenberg-x86_64-linux-clang-assert 3 join_split_example_proofs_join_split_tests --gtest_filter=-*full_proof* - # - *save_logs - - # # bb-bin-tests: - # # docker: - # # - image: aztecprotocol/alpine-build-image - # # resource_class: small - # # steps: - # # - *checkout - # # - *setup_env - # # - run: - # # name: "Test" - # # command: cond_spot_run_test_script ./scripts/bin-test.sh barretenberg-x86_64-linux-clang-assert - - # bb-js: - # machine: - # image: ubuntu-2004:202010-01 - # resource_class: large - # steps: - # - *checkout - # - *setup_env - # - run: - # name: "Build" - # command: build bb.js - - # bb-js-tests: - # docker: - # - image: aztecprotocol/alpine-build-image - # resource_class: small - # steps: - # - *checkout - # - *setup_env - # - run: - # name: "Test" - # command: cond_spot_run_tests bb.js - - # acir-tests: - # docker: - # - image: aztecprotocol/alpine-build-image - # resource_class: small - # steps: - # - *checkout - # - *setup_env - # - run: - # name: "Build and test" - # command: cond_spot_run_build acir-tests 32 - - # # These machines cost a fortune (10x other machines). - # # There is a branch coming for github actions that will build mac stuff on releases. - # # I'll leave this here for now, just in case proves useful to be able to do CI mac builds on branches, - # # but let's not uncomment in master. - # # - # # aarch64-macos-clang: - # # macos: - # # xcode: "14.3.1" - # # resource_class: macos.m1.medium.gen1 - # # steps: - # # - *checkout - # # - run: - # # name: "Install toolchain" - # # command: brew install cmake ninja - # # - run: - # # name: "Build" - # # working_directory: ~/project/cpp - # # command: cmake --preset default && cmake --build --preset default --target bb - # # - run: - # # name: "Test" - # # working_directory: ~/project/cpp/bin-test - # # command: ./bin-test.sh - - # ################################### - # # Aztec integration tests - # #circuits-wasm-linux-clang-builder-runner: - # # docker: - # # - image: aztecprotocol/alpine-build-image - # # resource_class: small - # # steps: - # # - *checkout - # # - *setup_env - # # - run: - # # name: "Build" - # # command: cond_spot_run_build barretenberg-circuits-wasm-linux-clang-builder-runner 64 - - # circuits-x86_64-linux-clang-builder-runner: - # docker: - # - image: aztecprotocol/alpine-build-image - # resource_class: small - # steps: - # - *checkout - # - *setup_env - # - run: - # name: "Build" - # command: cond_spot_run_build barretenberg-circuits-x86_64-linux-clang-builder-runner 64 - - # #circuits-wasm-tests: - # # docker: - # # - image: aztecprotocol/alpine-build-image - # # resource_class: small - # # steps: - # # - *checkout - # # - *setup_env - # # - *setup_aztec_commit - # # - run: - # # name: "Build" - # # command: cond_spot_run_test_script ./scripts/run_aztec_circuits_tests barretenberg-circuits-wasm-linux-clang-builder-runner "$AZTEC_COMMIT" 1 wasm scripts/a3-tests -*.skip*:*.circuit* - - # circuits-x86_64-tests: - # docker: - # - image: aztecprotocol/alpine-build-image - # resource_class: small - # steps: - # - *checkout - # - *setup_env - # - *setup_aztec_commit - # - run: - # name: "Build" - # command: cond_spot_run_test_script ./scripts/run_aztec_circuits_tests barretenberg-circuits-x86_64-linux-clang-builder-runner "$AZTEC_COMMIT" 1 x86_64 scripts/a3-tests -*.skip* - # # End Aztec integration tests - # ################################### - -# Repeatable config for defining the workflow below. -tag_regex: &tag_regex /v[0-9]+(\.[0-9]+)*(-[a-zA-Z-]+\.[0-9]+)?/ -defaults: &defaults - filters: - tags: - only: *tag_regex - context: - - slack - post-steps: - - slack/notify: - channel: $SLACK_BARRETENBERG_CHANNEL - event: fail - branch_pattern: "master" - -# bb_test: &bb_test -# requires: -# - x86_64-linux-clang-assert -# <<: *defaults - -# notmaster: ¬master -# filters: -# branches: -# ignore: -# - master -# <<: *defaults - -workflows: - system: - jobs: - # - aarch64-macos-clang: *defaults - # - x86_64-linux-gcc: *defaults - - x86_64-linux-clang: *defaults - # - x86_64-linux-clang-assert: *defaults - - x86_64-linux-clang-fuzzing: *defaults - # - wasm-linux-clang: *defaults - # - proof-system-tests: *bb_test - # - honk-tests: *bb_test - # - dsl-tests: *bb_test - # - barretenberg-tests: *bb_test - # - stdlib-tests: *bb_test - # - stdlib-recursion-turbo-tests: *bb_test - # - stdlib-recursion-ultra-tests: *bb_test - # - join-split-tests: *bb_test - # - bb-bin-tests: *bb_test - # - bb-js: - # requires: - # - wasm-linux-clang - # <<: *defaults - # - bb-js-tests: - # requires: - # - bb-js - # <<: *defaults - # - acir-tests: - # requires: - # - x86_64-linux-clang-assert - # - bb-js - # <<: *defaults - #- circuits-wasm-linux-clang-builder-runner: *notmaster - # - circuits-x86_64-linux-clang-builder-runner: *notmaster - #- circuits-wasm-tests: - # requires: - # - circuits-wasm-linux-clang-builder-runner - # <<: *notmaster - # - circuits-x86_64-tests: - # requires: - # - circuits-x86_64-linux-clang-builder-runner - # <<: *notmaster diff --git a/barretenberg/PROJECT b/barretenberg/PROJECT deleted file mode 100644 index adcaa3c7f54..00000000000 --- a/barretenberg/PROJECT +++ /dev/null @@ -1 +0,0 @@ -barretenberg diff --git a/barretenberg/barretenberg-wasm.nix b/barretenberg/barretenberg-wasm.nix deleted file mode 100644 index 7440f5bb6ac..00000000000 --- a/barretenberg/barretenberg-wasm.nix +++ /dev/null @@ -1,38 +0,0 @@ -{ stdenv, cmake, ninja, binaryen, callPackage }: -let - toolchain_file = ./cpp/cmake/toolchains/wasm32-wasi.cmake; - wasi-sdk = callPackage ./wasi-sdk.nix { }; -in -stdenv.mkDerivation -{ - pname = "barretenberg.wasm"; - version = "0.7.10"; # x-release-please-version - - src = ./cpp; - - nativeBuildInputs = [ cmake ninja wasi-sdk ]; - - buildInputs = [ ]; - - cmakeFlags = [ - "-GNinja" - "-DTESTING=OFF" - "-DBENCHMARKS=OFF" - "-DCMAKE_TOOLCHAIN_FILE=${toolchain_file}" - "-DCMAKE_C_COMPILER=${wasi-sdk}/bin/clang" - "-DCMAKE_CXX_COMPILER=${wasi-sdk}/bin/clang++" - "-DCMAKE_AR=${wasi-sdk}/bin/llvm-ar" - "-DCMAKE_RANLIB=${wasi-sdk}/bin/llvm-ranlib" - "-DCMAKE_SYSROOT=${wasi-sdk}/share/wasi-sysroot" - "-DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER" - "-DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY" - "-DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY" - "-DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=ONLY" - "-DCMAKE_C_COMPILER_WORKS=ON" - "-DCMAKE_CXX_COMPILER_WORKS=ON" - ]; - - buildPhase = '' - cmake --build . --target barretenberg.wasm --parallel - ''; -} diff --git a/barretenberg/barretenberg.nix b/barretenberg/barretenberg.nix deleted file mode 100644 index 04576b20335..00000000000 --- a/barretenberg/barretenberg.nix +++ /dev/null @@ -1,38 +0,0 @@ -{ overrideCC, stdenv, llvmPackages, cmake, ninja, lib, callPackage, gcc11 }: -let - targetPlatform = stdenv.targetPlatform; - buildEnv = - if (stdenv.targetPlatform.isGnu && stdenv.targetPlatform.isAarch64) then - # As per https://discourse.nixos.org/t/gcc11stdenv-and-clang/17734/7 since it seems that aarch64-linux uses - # gcc9 instead of gcc11 for the C++ stdlib, while all other targets we support provide the correct libstdc++ - overrideCC llvmPackages.stdenv (llvmPackages.clang.override { gccForLibs = gcc11.cc; }) - else - llvmPackages.stdenv; - optionals = lib.lists.optionals; - toolchain_file = ./cpp/cmake/toolchains/${targetPlatform.system}.cmake; -in -buildEnv.mkDerivation -{ - pname = "libbarretenberg"; - version = "0.7.10"; # x-release-please-version - - src = ./cpp; - - nativeBuildInputs = [ cmake ninja ]; - - buildInputs = [ llvmPackages.openmp ]; - - cmakeFlags = [ - "-DTESTING=OFF" - "-DBENCHMARKS=OFF" - "-DDISABLE_ASM=ON" - "-DDISABLE_ADX=ON" - "-DCMAKE_TOOLCHAIN_FILE=${toolchain_file}" - "-DCMAKE_BUILD_TYPE=RelWithAssert" - ]; - - NIX_CFLAGS_COMPILE = - optionals targetPlatform.isDarwin [ " -fno-aligned-allocation" ]; - - enableParallelBuilding = true; -} diff --git a/barretenberg/bootstrap_docker.sh b/barretenberg/bootstrap_docker.sh deleted file mode 100755 index e2f88e22525..00000000000 --- a/barretenberg/bootstrap_docker.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -# This script builds the projects listed in build_mainifest.sh. - -set -eu - -COMMIT_HASH=$(git rev-parse HEAD) -source ./build-system/scripts/setup_env $COMMIT_HASH '' mainframe_$USER $(git rev-parse --show-toplevel) -build_local diff --git a/barretenberg/build-system b/barretenberg/build-system deleted file mode 160000 index a109f3aef28..00000000000 --- a/barretenberg/build-system +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a109f3aef28cea4a50481cdf2d74fc3909212c0b diff --git a/barretenberg/build_manifest.json b/barretenberg/build_manifest.json deleted file mode 100644 index 564626daca9..00000000000 --- a/barretenberg/build_manifest.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "barretenberg-x86_64-linux-clang": { - "buildDir": "cpp", - "dockerfile": "dockerfiles/Dockerfile.x86_64-linux-clang", - "rebuildPatterns": ["^cpp/"], - "dependencies": [] - }, - "barretenberg-x86_64-linux-clang-assert": { - "buildDir": "cpp", - "dockerfile": "dockerfiles/Dockerfile.x86_64-linux-clang-assert", - "rebuildPatterns": ["^cpp/"], - "dependencies": [] - }, - "barretenberg-x86_64-linux-clang-fuzzing": { - "buildDir": "cpp", - "dockerfile": "dockerfiles/Dockerfile.x86_64-linux-clang-fuzzing", - "rebuildPatterns": ["^cpp/"], - "dependencies": [] - }, - "barretenberg-x86_64-linux-gcc": { - "buildDir": "cpp", - "dockerfile": "dockerfiles/Dockerfile.x86_64-linux-gcc", - "rebuildPatterns": ["^cpp/"], - "dependencies": [] - }, - "barretenberg-wasm-linux-clang": { - "buildDir": "cpp", - "dockerfile": "dockerfiles/Dockerfile.wasm-linux-clang", - "rebuildPatterns": ["^cpp/"], - "dependencies": [] - }, - "barretenberg-circuits-x86_64-linux-clang-builder-runner": { - "buildDir": "cpp", - "dockerfile": "dockerfiles/Dockerfile.circuits-x86_64-linux-clang-builder-runner", - "rebuildPatterns": ["^cpp/"], - "dependencies": [] - }, - "barretenberg-circuits-wasm-linux-clang-builder-runner": { - "buildDir": "cpp", - "dockerfile": "dockerfiles/Dockerfile.circuits-wasm-linux-clang-builder-runner", - "rebuildPatterns": ["^cpp/"], - "dependencies": [] - }, - "bb.js": { - "buildDir": "ts", - "rebuildPatterns": ["^ts/"], - "dependencies": ["barretenberg-wasm-linux-clang"] - }, - "acir-tests": { - "buildDir": "acir_tests", - "rebuildPatterns": ["^acir_tests/"], - "dependencies": ["bb.js", "barretenberg-x86_64-linux-clang-assert"] - } -} diff --git a/barretenberg/build_manifest.sh b/barretenberg/build_manifest.sh deleted file mode 100755 index 5768303cf86..00000000000 --- a/barretenberg/build_manifest.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -# Source this file to define the PROJECTS variable, needed by build_local, used by bootstrap_docker.sh. -# -# PROJECT elements have structure PROJECT_DIR_NAME:WORKING_DIR:DOCKERFILE:REPO. -# PROJECT_DIR_NAME: Should reflect the projects directory name. -# WORKING_DIR: Everything within this directory is copied into the docker context (excluding paths in .dockerignore). -# DOCKERFILE: Defaults to Dockerfile. However some projects have multiple build Dockerfiles located in subdirs. -# REPO: Defaults to . The docker repository name, used to name the resulting docker image. -# -# This file tells bootstrap_docker.sh which and in which order to build projects for locally testing the docker builds. -# To check *most* of the build works as expected, we can just do the minimum to produce the e2e tests, and run them -# locally to check they work. Other projects can be *temporarily* uncommented to test their Dockerfiles, but don't -# commit them, so that the most important build path remains fast and simple. - -PROJECTS=( - cpp:cpp:./dockerfiles/Dockerfile.wasm-linux-clang:barretenberg-wasm-linux-clang - cpp:cpp:./dockerfiles/Dockerfile.x86_64-linux-clang-assert:barretenberg-x86_64-linux-clang-assert - ts:ts:./Dockerfile:bb.js - acir_tests:acir_tests:./Dockerfile:acir_tests -) diff --git a/build-system/scripts/add_timestamps b/build-system/scripts/add_timestamps index 344cd569df4..4eeab6f56db 100755 --- a/build-system/scripts/add_timestamps +++ b/build-system/scripts/add_timestamps @@ -1,4 +1,4 @@ #!/bin/bash while IFS= read -r line; do echo "$(date '+%Y-%m-%d %H:%M:%S') $line" -done \ No newline at end of file +done diff --git a/build-system/scripts/build b/build-system/scripts/build index ac85ec318b8..132714b121e 100755 --- a/build-system/scripts/build +++ b/build-system/scripts/build @@ -58,7 +58,7 @@ function try_fetch_image() { return 0 } -docker_login +ecr_login # Ensure ECR repository exists. retry ensure_repo $REPOSITORY $ECR_REGION refresh_lifecycle diff --git a/build-system/scripts/check_rebuild b/build-system/scripts/check_rebuild index 1896ee21565..5762f47dbe7 100755 --- a/build-system/scripts/check_rebuild +++ b/build-system/scripts/check_rebuild @@ -9,8 +9,9 @@ REPOSITORY=$2 # If given nothing, then exit with failure to rebuild [ -n "$TAG" ] || exit 1 -# Requested a rebuild via [ci rebuild] commit message. -[ -z "${BUILD_SYSTEM_REBUILD:-}" ] || exit 1 +# Have we requested a rebuild via [ci rebuild] or [ci rebuild ] commit message? +[[ "$COMMIT_MESSAGE" == *"[ci rebuild $REPOSITORY]"* ]] && exit 1 +[[ "$COMMIT_MESSAGE" == *"[ci rebuild]"* ]] && exit 1 if ! image_exists $REPOSITORY $TAG; then echo "Rebuild required." diff --git a/build-system/scripts/cond_run_container b/build-system/scripts/cond_run_container new file mode 100755 index 00000000000..98562f1c379 --- /dev/null +++ b/build-system/scripts/cond_run_container @@ -0,0 +1,26 @@ +#!/bin/bash +# Conditionally runs a script if the REPOSITORY content hash has changed and we haven't had a successful run. +# +# Arguments are: +# 1. REPOSITORY: The project repository name in ECR. Used to determine if there are changes since last success. +# 2... ARGS: Script and arguments to run. +[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace +set -eu + +REPOSITORY=$1 +shift + +CONTENT_HASH=$(calculate_content_hash $REPOSITORY) +BASE_TAG=cache-$CONTENT_HASH +SUCCESS_TAG=$BASE_TAG-$JOB_NAME + +echo "Content hash: $CONTENT_HASH" + +if ! check_rebuild $SUCCESS_TAG $REPOSITORY; then + IMAGE_URI=$(calculate_image_uri $REPOSITORY) + # Login to pull our ecr images with docker. + ecr_login + retry docker pull $IMAGE_URI + docker run --rm -t $IMAGE_URI $@ + retry tag_remote_image $REPOSITORY $BASE_TAG $SUCCESS_TAG +fi diff --git a/build-system/scripts/create_dockerhub_manifest b/build-system/scripts/create_dockerhub_manifest index af23c5eedcc..2dfed1b512a 100755 --- a/build-system/scripts/create_dockerhub_manifest +++ b/build-system/scripts/create_dockerhub_manifest @@ -29,7 +29,7 @@ MANIFEST_DEPLOY_URI=$ACCOUNT/$REPOSITORY:$IMAGE_TAG MANIFEST_LATEST_URI=$ACCOUNT/$REPOSITORY:latest # Login to dockerhub. -docker_login +dockerhub_login export DOCKER_CLI_EXPERIMENTAL=enabled diff --git a/build-system/scripts/deploy_dockerhub b/build-system/scripts/deploy_dockerhub index a7655022c75..98fadb6c065 100755 --- a/build-system/scripts/deploy_dockerhub +++ b/build-system/scripts/deploy_dockerhub @@ -38,7 +38,7 @@ echo "Deploying to dockerhub: $IMAGE_DEPLOY_URI" retry ensure_repo $REPOSITORY $ECR_DEPLOY_REGION # Login to dockerhub. -docker_login +dockerhub_login echo "Pulling $IMAGE_COMMIT_URI" # Pull image. diff --git a/build-system/scripts/dockerhub_login b/build-system/scripts/dockerhub_login new file mode 100755 index 00000000000..97dc2177aa3 --- /dev/null +++ b/build-system/scripts/dockerhub_login @@ -0,0 +1,8 @@ +#!/bin/bash +set -eu +# Retries up to 3 times with 10 second intervals +for i in $(seq 1 3); do + echo "$DOCKERHUB_PASSWORD" | docker login -u aztecprotocolci --password-stdin && exit || sleep 10 +done +echo "$@ failed dockerhub_login after 3 attempts" +exit 1 diff --git a/build-system/scripts/ecr_login b/build-system/scripts/ecr_login new file mode 100755 index 00000000000..54a8e6d36da --- /dev/null +++ b/build-system/scripts/ecr_login @@ -0,0 +1,11 @@ +#!/bin/bash +set -eu +REGION=${1:-$ECR_REGION} +# Retries up to 3 times with 10 second intervals +for i in $(seq 1 3); do + aws ecr get-login-password --region $REGION \ + | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$REGION.amazonaws.com 2> /dev/null \ + && exit || sleep 10 +done +echo "$@ failed ecr_login after 3 attempts" +exit 1 diff --git a/build-system/scripts/ensure_repo b/build-system/scripts/ensure_repo index 56995700831..b767976354f 100755 --- a/build-system/scripts/ensure_repo +++ b/build-system/scripts/ensure_repo @@ -27,19 +27,8 @@ REPOSITORY=$1 REGION=$2 REFRESH_LIFECYCLE=${3:-} -function ecr_login() { - # Retries up to 3 times with 10 second intervals - for i in $(seq 1 3); do - aws ecr get-login-password --region $REGION \ - | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$REGION.amazonaws.com 2> /dev/null \ - && return || sleep 10 - done - echo "$@ failed docker_login after 3 attempts" - exit 1 -} - # Login to ECR. -ecr_login +ecr_login $REGION # Create the repository if it doesn't exist. if ! aws ecr describe-repositories --region $REGION --repository-names $REPOSITORY > /dev/null 2>&1; then diff --git a/build-system/scripts/image_exists b/build-system/scripts/image_exists index edf0e552ddc..0a4bee54d7a 100755 --- a/build-system/scripts/image_exists +++ b/build-system/scripts/image_exists @@ -1,3 +1,4 @@ #!/bin/bash +set -eu # Returns true if the given image exists in the current ECR. -aws ecr describe-images --region=$ECR_REGION --repository-name=$1 --image-ids=imageTag=$2 > /dev/null 2>&1 \ No newline at end of file +aws ecr describe-images --region=$ECR_REGION --repository-name=$1 --image-ids=imageTag=$2 > /dev/null 2>&1 diff --git a/build-system/scripts/setup_env b/build-system/scripts/setup_env index d6c71d7514e..9d37c3fefff 100755 --- a/build-system/scripts/setup_env +++ b/build-system/scripts/setup_env @@ -20,6 +20,7 @@ BUILD_SYSTEM_PATH=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) PROJECT=$(cat PROJECT) VERSION=$(cat VERSION) VERSION=${VERSION%% x-release-please-version*} +COMMIT_MESSAGE=`git log -n 1 --pretty=format:"%s" $COMMIT_HASH` echo "COMMIT_HASH=$COMMIT_HASH" echo "COMMIT_TAG=$COMMIT_TAG" @@ -80,6 +81,7 @@ echo export ECR_DEPLOY_URL=278380418400.dkr.ecr.eu-west-2.amazonaws.com >> $BASH echo export PROJECT=$PROJECT >> $BASH_ENV echo export COMMIT_HASH=$COMMIT_HASH >> $BASH_ENV echo export COMMIT_TAG=$COMMIT_TAG >> $BASH_ENV +echo "export COMMIT_MESSAGE=\"$COMMIT_MESSAGE\"" >> $BASH_ENV echo export JOB_NAME=$JOB_NAME >> $BASH_ENV echo export GIT_REPOSITORY_URL=$GIT_REPOSITORY_URL >> $BASH_ENV echo export VERSION_TAG=$VERSION_TAG >> $BASH_ENV @@ -110,14 +112,9 @@ if [ -n "$COMMIT_HASH" ]; then echo ${BUILD_INSTANCE_KEY:-} | base64 -d > ~/.ssh/build_instance_key chmod 600 ~/.ssh/build_instance_key - COMMIT_MESSAGE=`git log -n 1 --pretty=format:"%s" $COMMIT_HASH` - if [[ "$COMMIT_MESSAGE" == *"[ci debug]"* ]]; then echo export BUILD_SYSTEM_DEBUG=1 >> $BASH_ENV fi - if [[ "$COMMIT_MESSAGE" == *"[ci rebuild]"* ]]; then - echo export BUILD_SYSTEM_REBUILD=1 >> $BASH_ENV - fi fi # Print for visibility. diff --git a/yarn-project/Dockerfile b/yarn-project/Dockerfile index 665c05cc0df..ceac439a362 100644 --- a/yarn-project/Dockerfile +++ b/yarn-project/Dockerfile @@ -11,5 +11,5 @@ COPY --from=noir /usr/src/yarn-project/noir-contracts/target /usr/src/yarn-proje RUN cd /usr/src/yarn-project/noir-contracts && yarn build && ./scripts/types_all.sh && yarn build # Build the entire project and check formatting. -RUN yarn tsc -b && yarn formatting -RUN yarn test +RUN yarn tsc -b +ENTRYPOINT [ "yarn" ] diff --git a/yarn-project/Dockerfile.format b/yarn-project/Dockerfile.format new file mode 100644 index 00000000000..4a89d9bc188 --- /dev/null +++ b/yarn-project/Dockerfile.format @@ -0,0 +1,7 @@ +# Formats all our typescript packages. +FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project + +# Copy in the entire workspace. +COPY . . + +RUN yarn formatting diff --git a/yarn-project/Dockerfile.test b/yarn-project/Dockerfile.test new file mode 100644 index 00000000000..d6ee82b035d --- /dev/null +++ b/yarn-project/Dockerfile.test @@ -0,0 +1,7 @@ +# Runs all our typescript package non-e2e tests. +FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project + +# Copy in the entire workspace. +COPY . . + +RUN yarn test diff --git a/yarn-project/canary/scripts/run_tests b/yarn-project/canary/scripts/run_tests index ea52523134e..dd330d5fef2 100755 --- a/yarn-project/canary/scripts/run_tests +++ b/yarn-project/canary/scripts/run_tests @@ -13,19 +13,8 @@ if [ "$TEST" = "uniswap_trade_on_l1_from_l2.test.ts" ]; then export FORK_BLOCK_NUMBER=17514288 fi -function docker_login() { - # Retries up to 3 times with 10 second intervals - for i in $(seq 1 3); do - aws ecr get-login-password --region us-east-2 \ - | docker login --username AWS --password-stdin 278380418400.dkr.ecr.us-east-2.amazonaws.com \ - && return || sleep 10 - done - echo "$@ failed docker_login after 3 attempts" - exit 1 -} - -# Login to dockerhub. -docker_login +# Login to ECR. +ecr_login export PATH="$PATH:$(git rev-parse --show-toplevel)/build-system/scripts" IMAGE_URI=$(calculate_image_uri $IMAGE) diff --git a/yarn-project/end-to-end/scripts/run_tests_local b/yarn-project/end-to-end/scripts/run_tests_local index 1fcf1a3be26..c17d582371d 100755 --- a/yarn-project/end-to-end/scripts/run_tests_local +++ b/yarn-project/end-to-end/scripts/run_tests_local @@ -7,19 +7,8 @@ set -eu export TEST=$1 export COMPOSE_FILE=${2:-./scripts/docker-compose.yml} -function docker_login() { - # Retries up to 3 times with 10 second intervals - for i in $(seq 1 3); do - aws ecr get-login-password --region us-east-2 \ - | docker login --username AWS --password-stdin 278380418400.dkr.ecr.us-east-2.amazonaws.com \ - && return || sleep 10 - done - echo "$@ failed docker_login after 3 attempts" - exit 1 -} - -# Login to dockerhub. -docker_login +# Login to ECR. +ecr_login export PATH="$PATH:$(git rev-parse --show-toplevel)/build-system/scripts" diff --git a/yarn-project/package.json b/yarn-project/package.json index ccde2e9962f..411d94921a9 100644 --- a/yarn-project/package.json +++ b/yarn-project/package.json @@ -6,11 +6,11 @@ "prepare": "node ./yarn-project-base/scripts/update_package_jsons.mjs && yarn workspaces foreach run prepare && workspaces-to-typescript-project-references --tsconfigPath tsconfig.json && prettier -w */tsconfig.json", "prepare:check": "node ./yarn-project-base/scripts/update_package_jsons.mjs --check && workspaces-to-typescript-project-references --check --tsconfigPath tsconfig.json", "docs": "typedoc --out docs/dist && cd docs && yarn serve", - "formatting": "yarn workspaces foreach -p -v run formatting", + "formatting": "yarn workspaces foreach -p -j unlimited -v run formatting", "formatting:fix": "yarn workspaces foreach -p -v run formatting:fix", "lint": "yarn eslint --cache --ignore-pattern l1-artifacts .", "format": "yarn prettier --cache -w .", - "test": "yarn workspaces foreach --exclude @aztec/aztec3-packages --exclude @aztec/end-to-end --exclude @aztec/canary --exclude private-token -p -v run test", + "test": "yarn workspaces foreach --exclude @aztec/aztec3-packages --exclude @aztec/end-to-end --exclude @aztec/canary --exclude private-token -p -j unlimited -v run test", "build": "yarn workspace @aztec/l1-artifacts build && tsc -b tsconfig.json", "build:dev": "yarn workspace @aztec/l1-artifacts build && tsc -b tsconfig.json --watch", "clean": "yarn workspaces foreach -p -v run clean" diff --git a/yarn-project/yarn-project-base/Dockerfile b/yarn-project/yarn-project-base/Dockerfile index dbe9f4adf73..4b0b0f35b3f 100644 --- a/yarn-project/yarn-project-base/Dockerfile +++ b/yarn-project/yarn-project-base/Dockerfile @@ -14,6 +14,7 @@ RUN yarn --immutable # If everything's worked properly, we should no longer need access to the network. RUN echo "enableNetwork: false" >> .yarnrc.yml + # Yarn devs won't provide an extremely simple and useful feature of pruning dev dependencies from the local cache: # https://github.com/yarnpkg/berry/issues/1789 #