chore: squash yp ypb + other build improvements. (#4901)

* Get rid of `boxes-files` job as unbox command was removed from
aztec-cli.
* Get rid of `yarn-project-base`. Attempting to skip dependency
downloads was a bit of a false economy and was quite a bit of
complication.
* `ecr_login` more explicitly, rather than every time we call
`extract_repo`.
* We flatten the yarn-project jobs a bit. This means we do repeat work
(bootstrapping), but can better parallelise jobs.
* Remove all `.dockerignore` files from the repo, and ignore them from
git. Instead we compute the file from git when building in CI or using
`bootstrap_docker.sh`. This is much less error prone, and means when
testing building images locally we're getting the same as in CI. New
script is `create_docker_ignore`.
* Add `--platform=linux/amd` in some `FROM` directives where arch
doesn't matter, to improve fast bootstrap on arm systems.
* `build_local` logs into ecr just once.
* Add `[ci no-term <repo>]` option which allows connection to aws build
instances for debugging. (They still get reaped after 20m).
* Remove `l1-contracts` installing submodule deps in dockerfile. It's
expected this to have been done before building the image by the
build-system.
* Remove `end-to-end/scripts/docker-compose-browser.yml` and start
script, seems this was a workaround from some alpine build stuff, but we
just build using ubuntu now.
* Remove `unlimited` from number of jobs. Having this actually crashed
my mac when running the tests. By default we still use half the cores,
which is fine as each test suite runs tests in parallel as well.

.circleci/config.yml

@@ -435,77 +435,62 @@ jobs:
           command: build noir-projects
           aztec_manifest_key: noir-projects
 
-  boxes-files:
-    machine:
-      image: default
-    resource_class: medium
+  yarn-project-pre-join:
+    docker:
+      - image: cimg/base:2023.09
+    resource_class: small
     steps:
-      - *checkout
-      - *setup_env
       - run:
-          name: "Build"
-          command: build boxes-files
-          aztec_manifest_key: boxes-files
+          name: "Noop"
+          command: echo Noop
 
-  yarn-project-base:
-    machine:
-      image: default
-    resource_class: large
+  yarn-project-x86_64:
+    docker:
+      - image: aztecprotocol/alpine-build-image
+    resource_class: small
     steps:
       - *checkout
       - *setup_env
       - run:
           name: "Build"
-          command: build yarn-project-base | add_timestamps
-          aztec_manifest_key: yarn-project-base
+          command: cond_spot_run_build yarn-project 64
+          aztec_manifest_key: yarn-project
 
-  yarn-project:
-    machine:
-      image: default
-    resource_class: large
+  yarn-project-arm64:
+    docker:
+      - image: aztecprotocol/alpine-build-image
+    resource_class: small
     steps:
       - *checkout
       - *setup_env
       - run:
-          name: Build
-          command: build yarn-project | add_timestamps
+          name: "Build"
+          command: cond_spot_run_build yarn-project 64 arm64
           aztec_manifest_key: yarn-project
 
-  yarn-project-prod:
+  yarn-project-ecr-manifest:
     machine:
       image: default
-    resource_class: large
-    steps:
-      - *checkout
-      - *setup_env
-      - run:
-          name: Build
-          command: build yarn-project-prod | add_timestamps
-          aztec_manifest_key: yarn-project-prod
-
-  yarn-project-formatting:
-    docker:
-      - image: aztecprotocol/alpine-build-image
-    resource_class: small
+    resource_class: medium
     steps:
       - *checkout
       - *setup_env
       - run:
-          name: Check Formatting
-          command: cond_spot_run_container yarn-project 8 formatting | add_timestamps
+          name: "Create ECR manifest"
+          command: create_ecr_manifest yarn-project x86_64,arm64
           aztec_manifest_key: yarn-project
 
-  yarn-project-tests:
+  yarn-project-test:
     docker:
       - image: aztecprotocol/alpine-build-image
     resource_class: small
     steps:
       - *checkout
       - *setup_env
       - run:
-          name: Test
-          command: cond_spot_run_container yarn-project 64 test | add_timestamps
-          aztec_manifest_key: yarn-project
+          name: "Build and test"
+          command: cond_spot_run_build yarn-project-test 64
+          aztec_manifest_key: yarn-project-test
 
   aztec-package:
     machine:
@@ -985,7 +970,7 @@ jobs:
       - *setup_env
       - run:
           name: "Test"
-          command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-browser.yml TEST=e2e_aztec_js_browser.test.ts
+          command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_aztec_js_browser.test.ts
           aztec_manifest_key: end-to-end
 
   e2e-card-game:
@@ -1151,9 +1136,6 @@ jobs:
     steps:
       - *checkout
       - *setup_env
-      - run:
-          name: "Copy docs dockerignore"
-          command: cp docs/.dockerignore .
       - run:
           name: "Build docs"
           command: |
@@ -1317,14 +1299,14 @@ defaults: &defaults
         event: fail
         branch_pattern: "master"
 
-defaults_yarn_project: &defaults_yarn_project
+defaults_yarn_project_pre_join: &defaults_yarn_project_pre_join
   requires:
-    - yarn-project
+    - yarn-project-pre-join
   <<: *defaults
 
-defaults_yarn_project_prod: &defaults_yarn_project_prod
+defaults_yarn_project: &defaults_yarn_project
   requires:
-    - yarn-project-prod
+    - yarn-project-ecr-manifest
   <<: *defaults
 
 defaults_deploy: &defaults_deploy
@@ -1418,37 +1400,35 @@ workflows:
 
       - yellow-paper: *defaults
 
-      - boxes-files: *defaults
-
       - noir-projects:
           requires:
             - avm-transpiler
             - noir-ecr-manifest
           <<: *defaults
 
       # Yarn Project
-      - yarn-project-base:
+      - yarn-project-pre-join:
           requires:
             - bb-js
             - noir-packages
-          <<: *defaults
-      - yarn-project:
-          requires:
-            - yarn-project-base
             - l1-contracts
             - noir-projects
-            - boxes-files
           <<: *defaults
-      - yarn-project-prod: *defaults_yarn_project
-      - yarn-project-formatting: *defaults_yarn_project
-      - yarn-project-tests: *defaults_yarn_project
-      - end-to-end: *defaults_yarn_project
-      - build-docs: *defaults_yarn_project
+      - end-to-end: *defaults_yarn_project_pre_join
+      - aztec-faucet: *defaults_yarn_project_pre_join
+      - build-docs: *defaults_yarn_project_pre_join
+      - yarn-project-test: *defaults_yarn_project_pre_join
+      - yarn-project-x86_64: *defaults_yarn_project_pre_join
+      - yarn-project-arm64: *defaults_yarn_project_pre_join
+      - yarn-project-ecr-manifest:
+          requires:
+            - yarn-project-x86_64
+            - yarn-project-arm64
+          <<: *defaults
 
       # Artifacts
-      - aztec-package: *defaults_yarn_project_prod
-      - cli: *defaults_yarn_project_prod
-      - aztec-faucet: *defaults_yarn_project_prod
+      - aztec-package: *defaults_yarn_project
+      - cli: *defaults_yarn_project
 
       # Boxes.
       - boxes:

.gitignore

@@ -15,3 +15,5 @@ cmake-build-debug
 
 .graphite*
 .DS_Store
+
+**/*.dockerignore

barretenberg/sol/Dockerfile

@@ -23,20 +23,14 @@ RUN apk update && apk add git curl build-base openmp-dev bash
 COPY --from=0 /usr/src/barretenberg/cpp/build/bin /usr/src/barretenberg/cpp/build/bin
 COPY --from=0 /usr/src/barretenberg/cpp/srs_db /usr/src/barretenberg/cpp/srs_db
 WORKDIR /usr/src/barretenberg/sol
-RUN git init
 COPY ./sol .
 
 # Copy forge binary directly from foundry
 COPY --from=ghcr.io/foundry-rs/foundry:latest /usr/local/bin/forge /usr/local/bin/forge
 
-RUN forge install --no-commit \
-  https://github.com/foundry-rs/forge-std \
-  https://github.com/openzeppelin/openzeppelin-contracts \
-  https://github.com/Arachnid/solidity-stringutils
-
 RUN cd ../cpp/srs_db && ./download_ignition.sh 3 && cd ../../sol
 
 RUN ./scripts/init.sh
 
-# TestBase is excluded as it is just boilerplate 
+# TestBase is excluded as it is just boilerplate
 RUN forge test --no-match-contract TestBase

boxes/Dockerfile

@@ -1,4 +1,3 @@
-# Builds the boxes (they were copied into yarn-project-base so the cli can unbox).
 # Produces a container that can be run to test a specific box. See docker-compose.yml.
 FROM aztecprotocol/aztec AS aztec
 FROM aztecprotocol/noir as noir
@@ -10,7 +9,9 @@ RUN apt update && apt install netcat-openbsd
 COPY --from=aztec /usr/src /usr/src
 COPY --from=noir /usr/src/noir/noir-repo/target/release/nargo /usr/src/noir/noir-repo/target/release/nargo
 COPY --from=noir-projects /usr/src/noir-projects/aztec-nr /usr/src/noir-projects/aztec-nr
+COPY --from=noir-projects /usr/src/noir-projects/noir-protocol-circuits/crates/types /usr/src/noir-projects/noir-protocol-circuits/crates/types
 WORKDIR /usr/src/boxes
+COPY . .
 ENV AZTEC_NARGO=/usr/src/noir/noir-repo/target/release/nargo
 ENV AZTEC_CLI=/usr/src/yarn-project/cli/aztec-cli-dest
 RUN yarn && yarn build

build-system/scripts/build

@@ -31,28 +31,6 @@ echo "Working directory: $PWD"
 echo "Dockerfile: $DOCKERFILE"
 echo "Build directory: $BUILD_DIR"
 
-# Fetch images with retries
-function fetch_image() {
-  echo "Pulling: $1"
-  if ! retry docker pull $1 > /dev/null 2>&1; then
-    echo "Image not found: $1"
-    return 1
-  fi
-  return 0
-}
-
-# Fetch images, but don't assume this will work
-function try_fetch_image() {
-  echo "Pulling: $1"
-  if ! docker pull $1 > /dev/null 2>&1; then
-    echo "Image not found: $1"
-    return 1
-  fi
-  return 0
-}
-
-NOREPO=$(query_manifest noRepo $REPOSITORY)
-
 # Login to ECR and ensure repository exists.
 retry ensure_repo $REPOSITORY $ECR_REGION refresh_lifecycle
 # Login to dockerhub.
@@ -91,6 +69,7 @@ if [ -d $ROOT_PATH/$PROJECT_DIR/terraform ]; then
 fi
 
 # For each dependency, substitute references to the dependency in dockerfile, with the relevent built image uri.
+# This is necessary vs pulling and retagging the image, as that doesn't work with buildx.
 # We have to perform a bit of probing to determine which actual image we want to use.
 # When we used buildx to create a multiarch image, there will be no images with "-$ARCH" suffixes (normalise this?).
 # Also we sometimes build an arm image from an x86 parent, so there won't always be an arm parent, and we fallback.
@@ -115,9 +94,12 @@ for PARENT_REPO in $(query_manifest dependencies $REPOSITORY); do
   # Substitute references to parent repo, with the relevent built image uri.
   DEPLOY_URI=aztecprotocol/$PARENT_REPO
   PARENT_IMAGE_URI=$ECR_URL/$PARENT_REPO:$PARENT_IMAGE_TAG
-  awk '{if ($1 == "FROM" && $2 == "'$DEPLOY_URI'") $2 = "'$PARENT_IMAGE_URI'"; print $0}' $DOCKERFILE > _temp && mv _temp $DOCKERFILE
+  sed -i "s#^FROM \\(.*\\)${DEPLOY_URI}\\( \|$\\)#FROM \\1${PARENT_IMAGE_URI}\\2#" $DOCKERFILE
 done
 
+# Build a dockerignore file that only permits git files and untracked files.
+create_dockerignore $REPOSITORY
+
 COMMIT_TAG_VERSION=$(extract_tag_version $REPOSITORY false)
 echo "Commit tag version: $COMMIT_TAG_VERSION"