Merge remote-tracking branch 'origin' into mw/logs-hash

.circleci/config.yml

@@ -431,42 +431,6 @@ jobs:
           command: build aztec-builder
           aztec_manifest_key: aztec-builder
 
-  boxes:
-    docker:
-      - image: aztecprotocol/alpine-build-image
-    resource_class: small
-    steps:
-      - *checkout
-      - *setup_env
-      - run:
-          name: "Build"
-          command: cond_spot_run_build boxes 4
-          aztec_manifest_key: boxes
-
-  boxes-vanilla:
-    docker:
-      - image: aztecprotocol/alpine-build-image
-    resource_class: small
-    steps:
-      - *checkout
-      - *setup_env
-      - run:
-          name: "Test"
-          command: cond_spot_run_compose boxes 4 ./docker-compose.yml BOX=vanilla
-          aztec_manifest_key: boxes
-
-  boxes-react:
-    docker:
-      - image: aztecprotocol/alpine-build-image
-    resource_class: small
-    steps:
-      - *checkout
-      - *setup_env
-      - run:
-          name: "Test"
-          command: cond_spot_run_compose boxes 4 ./docker-compose.yml BOX=react
-          aztec_manifest_key: boxes
-
   end-to-end:
     machine:
       image: default
@@ -695,20 +659,6 @@ workflows:
       - aztec-package: *defaults_yarn_project
       - aztec-builder: *defaults_yarn_project
 
-      # Boxes.
-      - boxes:
-          requires:
-            - aztec-package
-          <<: *defaults
-      - boxes-vanilla:
-          requires:
-            - boxes
-          <<: *defaults
-      - boxes-react:
-          requires:
-            - boxes
-          <<: *defaults
-
       # End to end tests.
       - e2e-join:
           requires:
@@ -728,8 +678,6 @@ workflows:
             - barretenberg-acir-tests-bb
             - barretenberg-acir-tests-bb-sol
             - barretenberg-docs
-            - boxes-vanilla
-            - boxes-react
             - noir-packages-tests
             - e2e-join
             - aztec-builder

aztec-up/bin/docker-compose.yml

@@ -23,7 +23,7 @@ services:
     environment:
       DEBUG: # Loaded from the user shell if explicitly set
       HOST_WORKDIR: "${PWD}" # Loaded from the user shell to show log files absolute path in host
-      ETHEREUM_HOST: http://ethereum:${ANVIL_PORT:-8545}
+      ETHEREUM_HOST: ${ETHEREUM_HOST:-http://ethereum:${ANVIL_PORT:-8545}}
       CHAIN_ID: 31337
       ARCHIVER_POLLING_INTERVAL_MS: 50
       P2P_BLOCK_CHECK_INTERVAL_MS: 50
@@ -32,5 +32,7 @@ services:
       PXE_BLOCK_POLLING_INTERVAL_MS: 50
       ARCHIVER_VIEM_POLLING_INTERVAL_MS: 500
       PXE_PORT: ${PXE_PORT:-8080}
+      PORT:  ${AZTEC_NODE_PORT:-8080}
+      TEST_ACCOUNTS: ${TEST_ACCOUNTS:-true}
     volumes:
       - ./log:/usr/src/yarn-project/aztec/log:rw

barretenberg/cpp/CMakePresets.json

@@ -31,6 +31,31 @@
         "CXX": "$env{BREW_PREFIX}/opt/llvm/bin/clang++"
       }
     },
+    {
+      "name": "darwin-arm64",
+      "displayName": "Apple arm64 Cross-compile",
+      "inherits": "default",
+      "binaryDir": "build-darwin-arm64",
+      "cacheVariables": {
+        "CMAKE_BUILD_TYPE": "Release",
+        "CMAKE_TOOLCHAIN_FILE": "/opt/osxcross/toolchain.cmake"
+      },
+      "environment": {
+        "OSXCROSS_HOST": "arm64-apple-darwin23",
+        "OSXCROSS_TARGET_DIR": "/opt/osxcross",
+        "OSXCROSS_TARGET": "darwin23",
+        "OSXCROSS_SDK": "/opt/osxcross/SDK/MacOSX14.0.sdk"
+      }
+    },
+    {
+      "name": "darwin-amd64",
+      "displayName": "Apple amd64 Cross-compile",
+      "inherits": "darwin-arm64",
+      "binaryDir": "build-darwin-amd64",
+      "environment": {
+        "OSXCROSS_HOST": "x86_64-apple-darwin23"
+      }
+    },
     {
       "name": "clang16",
       "displayName": "Build with Clang-16",
@@ -358,6 +383,16 @@
       "inherits": "default",
       "configurePreset": "op-count"
     },
+    {
+      "name": "darwin-arm64",
+      "inherits": "default",
+      "configurePreset": "darwin-arm64"
+    },
+    {
+      "name": "darwin-amd64",
+      "inherits": "default",
+      "configurePreset": "darwin-amd64"
+    },
     {
       "name": "clang16-dbg",
       "inherits": "default",
@@ -456,18 +491,14 @@
       "configurePreset": "wasm-dbg",
       "inheritConfigureEnvironment": true,
       "jobs": 0,
-      "targets": [
-        "barretenberg.wasm"
-      ]
+      "targets": ["barretenberg.wasm"]
     },
     {
       "name": "wasm-threads",
       "configurePreset": "wasm-threads",
       "inheritConfigureEnvironment": true,
       "jobs": 0,
-      "targets": [
-        "barretenberg.wasm"
-      ]
+      "targets": ["barretenberg.wasm"]
     },
     {
       "name": "xray",

barretenberg/cpp/Earthfile

@@ -16,6 +16,14 @@ source:
     # for debugging rebuilds
     RUN echo CONTENT HASH $(find . -type f -exec sha256sum {} ';' | sort | sha256sum | awk '{print $1}') | tee .content-hash
 
+preset-darwin-arm64:
+    FROM +source
+    LET OSX_SDK="MacOSX14.0.sdk"
+    LET OSX_SDK_URL="https://github.com/joseluisq/macosx-sdks/releases/download/14.0/${OSX_SDK}.tar.xz"
+    RUN curl -sSL "$OSX_SDK_URL" | tar -xJ -C /opt/osxcross/SDK && rm -rf /opt/osxcross/SDK/$OSX_SDK/System
+    RUN cmake --preset darwin-arm64 -Bbuild && cmake --build build --target bb
+    SAVE ARTIFACT build/bin AS LOCAL build-darwin-arm64/bin
+
 preset-release:
     FROM +source
     RUN cmake --preset clang16 -Bbuild && cmake --build build --target bb

barretenberg/cpp/src/barretenberg/common/serialize.hpp

@@ -121,6 +121,20 @@ inline void write(uint8_t*& it, uint64_t value)
     it += 8;
 }
 
+#ifdef __APPLE__
+inline void read(uint8_t const*& it, unsigned long& value)
+{
+    value = ntohll(*reinterpret_cast<unsigned long const*>(it));
+    it += 8;
+}
+
+inline void write(uint8_t*& it, unsigned long value)
+{
+    *reinterpret_cast<unsigned long*>(it) = htonll(value);
+    it += 8;
+}
+#endif
+
 #ifndef __i386__
 inline void read(uint8_t const*& it, uint128_t& value)
 {

boxes/.earthlyignore

@@ -0,0 +1,46 @@
+# Must include the .gitignore for all child projects as this is used by Earthly
+
+# Note due to how we use Eartlhy each .gitignore MUST accompany any earthfile that might actually copy these artifacts
+
+**/Earthfile
+**/Readme.md
+**/README.md
+**/Dockerfile*
+\*\*/docker-compose*.yml
+
+# root .gitignore contents
+
+dest
+node_modules
+.cache
+scripts/.earthly
+.pnp.cjs
+.pnp.loader.mjs
+build/
+.idea
+cmake-build-debug
+.terraform\*
+.bootstrapped
+.tsbuildinfo
+
+.graphite\*
+.DS_Store
+
+\*_/_.dockerignore
+
+# .gitignore contents:
+
+.yarn/\*
+!.yarn/releases
+
+node_modules
+dist
+artifacts
+boxes/_/src/contracts/target
+boxes/_/src/contracts/log
+
+/test-results/
+/playwright-report/
+/blob-report/
+/playwright/.cache/
+codegenCache.json

boxes/Earthfile

@@ -1,18 +1,139 @@
 VERSION 0.8
-# Produces a container that can be run to test a specific box. See docker-compose.yml.
+
+deps:
+    LOCALLY
+    LET packages = $(git ls-files "**/package*.json" package*.json)
+    LET tsconfigs = $(git ls-files "**/tsconfig*.json" tsconfig*.json)
+    # use the build image to build the project so that nargo has access to the same libc
+    # later on, once everything is built and we don't need nargo, copy to a debian based image
+    # that's supported by playwright
+    FROM ../build-images+build
+
+    # copy the aztec yarn workspace, needed to resolve portal dependencies
+    COPY ../yarn-project+build-dev/usr/src /usr/src
+
+    WORKDIR /usr/src/boxes
+
+    COPY --dir .yarn .yarnrc.yml yarn.lock .
+    FOR file IN $packages
+        COPY $file $file
+    END
+    RUN yarn install --immutable
+    # don't install playwright here as it's not compatible with ubuntu 24.04 yet
+    # https://github.com/microsoft/playwright/issues/30368
+    FOR file IN $tsconfigs
+        COPY $file $file
+    END
 
 build:
-    # We need yarn. Start fresh container.
-    FROM node:18.19.0
-    RUN apt update && apt install netcat-openbsd
-    COPY ../yarn-project+build/build /usr/src
+    FROM +deps
+
     COPY ../noir/+nargo/nargo /usr/src/noir/noir-repo/target/release/nargo
     COPY ../noir-projects/+build/aztec-nr /usr/src/noir-projects/aztec-nr
     COPY ../noir-projects/+build/noir-protocol-circuits/crates/types /usr/src/noir-projects/noir-protocol-circuits/crates/types
+
     WORKDIR /usr/src/boxes
-    COPY . .
+
     ENV AZTEC_NARGO=/usr/src/noir/noir-repo/target/release/nargo
     ENV AZTEC_BUILDER=/usr/src/yarn-project/builder/aztec-builder-dest
-    RUN yarn && yarn build
-    RUN npx -y [email protected] install --with-deps
+    COPY . .
+    RUN yarn build
+
+    SAVE ARTIFACT /usr/src /usr/src
+
+boxes:
+    FROM node:18.19
+    COPY --dir +build/usr/src /usr
+    WORKDIR /usr/src/boxes
+    RUN yarn install-browsers
     ENTRYPOINT ["/bin/sh", "-c"]
+
+BOX_TEST_LOCAL:
+  FUNCTION
+  ARG box
+  ARG browser
+  ARG compose_file=./docker-compose.yml
+  ARG debug="aztec:*"
+  LOCALLY
+  ENV BOX=$box
+  ENV BROWSER=$browser
+  ENV DEBUG=$debug
+  ENV CI=1
+  WITH DOCKER \
+    --load aztecprotocol/aztec:latest=../yarn-project/+aztec \
+    --load aztecprotocol/boxes:latest=+boxes
+    RUN docker compose -f $compose_file up --exit-code-from=boxes --force-recreate
+  END
+
+# run locally and take from cache, used for our mainly x86 jobs
+BOX_TEST_FROM_CACHE:
+  FUNCTION
+  ARG box
+  ARG browser
+  ARG compose_file=./docker-compose.yml
+  ARG debug="aztec:*"
+  ARG aztec_docker_tag
+  LOCALLY
+  ENV BOX=$box
+  ENV BROWSER=$browser
+  ENV DEBUG=$debug
+  ENV AZTEC_DOCKER_TAG=$aztec_docker_tag
+  ENV CI=1
+  # need a different project name for each to run in parallel
+  LET project_name=$box_$browser
+  IF docker compose > /dev/null 2>&1
+    LET CMD="docker compose"
+  ELSE
+    LET CMD="docker-compose"
+  END
+  # In CI we do not use WITH DOCKER as we have had issues with earthly copying big images
+  RUN $CMD -p $project_name -f $compose_file up --exit-code-from=boxes --force-recreate
+
+BOX_TEST_FROM_BUILD:
+  FUNCTION
+  ARG box
+  ARG browser
+  ARG compose_file=./docker-compose.yml
+  ARG debug="aztec:*"
+  FROM earthly/dind:alpine-3.19-docker-25.0.2-r0
+  ENV BOX=$box
+  ENV BROWSER=$browser
+  ENV DEBUG=$debug
+  ENV CI=1
+  COPY $compose_file $compose_file
+  # For ARM, we do use WITH DOCKER as we don't have many e2e tests, but note BOX_TEST_FROM_CACHE
+  WITH DOCKER \
+    --load aztecprotocol/aztec:latest=../yarn-project/+aztec \
+    --load aztecprotocol/boxes:latest=+boxes
+    # Run our docker compose, ending whenever sandbox ends, filtering out noisy eth_getLogs
+    RUN docker compose -f $compose_file up --exit-code-from=boxes --force-recreate
+  END
+
+BOX_TEST:
+  FUNCTION
+  ARG box
+  ARG browser
+  ARG compose_file=./docker-compose.yml
+  ARG mode=local
+  ARG debug="aztec:*"
+  LOCALLY
+  IF [ $mode = local ]
+    DO +BOX_TEST_LOCAL --box=$box --browser=$browser --compose_file=$compose_file --debug=$debug
+  ELSE IF [ $mode = cache ]
+    DO +BOX_TEST_FROM_CACHE --box=$box --browser=$browser --aztec_docker_tag=$(git rev-parse HEAD) --compose_file=$compose_file --debug=$debug
+  ELSE
+    DO +BOX_TEST_FROM_BUILD --box=$box --browser=$browser --compose_file=$compose_file --debug=$debug
+  END
+
+test:
+  ARG box=vanilla
+  ARG browser=chromium
+  ARG mode=local
+  DO +BOX_TEST --box=$box --browser=$browser --mode=$mode
+
+# for use with mode=cache
+export-boxes:
+    BUILD +boxes
+    ARG EARTHLY_GIT_HASH
+    FROM +boxes
+    SAVE IMAGE aztecprotocol/boxes:$EARTHLY_GIT_HASH