Merge branch 'master' into gd/unsafe_ec_add2

AztecProtocol · Oct 10, 2024 · bb57173 · bb57173
2 parents bab195f + 1801f5b
commit bb57173
Show file tree

Hide file tree

Showing 755 changed files with 31,528 additions and 16,673 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -372,18 +372,6 @@ jobs:
           command: build cli-wallet
           aztec_manifest_key: cli-wallet
 
-  end-to-end:
-    machine:
-      image: default
-    resource_class: large
-    steps:
-      - *checkout
-      - *setup_env
-      - run:
-          name: "Build"
-          command: build end-to-end
-          aztec_manifest_key: end-to-end
-
   # For old e2e tests see yarn-project/end-to-end/Earthfile
   # Semantics are similar to Dockerfile
 
@@ -583,7 +571,6 @@ workflows:
             - l1-contracts
             - noir-projects
           <<: *defaults
-      - end-to-end: *defaults_yarn_project
       - yarn-project-x86_64: *defaults_yarn_project_pre_join
       - yarn-project-arm64: *defaults_yarn_project_pre_join
       - yarn-project-ecr-manifest:
@@ -599,7 +586,6 @@ workflows:
       # End to end tests.
       - e2e-join:
           requires:
-            - end-to-end
             - aztec-package
           <<: *defaults
 
@@ -613,4 +599,4 @@ workflows:
             - barretenberg-docs
             - e2e-join
             - cli-wallet
-          <<: *defaults
+          <<: *defaults
diff --git a/.github/img/bb_banner.png b/.github/img/bb_banner.png
diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml
@@ -81,8 +81,9 @@ jobs:
           concurrency_key: build-arm
       # prepare images locally, tagged by commit hash
       - name: "Build E2E Image"
-        timeout-minutes: 40
+        timeout-minutes: 60
         run: |
+          sudo shutdown -P 60 # necessary to get around builtin hard timeout
           earthly-ci ./yarn-project+export-e2e-test-images
 
   # all the non-bench end-to-end integration tests for aztec

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -305,24 +305,24 @@ jobs:
         run: earthly-ci --no-output +preset-gcc
 
   # barretenberg (prover) native, AVM (public VM) and Merkle tree (world state) tests
-  # only ran on x86 for resource reasons (memory intensive)
+  # ran on own runner for resource reasons (memory x cpu intensive)
   bb-native-tests:
-    needs: [build, changes]
-    runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
+    runs-on: ubuntu-20.04
+    needs: [build-images, changes]
     if: needs.changes.outputs.barretenberg-cpp == 'true'
     steps:
       - uses: actions/checkout@v4
         with: { ref: "${{ env.GIT_COMMIT }}" }
-      # Only allow one memory-hunger prover test to use this runner
       - uses: ./.github/ci-setup-action
-        with:
-          # must be globally unique for build x runner
-          concurrency_key: bb-native-tests-x86
       - name: "Native Prover Tests"
-        working-directory: ./barretenberg/cpp/
-        timeout-minutes: 40
-        # limit our parallelism to half our cores
-        run: earthly-ci --exec-stats --no-output +test --hardware_concurrency=64
+        uses: ./.github/ensure-tester
+        # note: tester by default times out at 30, would need to delay shutdwon for more
+        timeout-minutes: 30
+        with:
+          runner_type: 128core-tester-x86
+          run: |
+            echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u aztecprotocolci --password-stdin
+            scripts/earthly-ci --exec-stats --no-output --no-cache ./barretenberg/cpp+test --hardware_concurrency=32
 
   bb-js-test:
     needs: [build, changes]
@@ -474,7 +474,7 @@ jobs:
         run: earthly-ci --no-output ./noir+packages-test
 
   noir-projects:
-    needs: [build, changes, build]
+    needs: [build, changes]
     runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
     if: needs.changes.outputs.barretenberg == 'true' || needs.changes.outputs.noir == 'true' || needs.changes.outputs.noir-projects == 'true' || needs.changes.outputs.txe == 'true'
     steps:
@@ -486,7 +486,7 @@ jobs:
       - name: "Noir Projects"
         timeout-minutes: 40
         run: |
-          earthly-ci --no-output ./noir-projects/+test
+          earthly-ci --no-output ./noir-projects/+test --RAYON_NUM_THREADS=$(nproc)
 
   avm-format:
     needs: [build, changes]
@@ -542,6 +542,19 @@ jobs:
         timeout-minutes: 40
         run: earthly-ci --no-output ./yarn-project/+prover-client-test
 
+  network-test-fake-proofs:
+    needs: build
+    runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
+    steps:
+      - uses: actions/checkout@v4
+        with: { ref: "${{ env.GIT_COMMIT }}" }
+      - uses: ./.github/ci-setup-action
+        with:
+          concurrency_key: network-test-fake-proofs-x86
+      - name: "Prover Client Tests"
+        timeout-minutes: 40
+        run: earthly-ci --no-output ./yarn-project/+network-test-fake-proofs
+
   l1-contracts-test:
     needs: [build, changes]
     runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
@@ -766,6 +779,7 @@ jobs:
       - yarn-project-formatting
       - yarn-project-test
       - prover-client-test
+      - network-test-fake-proofs
       - l1-contracts-test
       - docs-preview
       # - bb-bench # non-blocking
@@ -821,6 +835,7 @@ jobs:
       - yarn-project-formatting
       - yarn-project-test
       - prover-client-test
+      - network-test-fake-proofs
       - l1-contracts-test
       - docs-preview
       # - bb-bench # non-blocking

diff --git a/.github/workflows/devnet-deploys.yml b/.github/workflows/devnet-deploys.yml
@@ -617,12 +617,12 @@ jobs:
           env
           ./.github/scripts/wait_for_infra.sh pxe ${{ env.DEPLOY_TAG }} ${{ env.API_KEY }}
 
-      - name: Deploy protocol contracts
+      - name: Setup protocol contracts
         run: |
           set -e
           set -o pipefail
           docker pull aztecprotocol/aztec:${{ env.DEPLOY_TAG }}
-          docker run aztecprotocol/aztec:${{ env.DEPLOY_TAG }} deploy-protocol-contracts \
+          docker run aztecprotocol/aztec:${{ env.DEPLOY_TAG }} setup-protocol-contracts \
             --rpc-url https://api.aztec.network/${{ env.DEPLOY_TAG }}/aztec-pxe/${{ env.API_KEY }} \
             --l1-chain-id ${{ env.L1_CHAIN_ID }} \
             --json | tee ./protocol_contracts.json

diff --git a/.github/workflows/network-deploy.yml b/.github/workflows/network-deploy.yml
@@ -54,4 +54,4 @@ jobs:
       - name: Deploy Helm chart
         run: |
           helm dependency update ${{ env.CHART_PATH }}
-          helm upgrade --install ${{ env.NAMESPACE }} ${{ env.CHART_PATH }} --namespace ${{ env.NAMESPACE }} --set network.public=true --atomic
+          helm upgrade --install ${{ env.NAMESPACE }} ${{ env.CHART_PATH }} --namespace ${{ env.NAMESPACE }} --set network.public=true --atomic --create-namespace --timeout 20m
diff --git a/.github/workflows/nightly-kind-test.yml b/.github/workflows/nightly-kind-test.yml
@@ -75,7 +75,7 @@ jobs:
             ./spartan/scripts/setup_local_k8s.sh
             export FORCE_COLOR=1
             export EARTHLY_BUILD_ARGS="${{ env.EARTHLY_BUILD_ARGS }}"
-            ./scripts/earthly-ci --exec-stats -P --no-output ./yarn-project/end-to-end/+network-transfer --values-file=${{ matrix.values_file }}
+            ./scripts/earthly-ci --exec-stats -P --no-output ./yarn-project/end-to-end/+kind-network-transfer --values-file=${{ matrix.values_file }}
 
   success-check:
     runs-on: ubuntu-20.04

diff --git a/.github/workflows/publish-aztec-packages.yml b/.github/workflows/publish-aztec-packages.yml
@@ -1,6 +1,10 @@
 name: Publish Aztec Packages
 
 on:
+  push:
+    branches:
+      - master
+      - "*/release-master*"
   workflow_dispatch:
     inputs:
       tag:
@@ -19,16 +23,25 @@ permissions:
 
 env:
   # Add any global environment variables here
-  DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
-  GIT_COMMIT: ${{ github.sha }}
+  DOCKERHUB_PASSWORD: "${{ secrets.DOCKERHUB_PASSWORD }}"
+  RUN_ID: ${{ github.run_id }}
+  RUN_ATTEMPT: ${{ github.run_attempt }}
+  USERNAME: ${{ github.actor }}
+  GITHUB_TOKEN: ${{ github.token }}
+  GH_SELF_HOSTED_RUNNER_TOKEN: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }}
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }}
+  GIT_COMMIT: ${{ github.event.pull_request.head.sha || github.sha }}
+  SHOULD_PUBLISH_DOCKER_IMAGES: ${{ github.event.inputs.publish == 'true' || github.event_name == 'push' }}
   DEPLOY_TAG: ${{ github.event.inputs.tag }}
-
 jobs:
   check-version:
     runs-on: ubuntu-latest
     steps:
       - name: Check if tag is valid
         id: check_tag
+        if: github.event_name == 'workflow_dispatch'
         run: |
           TAG=${{ github.event.inputs.tag }}
           if [[ "$TAG" == aztec-packages-v* ]]; then
@@ -164,14 +177,12 @@ jobs:
 
   publish-manifests:
     needs:
-      [
-        build-aztec-x86,
-        build-aztec-arm,
-        build-nargo-x86,
-        build-nargo-arm,
-        build-cli-wallet-x86,
-        build-cli-wallet-arm,
-      ]
+      - build-aztec-x86
+      - build-aztec-arm
+      - build-nargo-x86
+      - build-nargo-arm
+      - build-cli-wallet-x86
+      - build-cli-wallet-arm
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -181,10 +192,16 @@ jobs:
         with:
           dockerhub_password: "${{ env.DOCKERHUB_PASSWORD }}"
       - name: Publish aztec manifests
-        if: ${{ github.event.inputs.publish == 'true' }}
+        if: ${{ env.SHOULD_PUBLISH_DOCKER_IMAGES == 'true' }}
         run: |
-          DEPLOY_TAG=${{ env.DEPLOY_TAG }}
-          VERSION=${DEPLOY_TAG#aztec-packages-v}
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            TAG=${{ env.DEPLOY_TAG }}
+            VERSION=${TAG#aztec-packages-v}
+            DIST_TAG=latest
+          else
+            VERSION=""
+            DIST_TAG=$(echo "${{ github.ref_name }}" | sed 's/\//-/g')
+          fi
 
           docker pull aztecprotocol/aztec:${{ env.GIT_COMMIT }}-x86_64
           docker pull aztecprotocol/aztec:${{ env.GIT_COMMIT }}-arm64
@@ -193,34 +210,41 @@ jobs:
           docker pull aztecprotocol/cli-wallet:${{ env.GIT_COMMIT }}-x86_64
           docker pull aztecprotocol/cli-wallet:${{ env.GIT_COMMIT }}-arm64
 
-          docker manifest create aztecprotocol/aztec:$VERSION \
-            aztecprotocol/aztec:${{ env.GIT_COMMIT }}-x86_64 \
-            aztecprotocol/aztec:${{ env.GIT_COMMIT }}-arm64
-          docker manifest create aztecprotocol/aztec:latest \
+          # Create and push versioned manifest if a version is provided
+          if [ -n "$VERSION" ]; then
+            docker manifest create aztecprotocol/aztec:$VERSION \
+              aztecprotocol/aztec:${{ env.GIT_COMMIT }}-x86_64 \
+              aztecprotocol/aztec:${{ env.GIT_COMMIT }}-arm64
+            docker manifest push aztecprotocol/aztec:$VERSION
+
+            docker manifest create aztecprotocol/aztec-nargo:$VERSION \
+              aztecprotocol/aztec-nargo:${{ env.GIT_COMMIT }}-x86_64 \
+              aztecprotocol/aztec-nargo:${{ env.GIT_COMMIT }}-arm64
+            docker manifest push aztecprotocol/aztec-nargo:$VERSION
+
+            docker manifest create aztecprotocol/cli-wallet:$VERSION \
+              aztecprotocol/cli-wallet:${{ env.GIT_COMMIT }}-x86_64 \
+              aztecprotocol/cli-wallet:${{ env.GIT_COMMIT }}-arm64
+            docker manifest push aztecprotocol/cli-wallet:$VERSION
+          fi
+
+          docker manifest create aztecprotocol/aztec:$DIST_TAG \
             aztecprotocol/aztec:${{ env.GIT_COMMIT }}-x86_64 \
             aztecprotocol/aztec:${{ env.GIT_COMMIT }}-arm64
-          docker manifest push aztecprotocol/aztec:$VERSION
-          docker manifest push aztecprotocol/aztec:latest
+          docker manifest push aztecprotocol/aztec:$DIST_TAG
 
-          docker manifest create aztecprotocol/aztec-nargo:$VERSION \
-            aztecprotocol/aztec-nargo:${{ env.GIT_COMMIT }}-x86_64 \
-            aztecprotocol/aztec-nargo:${{ env.GIT_COMMIT }}-arm64
-          docker manifest create aztecprotocol/aztec-nargo:latest \
+          docker manifest create aztecprotocol/aztec-nargo:$DIST_TAG \
             aztecprotocol/aztec-nargo:${{ env.GIT_COMMIT }}-x86_64 \
             aztecprotocol/aztec-nargo:${{ env.GIT_COMMIT }}-arm64
-          docker manifest push aztecprotocol/aztec-nargo:$VERSION
-          docker manifest push aztecprotocol/aztec-nargo:latest
+          docker manifest push aztecprotocol/aztec-nargo:$DIST_TAG
 
-          docker manifest create aztecprotocol/cli-wallet:$VERSION \
-            aztecprotocol/cli-wallet:${{ env.GIT_COMMIT }}-x86_64 \
-            aztecprotocol/cli-wallet:${{ env.GIT_COMMIT }}-arm64
-          docker manifest create aztecprotocol/cli-wallet:latest \
+          docker manifest create aztecprotocol/cli-wallet:$DIST_TAG \
             aztecprotocol/cli-wallet:${{ env.GIT_COMMIT }}-x86_64 \
             aztecprotocol/cli-wallet:${{ env.GIT_COMMIT }}-arm64
-          docker manifest push aztecprotocol/cli-wallet:$VERSION
-          docker manifest push aztecprotocol/cli-wallet:latest
+          docker manifest push aztecprotocol/cli-wallet:$DIST_TAG
 
   publish-npm:
+    if: github.event_name == 'workflow_dispatch'
     needs: publish-manifests
     runs-on: ${{ github.actor }}-x86
     env:
@@ -257,3 +281,33 @@ jobs:
             --DIST_TAG=latest \
             --VERSION=$VERSION \
             --DRY_RUN=${{ (github.event.inputs.publish == 'false') && '1' || '0' }}
+
+  # Sometimes runners get killed because they can be spot, we try once more for good measure
+  rerun-check:
+    runs-on: ubuntu-20.04
+    permissions:
+      actions: write
+    needs:
+      - setup-x86
+      - setup-arm
+      - build-aztec-x86
+      - build-aztec-arm
+      - build-nargo-x86
+      - build-nargo-arm
+      - build-cli-wallet-x86
+      - build-cli-wallet-arm
+      - publish-npm
+      - publish-manifests
+    if: github.event.pull_request.draft == false && !cancelled()
+    steps:
+      - name: Check for Rerun
+        env:
+          # We treat any skipped or failing jobs as a failure for the workflow as a whole.
+          HAD_FAILURE: ${{ contains(needs.*.result, 'failure') }}
+          GH_REPO: ${{ github.repository }}
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          if [[ $HAD_FAILURE == true ]] && [[ $RUN_ATTEMPT -lt 2 ]] ; then
+            echo "Retrying first workflow failure. This is a stop-gap until things are more stable."
+            gh workflow run rerun.yml -F run_id=${{ github.run_id }}
+          fi
diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml
@@ -38,7 +38,7 @@ jobs:
 
   pdf:
     needs: setup
-    runs-on: master-x86
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -54,6 +54,7 @@ jobs:
       - name: Serve docs
         run: |
           cd aztec-packages/docs
+          yarn install
           yarn build
           yarn serve &
       - name: Checkout PDF repo

diff --git a/.gitignore b/.gitignore
@@ -26,4 +26,8 @@ terraform.tfstate*
 .bb_tmp
 
 # Terraform
-*.tfvars
+*.tfvars
+
+# tmux
+tmux-client-*.log
+.supermavenignore
diff --git a/.noir-sync-commit b/.noir-sync-commit
@@ -1 +1 @@
-c3cb38a7c4de6fc321b367eda3fca6d06e76b77a
+c4273a0c8f8b751a3dbe097e070e4e7b2c8ec438
diff --git a/.release-please-manifest.json b/.release-please-manifest.json
@@ -1,7 +1,7 @@
 {
-  ".": "0.56.0",
+  ".": "0.57.0",
   "yarn-project/cli": "0.35.1",
-  "yarn-project/aztec": "0.56.0",
-  "barretenberg": "0.56.0",
-  "barretenberg/ts": "0.56.0"
+  "yarn-project/aztec": "0.57.0",
+  "barretenberg": "0.57.0",
+  "barretenberg/ts": "0.57.0"
 }
diff --git a/.supermavenignore b/.supermavenignore
@@ -0,0 +1 @@
+*.md
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		c3cb38a7c4de6fc321b367eda3fca6d06e76b77a
		c4273a0c8f8b751a3dbe097e070e4e7b2c8ec438