Merge branch 'master' into zk-relation-length-adjustments

.github/scripts/wait_for_fork.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -e
+
+DEPLOY_TAG=$1
+TEST_FORK_API_KEY=$2
+
+# When destroying and applying mainnet fork terraform, it may not be
+# ready for a while, as it must register with DNS etc.
+# This script waits on a healthy status from the fork - a valid response to the chainid request
+# We retry every 20 seconds, and wait for a total of 5 minutes (15 times)
+export ETHEREUM_HOST="https://$DEPLOY_TAG-mainnet-fork.aztec.network:8545/$TEST_FORK_API_KEY"
+
+curl -H "Content-Type: application/json" -X POST --data '{"method":"eth_chainId","params":[],"id":49,"jsonrpc":"2.0"}' \
+  --connect-timeout 30 \
+  --retry 15 \
+  --retry-delay 20 \
+  $ETHEREUM_HOST

.github/workflows/ci-arm.yml

@@ -22,58 +22,114 @@ env:
   # kludge until we move away from runners
   WAIT_FOR_RUNNERS: false
 jobs:
+  setup:
+    uses: ./.github/workflows/setup-runner.yml
+    with:
+      username: ${{ github.event.pull_request.user.login || github.actor }}
+      runner_type: builder-arm
+    secrets: inherit
+
+  changes:
+    runs-on: ubuntu-20.04
+    # Required permissions.
+    permissions:
+      pull-requests: read
+    # Set job outputs to values from filter step
+    outputs:
+      build-images: ${{ steps.filter.outputs.build-images }}
+    steps:
+      - uses: actions/checkout@v4
+        with: { ref: "${{ env.GIT_COMMIT }}" }
+      - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36
+        id: filter
+        with:
+          filters: |
+            build-images:
+              - 'build-images/**'
+
+  build-images:
+    needs: [setup, changes]
+    runs-on: ${{ github.event.pull_request.user.login || github.actor }}-arm
+    steps:
+      # permission kludge before checkout, see https://github.com/actions/checkout/issues/211#issuecomment-611986243
+      - run: sudo chown -R $USER:$USER /home/ubuntu/
+      - uses: actions/checkout@v4
+        with: { ref: "${{ env.GIT_COMMIT }}" }
+      - uses: ./.github/ci-setup-action
+        with:
+          concurrency_key: build-images-arm
+      - name: "Push Build Images If Changed"
+        if: ${{ needs.changes.outputs.build-images }}
+        timeout-minutes: 40
+        run: |
+          earthly-ci --push ./build-images/+build
+
   build:
-    runs-on: ubuntu-latest
+    needs: [build-images]
+    runs-on: ${{ github.event.pull_request.user.login || github.actor }}-arm
     steps:
+      # permission kludge before checkout, see https://github.com/actions/checkout/issues/211#issuecomment-611986243
+      - run: sudo chown -R $USER:$USER /home/ubuntu/
       - uses: actions/checkout@v4
-        with: { ref: "${{ github.event.pull_request.head.sha }}" }
+        with: { ref: "${{ env.GIT_COMMIT }}" }
+      - uses: ./.github/ci-setup-action
+        with:
+          concurrency_key: build-arm
+      # prepare images locally, tagged by commit hash
       - name: "Build E2E Image"
         timeout-minutes: 40
-        uses: ./.github/ensure-builder
-        with:
-          runner_type: builder-arm
-          run: |
-            set -eux
-            git submodule update --init --recursive --recommend-shallow
-            echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u aztecprotocolci --password-stdin
-            scripts/earthly-ci \
-              --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \
-              --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
-              ./yarn-project+export-e2e-test-images
+        run: |
+          earthly-ci ./yarn-project+export-e2e-test-images
 
-  # all the end-to-end integration tests for aztec
+  # all the non-bench end-to-end integration tests for aztec
   e2e:
-    needs: build
-    runs-on: ubuntu-latest
+    needs: [build]
+    runs-on: ${{ github.event.pull_request.user.login || github.actor }}-arm
     steps:
+      # permission kludge before checkout, see https://github.com/actions/checkout/issues/211#issuecomment-611986243
+      - run: sudo chown -R $USER:$USER /home/ubuntu/
       - uses: actions/checkout@v4
-        with: { ref: "${{ github.event.pull_request.head.sha }}" }
-      - name: "Test"
-        timeout-minutes: 25
-        uses: ./.github/ensure-builder
+        with: { ref: "${{ env.GIT_COMMIT }}" }
+      - uses: ./.github/ci-setup-action
         with:
-          runner_type: builder-arm
-          run: |
-            sudo shutdown -P 25 # hack until core part of the scripts
-            set -eux
-            echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u aztecprotocolci --password-stdin
-            scripts/earthly-ci \
-              --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \
-              --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
-              --no-output ./yarn-project/end-to-end/+uniswap-trade-on-l1-from-l2
+          concurrency_key: e2e-arm
+      # prepare images locally, tagged by commit hash
+      - name: "Build E2E Image"
+        timeout-minutes: 40
+        run: |
+          earthly-ci ./yarn-project/end-to-end+uniswap-trade-on-l1-from-l2
 
-  # not notifying failures right now
-  # notify:
-  #   needs: [e2e]
-  #   runs-on: ubuntu-latest
-  #   if: ${{ github.ref == 'refs/heads/master' && failure() }}
-  #   steps:
-  #     - name: Send notification to aztec3-ci channel if workflow failed on master
-  #       uses: slackapi/[email protected]
-  #       with:
-  #         payload: |
-  #           {
-  #             "url": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-  #           }
-  #       env:
-  #         SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFY_WORKFLOW_TRIGGER_URL }}
+  rerun-check:
+    runs-on: ubuntu-20.04
+    permissions:
+      actions: write
+    needs: [setup, build-images, build, e2e]
+    if: ${{ !cancelled() }}
+    steps:
+      - name: Check for Rerun
+        env:
+          # We treat any skipped or failing jobs as a failure for the workflow as a whole.
+          HAD_FAILURE: ${{ contains(needs.*.result, 'failure') }}
+          GH_REPO: ${{ github.repository }}
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          if [[ $HAD_FAILURE == true ]] && [[ $RUN_ATTEMPT -lt 2 ]] ; then
+            echo "Retrying first workflow failure. This is a stop-gap until things are more stable."
+            gh workflow run rerun.yml -F run_id=${{ github.run_id }}
+          fi
+
+  # NOTE: we only notify failures after a rerun has occurred
+  notify:
+    needs: [e2e]
+    runs-on: ubuntu-latest
+    if: ${{ github.ref == 'refs/heads/master' && failure() && github.run_attempt >= 2 }}
+    steps:
+      - name: Send notification to aztec3-ci channel if workflow failed on master
+        uses: slackapi/[email protected]
+        with:
+          payload: |
+            {
+              "url": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFY_WORKFLOW_TRIGGER_URL }}