feat: benchmark private proving #9228
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: [master] | |
| pull_request: {} | |
| workflow_dispatch: | |
| inputs: {} | |
| concurrency: | |
| # force parallelism in master | |
| group: ci-${{ github.ref_name == 'master' && github.run_id || github.ref_name }} | |
| cancel-in-progress: true | |
| env: | |
| DOCKERHUB_PASSWORD: "${{ secrets.DOCKERHUB_PASSWORD }}" | |
| RUN_ID: ${{ github.run_id }} | |
| RUN_ATTEMPT: ${{ github.run_attempt }} | |
| USERNAME: ${{ github.event.pull_request.user.login || github.actor }} | |
| GITHUB_TOKEN: ${{ github.token }} | |
| GH_SELF_HOSTED_RUNNER_TOKEN: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }} | |
| GIT_COMMIT: ${{ github.event.pull_request.head.sha || github.sha }} | |
| # kludge until we move away from runners | |
| WAIT_FOR_RUNNERS: false | |
| jobs: | |
| setup: | |
| uses: ./.github/workflows/setup-runner.yml | |
| with: | |
| username: ${{ github.event.pull_request.user.login || github.actor }} | |
| runner_type: builder-x86 | |
| secrets: inherit | |
| # job just to alert that you *might* hit missing CI files, but if not you can continue | |
| ci-consistency: | |
| runs-on: ubuntu-latest | |
| # Only check PRs for consistency (not master) | |
| if: ${{ github.event.pull_request.head.sha != '' }} | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Checkout Merge Pipeline Files | |
| uses: actions/checkout@v4 | |
| with: | |
| path: merge-commit-pipeline-files | |
| sparse-checkout: | | |
| .github/workflows/ci.yml | |
| .github/workflows/setup-runner.yml | |
| - name: Ensure CI Consistency | |
| if: ${{ github.event.pull_request.head.sha != '' }} | |
| run: | | |
| # Compare the checked-out CI configuration files with the reference files | |
| if ! git diff --no-index .github/workflows/ci.yml merge-commit-pipeline-files/.github/workflows/ci.yml; then | |
| echo "Error: ci.yml changes in master (or PR base). *Usually* you can continue just fine. If you hit trouble with missing files, please merge these changes. This is to alert about potential surprises from Github Action's merge behavior." | |
| exit 1 | |
| fi | |
| if ! git diff --no-index .github/workflows/setup-runner.yml merge-commit-pipeline-files/.github/workflows/setup-runner.yml; then | |
| echo "Error: setup-runner.yml changes in master (or PR base). *Usually* you can continue just fine. If you hit trouble with missing files, please merge these changes. This is to alert about potential surprises from Github Action's merge behavior." | |
| exit 1 | |
| fi | |
| build: | |
| needs: setup | |
| runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 | |
| outputs: | |
| e2e_list: ${{ steps.e2e_list.outputs.list }} | |
| bench_list: ${{ steps.bench_list.outputs.list }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: { ref: "${{ env.GIT_COMMIT }}" } | |
| - uses: ./.github/ci-setup-action | |
| with: | |
| concurrency_key: build-x86 | |
| # prepare images locally, tagged by commit hash | |
| - name: "Build E2E Image" | |
| timeout-minutes: 40 | |
| run: earthly-ci ./yarn-project+export-e2e-test-images | |
| # We base our e2e list used in e2e-x86 off the targets in ./yarn-project/end-to-end | |
| # (Note ARM uses just 2 tests as a smoketest) | |
| - name: Create list of non-bench end-to-end jobs | |
| id: e2e_list | |
| run: echo "list=$(earthly ls ./yarn-project/end-to-end | grep -v '+base' | grep -v '+bench' | sed 's/+//' | jq -R . | jq -cs .)" >> $GITHUB_OUTPUT | |
| - name: Create list of bench end-to-end jobs | |
| id: bench_list | |
| run: echo "list=$(earthly ls ./yarn-project/end-to-end | grep '+bench' | sed 's/+//' | jq -R . | jq -cs .)" >> $GITHUB_OUTPUT | |
| # all the non-bench end-to-end integration tests for aztec | |
| e2e: | |
| needs: build | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| test: ${{ fromJson( needs.build.outputs.e2e_list )}} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: { ref: "${{ env.GIT_COMMIT }}" } | |
| - uses: ./.github/ci-setup-action | |
| - name: Setup and Test | |
| timeout-minutes: 40 | |
| uses: ./.github/ensure-tester-with-images | |
| with: | |
| runner_type: ${{ contains(matrix.test, 'prover') && '64core-tester-x86' || '8core-tester-x86' }} | |
| builder_type: builder-x86 | |
| # these are copied to the tester and expected by the earthly command below | |
| # if they fail to copy, it will try to build them on the tester and fail | |
| builder_images_to_copy: aztecprotocol/aztec:${{ env.GIT_COMMIT }} aztecprotocol/end-to-end:${{ env.GIT_COMMIT }} | |
| # command to produce the images in case they don't exist | |
| builder_command: scripts/earthly-ci ./yarn-project+export-e2e-test-images | |
| run: | | |
| set -eux | |
| cd ./yarn-project/end-to-end/ | |
| export FORCE_COLOR=1 | |
| ../../scripts/earthly-ci -P --no-output +${{ matrix.test }} | |
| # all the benchmarking end-to-end integration tests for aztec (not required to merge) | |
| bench-e2e: | |
| needs: build | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| test: ${{ fromJson( needs.build.outputs.bench_list )}} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: { ref: "${{ env.GIT_COMMIT }}" } | |
| - uses: ./.github/ci-setup-action | |
| - name: Setup and Test | |
| uses: ./.github/ensure-tester-with-images | |
| timeout-minutes: 40 | |
| with: | |
| runner_type: ${{ contains(matrix.test, 'prover') && '64core-tester-x86' || '16core-tester-x86' }} | |
| builder_type: builder-x86 | |
| # these are copied to the tester and expected by the earthly command below | |
| # if they fail to copy, it will try to build them on the tester and fail | |
| builder_images_to_copy: aztecprotocol/aztec:${{ env.GIT_COMMIT }} aztecprotocol/end-to-end:${{ env.GIT_COMMIT }} | |
| # command to produce the images in case they don't exist | |
| builder_command: cd yarn-project/end-to-end/ && ../../scripts/earthly-ci +${{ matrix.test }} | |
| run: | | |
| set -eux | |
| cd ./yarn-project/end-to-end/ | |
| export FORCE_COLOR=1 | |
| export EARTHLY_BUILD_ARGS="${{ env.EARTHLY_BUILD_ARGS }}" | |
| ../../scripts/earthly-ci -P \ | |
| --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \ | |
| --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \ | |
| --no-output \ | |
| +${{ matrix.test }} | |
| bench-summary: | |
| needs: bench-e2e | |
| runs-on: ${{ inputs.username || github.actor }}-x86 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 100 # Downloading base benchmark from master requires access to history | |
| ref: "${{ github.event.pull_request.head.sha }}" | |
| - uses: ./.github/ci-setup-action | |
| with: | |
| dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" | |
| concurrency_key: build-x86 | |
| - name: "Build and upload bench aggregate file" | |
| working-directory: ./yarn-project/scripts | |
| run: earthly-ci -P --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} +bench-aggregate | |
| - name: "Download base benchmark" | |
| if: ${{ github.event_name == 'pull_request' }} | |
| run: scripts/logs/download_base_benchmark_from_s3.sh | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| BENCH_FOLDER: ./yarn-project/scripts/bench | |
| PULL_REQUEST: "${{ github.event.pull_request.number }}" | |
| - name: "Generate summary comment if pull request" | |
| if: ${{ github.event_name == 'pull_request' }} | |
| working-directory: ./yarn-project/scripts | |
| run: earthly-ci -P --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} --secret AZTEC_BOT_COMMENTER_GITHUB_TOKEN=${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} +bench-comment | |
| noir-format: | |
| needs: setup | |
| runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: { ref: "${{ env.GIT_COMMIT }}" } | |
| - uses: ./.github/ci-setup-action | |
| with: | |
| concurrency_key: noir-format-x86 | |
| - name: "Format Noir" | |
| working-directory: ./noir/ | |
| timeout-minutes: 40 | |
| run: earthly-ci --no-output ./+format | |
| - name: "Format noir-projects" | |
| working-directory: ./noir-projects/ | |
| timeout-minutes: 40 | |
| run: earthly-ci --no-output ./+format | |
| # barretenberg (prover) native and AVM (public VM) tests | |
| # only ran on x86 for resource reasons (memory intensive) | |
| bb-native-tests: | |
| needs: setup | |
| runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: { ref: "${{ env.GIT_COMMIT }}" } | |
| # Only allow one memory-hunger prover test to use this runner | |
| - uses: ./.github/ci-setup-action | |
| with: | |
| # must be globally unique for build x runner | |
| concurrency_key: bb-native-tests-x86 | |
| - name: "Native Prover Tests" | |
| working-directory: ./barretenberg/cpp/ | |
| timeout-minutes: 40 | |
| # limit our parallelism to half our cores | |
| run: earthly-ci --no-output +test --hardware_concurrency=64 | |
| bb-js-test: | |
| needs: setup | |
| runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: { ref: "${{ env.GIT_COMMIT }}" } | |
| - uses: ./.github/ci-setup-action | |
| with: | |
| concurrency_key: bb-js-test-x86 | |
| - name: "bb.js Tests" | |
| working-directory: ./barretenberg/ts/ | |
| timeout-minutes: 40 | |
| run: earthly-ci --no-output ./+test | |
| noir-test: | |
| needs: setup | |
| runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: { ref: "${{ env.GIT_COMMIT }}" } | |
| - uses: ./.github/ci-setup-action | |
| with: | |
| concurrency_key: noir-x86 | |
| - name: "Test Noir JS packages" | |
| run: earthly-ci --no-output ./noir+test | |
| noir-packages-test: | |
| needs: setup | |
| runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: { ref: "${{ env.GIT_COMMIT }}" } | |
| - uses: ./.github/ci-setup-action | |
| with: | |
| concurrency_key: noir-packages-x86 | |
| - name: "Test Noir JS packages" | |
| run: earthly-ci --no-output ./noir+packages-test | |
| noir-projects: | |
| needs: setup | |
| runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: { ref: "${{ env.GIT_COMMIT }}" } | |
| - uses: ./.github/ci-setup-action | |
| with: | |
| concurrency_key: noir-projects-x86 | |
| - name: "Noir Projects" | |
| timeout-minutes: 40 | |
| run: earthly-ci --no-output ./noir-projects/+test | |
| yarn-project-formatting: | |
| needs: noir-projects | |
| runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: { ref: "${{ env.GIT_COMMIT }}" } | |
| # Only allow one memory-hunger prover test to use this runner | |
| - uses: ./.github/ci-setup-action | |
| with: | |
| concurrency_key: yarn-project-formatting-x86 | |
| - name: "Yarn Project Tests" | |
| timeout-minutes: 40 | |
| run: earthly-ci --no-output ./yarn-project/+format-check | |
| yarn-project-test: | |
| # quiet machine | |
| needs: [noir-projects, build, bb-native-tests] | |
| runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: { ref: "${{ env.GIT_COMMIT }}" } | |
| # Only allow one memory-hunger prover test to use this runner | |
| - uses: ./.github/ci-setup-action | |
| with: | |
| concurrency_key: yarn-project-test-x86 | |
| - name: "Yarn Project Tests" | |
| timeout-minutes: 40 | |
| run: earthly-ci --no-output ./yarn-project/+test | |
| prover-client-test: | |
| needs: noir-projects | |
| runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: { ref: "${{ env.GIT_COMMIT }}" } | |
| - uses: ./.github/ci-setup-action | |
| with: | |
| concurrency_key: prover-client-test-x86 | |
| - name: "Prover Client Tests" | |
| timeout-minutes: 40 | |
| run: earthly-ci --no-output ./yarn-project/+prover-client-test | |
| build-acir-tests: | |
| needs: build | |
| runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: { ref: "${{ env.GIT_COMMIT }}" } | |
| - uses: ./.github/ci-setup-action | |
| with: | |
| concurrency_key: build-acir-tests-x86 | |
| - name: "Build Acir Tests" | |
| timeout-minutes: 40 | |
| run: earthly-ci --no-output ./noir/+build-acir-tests | |
| barretenberg-acir-tests-bb: | |
| needs: build-acir-tests | |
| runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: { ref: "${{ env.GIT_COMMIT }}" } | |
| - uses: ./.github/ci-setup-action | |
| with: | |
| concurrency_key: barretenberg-acir-tests-bb-x86 | |
| - name: "BB Native Acir Tests" | |
| working-directory: ./noir/ | |
| timeout-minutes: 40 | |
| run: earthly-ci --no-output ./+barretenberg-acir-tests-bb | |
| barretenberg-acir-tests-sol: | |
| needs: build-acir-tests | |
| runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: { ref: "${{ env.GIT_COMMIT }}" } | |
| - uses: ./.github/ci-setup-action | |
| with: | |
| concurrency_key: barretenberg-acir-tests-sol-x86 | |
| - name: "BB Solidity Acir Tests" | |
| working-directory: ./noir/ | |
| timeout-minutes: 40 | |
| run: earthly-ci --no-output ./+barretenberg-acir-tests-sol | |
| barretenberg-acir-tests-bb-js: | |
| needs: build-acir-tests | |
| runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: { ref: "${{ env.GIT_COMMIT }}" } | |
| - uses: ./.github/ci-setup-action | |
| with: | |
| concurrency_key: barretenberg-acir-tests-bb-js-x86 | |
| - name: "BB JS Acir Tests" | |
| working-directory: ./noir/ | |
| timeout-minutes: 40 | |
| run: earthly-ci --no-output ./+barretenberg-acir-tests-bb.js | |
| l1-contracts-test: | |
| needs: setup | |
| runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: { ref: "${{ env.GIT_COMMIT }}" } | |
| - uses: ./.github/ci-setup-action | |
| with: | |
| concurrency_key: l1-contracts-${{ github.event.pull_request.user.login || github.actor }}-x86 | |
| - name: "Test l1 contracts" | |
| run: earthly-ci --no-output ./l1-contracts+test | |
| docs-preview: | |
| needs: build | |
| runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 | |
| if: github.event.number | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: { ref: "${{ env.GIT_COMMIT }}" } | |
| - uses: ./.github/ci-setup-action | |
| with: | |
| concurrency_key: docs-preview-x86 | |
| - name: "Docs Preview" | |
| timeout-minutes: 40 | |
| run: earthly-ci --no-output ./docs/+deploy-preview --PR=${{ github.event.number }} --AZTEC_BOT_COMMENTER_GITHUB_TOKEN=${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} --NETLIFY_AUTH_TOKEN=${{ secrets.NETLIFY_AUTH_TOKEN }} --NETLIFY_SITE_ID=${{ secrets.NETLIFY_SITE_ID }} | |
| bb-bench: | |
| runs-on: ubuntu-latest | |
| needs: setup | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: { ref: "${{ env.GIT_COMMIT }}" } | |
| - uses: ./.github/ci-setup-action | |
| - name: Build Bench Binaries | |
| uses: ./.github/ensure-builder | |
| with: | |
| runner_type: builder-x86 | |
| run: | | |
| set -eux | |
| echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u aztecprotocolci --password-stdin | |
| scripts/earthly-ci --push ./barretenberg/cpp/+bench-binaries | |
| - name: Run Bench | |
| uses: ./.github/ensure-tester | |
| timeout-minutes: 40 | |
| with: | |
| runner_type: 16core-tester-x86 | |
| run: | | |
| scripts/earthly-ci --no-output ./barretenberg/cpp/+bench-client-ivc --bench_mode=cache | |
| scripts/earthly-ci --no-output ./barretenberg/cpp/+bench-ultra-honk --bench_mode=cache | |
| protocol-circuits-gates-report: | |
| needs: setup | |
| runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 | |
| permissions: | |
| pull-requests: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: { ref: "${{ env.GIT_COMMIT }}" } | |
| # Only allow one memory-hunger prover test to use this runner | |
| - uses: ./.github/ci-setup-action | |
| with: | |
| concurrency_key: protocol-circuits-gates-report-x86 | |
| - name: "Noir Protocol Circuits Report" | |
| working-directory: ./noir-projects/ | |
| timeout-minutes: 40 | |
| run: | | |
| earthly-ci --artifact +gates-report/gates_report.json | |
| mv gates_report.json ../protocol_circuits_report.json | |
| - name: Compare gates reports | |
| id: gates_diff | |
| uses: vezenovm/noir-gates-diff@acf12797860f237117e15c0d6e08d64253af52b6 | |
| with: | |
| report: protocol_circuits_report.json | |
| summaryQuantile: 0 # Display any diff in gate count | |
| - name: Add gates diff to sticky comment | |
| if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target' | |
| uses: marocchino/sticky-pull-request-comment@v2 | |
| with: | |
| # delete the comment in case changes no longer impact circuit sizes | |
| delete: ${{ !steps.gates_diff.outputs.markdown }} | |
| message: ${{ steps.gates_diff.outputs.markdown }} | |
| merge-check: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - setup | |
| - e2e | |
| - bb-native-tests | |
| - bb-bench | |
| - yarn-project-formatting | |
| - yarn-project-test | |
| - prover-client-test | |
| - bb-js-test | |
| - barretenberg-acir-tests-bb-js | |
| - barretenberg-acir-tests-bb | |
| - barretenberg-acir-tests-sol | |
| - noir-test | |
| - noir-packages-test | |
| if: always() | |
| steps: | |
| - run: | | |
| failed=${{ contains(needs.*.result, 'failure') }} | |
| if $failed | |
| then | |
| echo "At least one job failed, merging not allowed." | |
| exit 1 | |
| fi | |
| echo "All jobs succeeded, merge allowed." | |
| notify: | |
| needs: | |
| - merge-check | |
| runs-on: ubuntu-latest | |
| if: ${{ github.ref == 'refs/heads/master' && failure() }} | |
| steps: | |
| - name: Send notification to aztec3-ci channel if workflow failed on master | |
| uses: slackapi/[email protected] | |
| with: | |
| payload: | | |
| { | |
| "url": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFY_WORKFLOW_TRIGGER_URL }} |