diff --git a/.github/workflows/bberg_bench.yml b/.github/workflows/bberg_bench.yml deleted file mode 100644 index 436f5766ab..0000000000 --- a/.github/workflows/bberg_bench.yml +++ /dev/null @@ -1,65 +0,0 @@ -name: Barretenberg Benchmarks - -# For now, run basically whenever anything happens in a PR -on: - pull_request: - types: ["labeled", "opened", "synchronize", "reopened"] - -# This will cancel previous runs when a branch or PR is updated -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.ref || github.run_id }} - cancel-in-progress: true - -jobs: - bberg-bench: - name: Barretenberg Benchmarks - runs-on: ubuntu-latest # run in linux environment - - # env: - # BASE_DIR: ${{ github.workspace }}/cpp - # BUILD_DIR: ${{ github.workspace }}/cpp/build-bench - # BENCH_RESULTS_DIR: ${{ github.workspace }}/cpp/src/barretenberg/benchmark/honk_bench - - steps: - - - name: Checkout barretenberg - uses: actions/checkout@v3 - with: - ref: ${{ github.head_ref }} # checkout HEAD of triggering branch - token: ${{ secrets.GITHUB_TOKEN }} - - # Adding some credentials to avoid complaints from git - - name: Configure git - run: | - git config user.name ledwards2225 - git config user.email l.edwards.d@gmail.com - - # Only run in Linux environment for now - - name: Setup Linux environment - run: | - sudo apt update - sudo apt install libomp-dev cmake ninja-build - - - name: Download SRS elements - working-directory: cpp/srs_db - run: ./download_ignition.sh 3 # only download first 4 transcript files - - - name: Build Honk benchmarks - working-directory: cpp - run: | - cmake --preset bench - cmake --build --preset bench --target honk_bench - - - name: Run Honk benchmarks - working-directory: cpp/build-bench - run: | - bin/honk_bench --benchmark_format=json > ../src/barretenberg/benchmark/honk_bench/bench_results.json - - # # Commit benchmark results json to the branch - # - name: Commit result - # working-directory: cpp - # run: | - # git add src/barretenberg/benchmark/honk_bench/bench_results.json - # git commit -m "add bench results" - # git push - \ No newline at end of file diff --git a/barretenberg.code-workspace b/barretenberg.code-workspace index 0d0d3106e5..bf2de10d0e 100644 --- a/barretenberg.code-workspace +++ b/barretenberg.code-workspace @@ -115,7 +115,7 @@ // which ensures SRS can be read "testMate.cpp.test.workingDirectory": "${workspaceFolder}/cpp/build", // Filter all binaries that are not tests or benchmarks - "testMate.cpp.test.executables": "${workspaceFolder}/cpp/{build}/bin/*{test,bench}*", + "testMate.cpp.test.executables": "${workspaceFolder}/cpp/{build}/bin/*{test,Test,TEST,bench}*", // // Other // diff --git a/cpp/src/barretenberg/benchmark/honk_bench/bench_results.json b/cpp/src/barretenberg/benchmark/honk_bench/bench_results.json deleted file mode 100644 index 67285852ad..0000000000 --- a/cpp/src/barretenberg/benchmark/honk_bench/bench_results.json +++ /dev/null @@ -1,614 +0,0 @@ -{ - "context": { - "date": "2023-05-26T19:53:29+00:00", - "host_name": "fv-az491-631", - "executable": "bin/honk_bench", - "num_cpus": 2, - "mhz_per_cpu": 2594, - "cpu_scaling_enabled": false, - "caches": [ - { - "type": "Data", - "level": 1, - "size": 32768, - "num_sharing": 1 - }, - { - "type": "Instruction", - "level": 1, - "size": 32768, - "num_sharing": 1 - }, - { - "type": "Unified", - "level": 2, - "size": 1048576, - "num_sharing": 1 - }, - { - "type": "Unified", - "level": 3, - "size": 37486592, - "num_sharing": 2 - } - ], - "load_avg": [2.71338,2.04248,0.973633], - "library_build_type": "release" - }, - "benchmarks": [ - { - "name": "create_prover_standard/16/repeats:5", - "family_index": 0, - "per_family_instance_index": 0, - "run_name": "create_prover_standard/16/repeats:5", - "run_type": "iteration", - "repetitions": 5, - "repetition_index": 0, - "threads": 1, - "iterations": 12, - "real_time": 5.5504216499988012e+07, - "cpu_time": 5.5428016666666664e+07, - "time_unit": "ns" - }, - { - "name": "create_prover_standard/16/repeats:5", - "family_index": 0, - "per_family_instance_index": 0, - "run_name": "create_prover_standard/16/repeats:5", - "run_type": "iteration", - "repetitions": 5, - "repetition_index": 1, - "threads": 1, - "iterations": 12, - "real_time": 5.7801632166667603e+07, - "cpu_time": 5.7482741666666597e+07, - "time_unit": "ns" - }, - { - "name": "create_prover_standard/16/repeats:5", - "family_index": 0, - "per_family_instance_index": 0, - "run_name": "create_prover_standard/16/repeats:5", - "run_type": "iteration", - "repetitions": 5, - "repetition_index": 2, - "threads": 1, - "iterations": 12, - "real_time": 5.7450450666668952e+07, - "cpu_time": 5.7233533333333291e+07, - "time_unit": "ns" - }, - { - "name": "create_prover_standard/16/repeats:5", - "family_index": 0, - "per_family_instance_index": 0, - "run_name": "create_prover_standard/16/repeats:5", - "run_type": "iteration", - "repetitions": 5, - "repetition_index": 3, - "threads": 1, - "iterations": 12, - "real_time": 5.7718005583339505e+07, - "cpu_time": 5.7534408333333343e+07, - "time_unit": "ns" - }, - { - "name": "create_prover_standard/16/repeats:5", - "family_index": 0, - "per_family_instance_index": 0, - "run_name": "create_prover_standard/16/repeats:5", - "run_type": "iteration", - "repetitions": 5, - "repetition_index": 4, - "threads": 1, - "iterations": 12, - "real_time": 5.7766381500006974e+07, - "cpu_time": 5.7489225000000007e+07, - "time_unit": "ns" - }, - { - "name": "create_prover_standard/16/repeats:5_mean", - "family_index": 0, - "per_family_instance_index": 0, - "run_name": "create_prover_standard/16/repeats:5", - "run_type": "aggregate", - "repetitions": 5, - "threads": 1, - "aggregate_name": "mean", - "aggregate_unit": "time", - "iterations": 5, - "real_time": 5.7248137283334211e+07, - "cpu_time": 5.7033584999999985e+07, - "time_unit": "ns" - }, - { - "name": "create_prover_standard/16/repeats:5_median", - "family_index": 0, - "per_family_instance_index": 0, - "run_name": "create_prover_standard/16/repeats:5", - "run_type": "aggregate", - "repetitions": 5, - "threads": 1, - "aggregate_name": "median", - "aggregate_unit": "time", - "iterations": 5, - "real_time": 5.7718005583339497e+07, - "cpu_time": 5.7482741666666605e+07, - "time_unit": "ns" - }, - { - "name": "create_prover_standard/16/repeats:5_stddev", - "family_index": 0, - "per_family_instance_index": 0, - "run_name": "create_prover_standard/16/repeats:5", - "run_type": "aggregate", - "repetitions": 5, - "threads": 1, - "aggregate_name": "stddev", - "aggregate_unit": "time", - "iterations": 5, - "real_time": 9.8461925090377196e+05, - "cpu_time": 9.0526274376913963e+05, - "time_unit": "ns" - }, - { - "name": "create_prover_standard/16/repeats:5_cv", - "family_index": 0, - "per_family_instance_index": 0, - "run_name": "create_prover_standard/16/repeats:5", - "run_type": "aggregate", - "repetitions": 5, - "threads": 1, - "aggregate_name": "cv", - "aggregate_unit": "percentage", - "iterations": 5, - "real_time": 1.7199149136166030e-02, - "cpu_time": 1.5872450307466028e-02, - "time_unit": "ns" - }, - { - "name": "construct_proof_standard/16/repeats:5", - "family_index": 1, - "per_family_instance_index": 0, - "run_name": "construct_proof_standard/16/repeats:5", - "run_type": "iteration", - "repetitions": 5, - "repetition_index": 0, - "threads": 1, - "iterations": 1, - "real_time": 4.5438027659999990e+09, - "cpu_time": 3.2067761999999995e+09, - "time_unit": "ns" - }, - { - "name": "construct_proof_standard/16/repeats:5", - "family_index": 1, - "per_family_instance_index": 0, - "run_name": "construct_proof_standard/16/repeats:5", - "run_type": "iteration", - "repetitions": 5, - "repetition_index": 1, - "threads": 1, - "iterations": 1, - "real_time": 3.2842767270000196e+09, - "cpu_time": 1.9386794000000017e+09, - "time_unit": "ns" - }, - { - "name": "construct_proof_standard/16/repeats:5", - "family_index": 1, - "per_family_instance_index": 0, - "run_name": "construct_proof_standard/16/repeats:5", - "run_type": "iteration", - "repetitions": 5, - "repetition_index": 2, - "threads": 1, - "iterations": 1, - "real_time": 3.3388418679999747e+09, - "cpu_time": 1.9140537000000002e+09, - "time_unit": "ns" - }, - { - "name": "construct_proof_standard/16/repeats:5", - "family_index": 1, - "per_family_instance_index": 0, - "run_name": "construct_proof_standard/16/repeats:5", - "run_type": "iteration", - "repetitions": 5, - "repetition_index": 3, - "threads": 1, - "iterations": 1, - "real_time": 3.2109865110000443e+09, - "cpu_time": 1.9802719000000000e+09, - "time_unit": "ns" - }, - { - "name": "construct_proof_standard/16/repeats:5", - "family_index": 1, - "per_family_instance_index": 0, - "run_name": "construct_proof_standard/16/repeats:5", - "run_type": "iteration", - "repetitions": 5, - "repetition_index": 4, - "threads": 1, - "iterations": 1, - "real_time": 3.3167836069999909e+09, - "cpu_time": 1.9449394000000010e+09, - "time_unit": "ns" - }, - { - "name": "construct_proof_standard/16/repeats:5_mean", - "family_index": 1, - "per_family_instance_index": 0, - "run_name": "construct_proof_standard/16/repeats:5", - "run_type": "aggregate", - "repetitions": 5, - "threads": 1, - "aggregate_name": "mean", - "aggregate_unit": "time", - "iterations": 5, - "real_time": 3.5389382958000054e+09, - "cpu_time": 2.1969441200000005e+09, - "time_unit": "ns" - }, - { - "name": "construct_proof_standard/16/repeats:5_median", - "family_index": 1, - "per_family_instance_index": 0, - "run_name": "construct_proof_standard/16/repeats:5", - "run_type": "aggregate", - "repetitions": 5, - "threads": 1, - "aggregate_name": "median", - "aggregate_unit": "time", - "iterations": 5, - "real_time": 3.3167836069999909e+09, - "cpu_time": 1.9449394000000007e+09, - "time_unit": "ns" - }, - { - "name": "construct_proof_standard/16/repeats:5_stddev", - "family_index": 1, - "per_family_instance_index": 0, - "run_name": "construct_proof_standard/16/repeats:5", - "run_type": "aggregate", - "repetitions": 5, - "threads": 1, - "aggregate_name": "stddev", - "aggregate_unit": "time", - "iterations": 5, - "real_time": 5.6381485822511280e+08, - "cpu_time": 5.6500922889444625e+08, - "time_unit": "ns" - }, - { - "name": "construct_proof_standard/16/repeats:5_cv", - "family_index": 1, - "per_family_instance_index": 0, - "run_name": "construct_proof_standard/16/repeats:5", - "run_type": "aggregate", - "repetitions": 5, - "threads": 1, - "aggregate_name": "cv", - "aggregate_unit": "percentage", - "iterations": 5, - "real_time": 1.5931751590420365e-01, - "cpu_time": 2.5717960859853195e-01, - "time_unit": "ns" - }, - { - "name": "construct_proof_standard/repeats:5_BigO", - "family_index": 1, - "per_family_instance_index": 0, - "run_name": "construct_proof_standard/repeats:5", - "run_type": "aggregate", - "repetitions": 5, - "threads": 1, - "aggregate_name": "BigO", - "aggregate_unit": "time", - "cpu_coefficient": 3.3522706909179695e+04, - "real_coefficient": 5.3999912960815520e+04, - "big_o": "N", - "time_unit": "ns" - }, - { - "name": "construct_proof_standard/repeats:5_RMS", - "family_index": 1, - "per_family_instance_index": 0, - "run_name": "construct_proof_standard/repeats:5", - "run_type": "aggregate", - "repetitions": 5, - "threads": 1, - "aggregate_name": "RMS", - "aggregate_unit": "percentage", - "rms": 2.3002843490124258e-01 - }, - { - "name": "create_prover_ultra/16/repeats:5", - "family_index": 2, - "per_family_instance_index": 0, - "run_name": "create_prover_ultra/16/repeats:5", - "run_type": "iteration", - "repetitions": 5, - "repetition_index": 0, - "threads": 1, - "iterations": 3, - "real_time": 1.8232255199999750e+08, - "cpu_time": 1.8199536666666678e+08, - "time_unit": "ns" - }, - { - "name": "create_prover_ultra/16/repeats:5", - "family_index": 2, - "per_family_instance_index": 0, - "run_name": "create_prover_ultra/16/repeats:5", - "run_type": "iteration", - "repetitions": 5, - "repetition_index": 1, - "threads": 1, - "iterations": 3, - "real_time": 1.8351760733330974e+08, - "cpu_time": 1.8311056666666779e+08, - "time_unit": "ns" - }, - { - "name": "create_prover_ultra/16/repeats:5", - "family_index": 2, - "per_family_instance_index": 0, - "run_name": "create_prover_ultra/16/repeats:5", - "run_type": "iteration", - "repetitions": 5, - "repetition_index": 2, - "threads": 1, - "iterations": 3, - "real_time": 1.8638982566667056e+08, - "cpu_time": 1.8603030000000110e+08, - "time_unit": "ns" - }, - { - "name": "create_prover_ultra/16/repeats:5", - "family_index": 2, - "per_family_instance_index": 0, - "run_name": "create_prover_ultra/16/repeats:5", - "run_type": "iteration", - "repetitions": 5, - "repetition_index": 3, - "threads": 1, - "iterations": 3, - "real_time": 1.8593535066668209e+08, - "cpu_time": 1.8553553333333519e+08, - "time_unit": "ns" - }, - { - "name": "create_prover_ultra/16/repeats:5", - "family_index": 2, - "per_family_instance_index": 0, - "run_name": "create_prover_ultra/16/repeats:5", - "run_type": "iteration", - "repetitions": 5, - "repetition_index": 4, - "threads": 1, - "iterations": 3, - "real_time": 1.8335277099998847e+08, - "cpu_time": 1.8314713333333185e+08, - "time_unit": "ns" - }, - { - "name": "create_prover_ultra/16/repeats:5_mean", - "family_index": 2, - "per_family_instance_index": 0, - "run_name": "create_prover_ultra/16/repeats:5", - "run_type": "aggregate", - "repetitions": 5, - "threads": 1, - "aggregate_name": "mean", - "aggregate_unit": "time", - "iterations": 5, - "real_time": 1.8430362133332968e+08, - "cpu_time": 1.8396378000000057e+08, - "time_unit": "ns" - }, - { - "name": "create_prover_ultra/16/repeats:5_median", - "family_index": 2, - "per_family_instance_index": 0, - "run_name": "create_prover_ultra/16/repeats:5", - "run_type": "aggregate", - "repetitions": 5, - "threads": 1, - "aggregate_name": "median", - "aggregate_unit": "time", - "iterations": 5, - "real_time": 1.8351760733330971e+08, - "cpu_time": 1.8314713333333188e+08, - "time_unit": "ns" - }, - { - "name": "create_prover_ultra/16/repeats:5_stddev", - "family_index": 2, - "per_family_instance_index": 0, - "run_name": "create_prover_ultra/16/repeats:5", - "run_type": "aggregate", - "repetitions": 5, - "threads": 1, - "aggregate_name": "stddev", - "aggregate_unit": "time", - "iterations": 5, - "real_time": 1.7650325213651452e+06, - "cpu_time": 1.7328047547552765e+06, - "time_unit": "ns" - }, - { - "name": "create_prover_ultra/16/repeats:5_cv", - "family_index": 2, - "per_family_instance_index": 0, - "run_name": "create_prover_ultra/16/repeats:5", - "run_type": "aggregate", - "repetitions": 5, - "threads": 1, - "aggregate_name": "cv", - "aggregate_unit": "percentage", - "iterations": 5, - "real_time": 9.5767652778396863e-03, - "cpu_time": 9.4192713084894786e-03, - "time_unit": "ns" - }, - { - "name": "construct_proof_ultra/16/repeats:5", - "family_index": 3, - "per_family_instance_index": 0, - "run_name": "construct_proof_ultra/16/repeats:5", - "run_type": "iteration", - "repetitions": 5, - "repetition_index": 0, - "threads": 1, - "iterations": 1, - "real_time": 1.0382158091999998e+10, - "cpu_time": 8.1326265999999981e+09, - "time_unit": "ns" - }, - { - "name": "construct_proof_ultra/16/repeats:5", - "family_index": 3, - "per_family_instance_index": 0, - "run_name": "construct_proof_ultra/16/repeats:5", - "run_type": "iteration", - "repetitions": 5, - "repetition_index": 1, - "threads": 1, - "iterations": 1, - "real_time": 1.0256200592000027e+10, - "cpu_time": 8.0500114000000057e+09, - "time_unit": "ns" - }, - { - "name": "construct_proof_ultra/16/repeats:5", - "family_index": 3, - "per_family_instance_index": 0, - "run_name": "construct_proof_ultra/16/repeats:5", - "run_type": "iteration", - "repetitions": 5, - "repetition_index": 2, - "threads": 1, - "iterations": 1, - "real_time": 1.0365316951000011e+10, - "cpu_time": 8.0008304000000048e+09, - "time_unit": "ns" - }, - { - "name": "construct_proof_ultra/16/repeats:5", - "family_index": 3, - "per_family_instance_index": 0, - "run_name": "construct_proof_ultra/16/repeats:5", - "run_type": "iteration", - "repetitions": 5, - "repetition_index": 3, - "threads": 1, - "iterations": 1, - "real_time": 1.0341458550999960e+10, - "cpu_time": 8.1200064000000086e+09, - "time_unit": "ns" - }, - { - "name": "construct_proof_ultra/16/repeats:5", - "family_index": 3, - "per_family_instance_index": 0, - "run_name": "construct_proof_ultra/16/repeats:5", - "run_type": "iteration", - "repetitions": 5, - "repetition_index": 4, - "threads": 1, - "iterations": 1, - "real_time": 1.0240077385999996e+10, - "cpu_time": 8.0946810999999952e+09, - "time_unit": "ns" - }, - { - "name": "construct_proof_ultra/16/repeats:5_mean", - "family_index": 3, - "per_family_instance_index": 0, - "run_name": "construct_proof_ultra/16/repeats:5", - "run_type": "aggregate", - "repetitions": 5, - "threads": 1, - "aggregate_name": "mean", - "aggregate_unit": "time", - "iterations": 5, - "real_time": 1.0317042314399998e+10, - "cpu_time": 8.0796311800000048e+09, - "time_unit": "ns" - }, - { - "name": "construct_proof_ultra/16/repeats:5_median", - "family_index": 3, - "per_family_instance_index": 0, - "run_name": "construct_proof_ultra/16/repeats:5", - "run_type": "aggregate", - "repetitions": 5, - "threads": 1, - "aggregate_name": "median", - "aggregate_unit": "time", - "iterations": 5, - "real_time": 1.0341458550999960e+10, - "cpu_time": 8.0946810999999952e+09, - "time_unit": "ns" - }, - { - "name": "construct_proof_ultra/16/repeats:5_stddev", - "family_index": 3, - "per_family_instance_index": 0, - "run_name": "construct_proof_ultra/16/repeats:5", - "run_type": "aggregate", - "repetitions": 5, - "threads": 1, - "aggregate_name": "stddev", - "aggregate_unit": "time", - "iterations": 5, - "real_time": 6.4791922778365932e+07, - "cpu_time": 5.4203476177260555e+07, - "time_unit": "ns" - }, - { - "name": "construct_proof_ultra/16/repeats:5_cv", - "family_index": 3, - "per_family_instance_index": 0, - "run_name": "construct_proof_ultra/16/repeats:5", - "run_type": "aggregate", - "repetitions": 5, - "threads": 1, - "aggregate_name": "cv", - "aggregate_unit": "percentage", - "iterations": 5, - "real_time": 6.2800869477808276e-03, - "cpu_time": 6.7086572356710641e-03, - "time_unit": "ns" - }, - { - "name": "construct_proof_ultra/repeats:5_BigO", - "family_index": 3, - "per_family_instance_index": 0, - "run_name": "construct_proof_ultra/repeats:5", - "run_type": "aggregate", - "repetitions": 5, - "threads": 1, - "aggregate_name": "BigO", - "aggregate_unit": "time", - "cpu_coefficient": 1.2328538787841803e+05, - "real_coefficient": 1.5742557242431637e+05, - "big_o": "N", - "time_unit": "ns" - }, - { - "name": "construct_proof_ultra/repeats:5_RMS", - "family_index": 3, - "per_family_instance_index": 0, - "run_name": "construct_proof_ultra/repeats:5", - "run_type": "aggregate", - "repetitions": 5, - "threads": 1, - "aggregate_name": "RMS", - "aggregate_unit": "percentage", - "rms": 6.0004054467059538e-03 - } - ] -}