chore: Reorganize benchmarks

barretenberg/.github/workflows/benchmarks.yml

@@ -43,7 +43,7 @@ jobs:
       - name: Run Honk benchmarks
         working-directory: cpp/build-bench
         run: |
-          bin/ultra_honk_bench --benchmark_format=json > ../src/barretenberg/benchmark/honk_bench/bench_results.json
+          bin/ultra_honk_bench --benchmark_format=json > ../src/barretenberg/benchmark/ultra_bench/bench_results.json
 
       # Utilize github-action-benchmark to automatically update the plots at 
       # https://aztecprotocol.github.io/barretenberg/dev/bench/ with new benchmark data.
@@ -53,7 +53,7 @@ jobs:
         with:
           name: C++ Benchmark
           tool: 'googlecpp'
-          output-file-path: cpp/src/barretenberg/benchmark/honk_bench/bench_results.json
+          output-file-path: cpp/src/barretenberg/benchmark/ultra_bench/bench_results.json
           # Access token to deploy GitHub Pages branch
           github-token: ${{ secrets.GITHUB_TOKEN }}
           # Push and deploy GitHub pages branch automatically

barretenberg/cpp/src/barretenberg/benchmark/CMakeLists.txt

@@ -1,6 +1,7 @@
 add_subdirectory(decrypt_bench)
 add_subdirectory(pippenger_bench)
 add_subdirectory(plonk_bench)
-add_subdirectory(honk_bench)
+add_subdirectory(ultra_bench)
+add_subdirectory(goblin_bench)
 add_subdirectory(relations_bench)
 add_subdirectory(widgets_bench)

barretenberg/cpp/src/barretenberg/benchmark/goblin_bench/CMakeLists.txt

@@ -0,0 +1,21 @@
+# Each source represents a separate benchmark suite 
+set(BENCHMARK_SOURCES
+  goblin.bench.cpp
+  eccvm.bench.cpp
+)
+
+# Required libraries for benchmark suites
+set(LINKED_LIBRARIES
+  ultra_honk
+  eccvm
+  stdlib_recursion
+  benchmark::benchmark
+)
+
+# Add executable and custom target for each suite, e.g. ultra_honk_bench
+foreach(BENCHMARK_SOURCE ${BENCHMARK_SOURCES})
+  get_filename_component(BENCHMARK_NAME ${BENCHMARK_SOURCE} NAME_WE) # extract name without extension
+  add_executable(${BENCHMARK_NAME}_bench main.bench.cpp ${BENCHMARK_SOURCE})
+  target_link_libraries(${BENCHMARK_NAME}_bench ${LINKED_LIBRARIES})
+  add_custom_target(run_${BENCHMARK_NAME} COMMAND ${BENCHMARK_NAME} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+endforeach()

barretenberg/cpp/src/barretenberg/benchmark/honk_bench/eccvm.bench.cpp → barretenberg/cpp/src/barretenberg/benchmark/goblin_bench/eccvm.bench.cpp

@@ -1,6 +1,6 @@
 #include <benchmark/benchmark.h>
 
-#include "barretenberg/benchmark/honk_bench/benchmark_utilities.hpp"
+#include "barretenberg/benchmark/ultra_bench/benchmark_utilities.hpp"
 #include "barretenberg/eccvm/eccvm_composer.hpp"
 #include "barretenberg/proof_system/circuit_builder/eccvm/eccvm_circuit_builder.hpp"
 

barretenberg/cpp/src/barretenberg/benchmark/honk_bench/goblin.bench.cpp → barretenberg/cpp/src/barretenberg/benchmark/goblin_bench/goblin.bench.cpp

@@ -1,7 +1,7 @@
 
 #include <benchmark/benchmark.h>
 
-#include "barretenberg/benchmark/honk_bench/benchmark_utilities.hpp"
+#include "barretenberg/benchmark/ultra_bench/benchmark_utilities.hpp"
 #include "barretenberg/goblin/goblin.hpp"
 #include "barretenberg/goblin/mock_circuits.hpp"
 #include "barretenberg/proof_system/circuit_builder/ultra_circuit_builder.hpp"

barretenberg/cpp/src/barretenberg/benchmark/plonk_bench/CMakeLists.txt

@@ -1,13 +1,20 @@
-add_executable(plonk_bench plonk.bench.cpp)
+# Each source represents a separate benchmark suite 
+set(BENCHMARK_SOURCES
+  plonk.bench.cpp
+  standard_plonk.bench.cpp
+)
 
-target_link_libraries(
-  plonk_bench
+# Required libraries for benchmark suites
+set(LINKED_LIBRARIES
+  plonk
   stdlib_primitives
   benchmark::benchmark
 )
 
-add_custom_target(
-    run_plonk_bench
-    COMMAND plonk_bench
-    WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
-)
+# Add executable and custom target for each suite, e.g. ultra_honk_bench
+foreach(BENCHMARK_SOURCE ${BENCHMARK_SOURCES})
+  get_filename_component(BENCHMARK_NAME ${BENCHMARK_SOURCE} NAME_WE) # extract name without extension
+  add_executable(${BENCHMARK_NAME}_bench main.bench.cpp ${BENCHMARK_SOURCE})
+  target_link_libraries(${BENCHMARK_NAME}_bench ${LINKED_LIBRARIES})
+  add_custom_target(run_${BENCHMARK_NAME} COMMAND ${BENCHMARK_NAME} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+endforeach()

barretenberg/cpp/src/barretenberg/benchmark/plonk_bench/main.bench.cpp

@@ -0,0 +1,3 @@
+#include <benchmark/benchmark.h>
+
+BENCHMARK_MAIN();

barretenberg/cpp/src/barretenberg/benchmark/plonk_bench/plonk.bench.cpp

@@ -107,6 +107,4 @@ void verify_proofs_bench(State& state) noexcept
         state.ResumeTiming();
     }
 }
-BENCHMARK(verify_proofs_bench)->RangeMultiplier(2)->Range(START, MAX_GATES);
-
-BENCHMARK_MAIN();
+BENCHMARK(verify_proofs_bench)->RangeMultiplier(2)->Range(START, MAX_GATES);

barretenberg/cpp/src/barretenberg/benchmark/honk_bench/standard_plonk.bench.cpp → barretenberg/cpp/src/barretenberg/benchmark/plonk_bench/standard_plonk.bench.cpp

@@ -1,4 +1,4 @@
-#include "barretenberg/benchmark/honk_bench/benchmark_utilities.hpp"
+#include "barretenberg/benchmark/ultra_bench/benchmark_utilities.hpp"
 #include "barretenberg/plonk/composer/standard_composer.hpp"
 #include "barretenberg/proof_system/circuit_builder/standard_circuit_builder.hpp"
 

barretenberg/cpp/src/barretenberg/benchmark/honk_bench/CMakeLists.txt → barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/CMakeLists.txt

@@ -1,8 +1,5 @@
 # Each source represents a separate benchmark suite 
 set(BENCHMARK_SOURCES
-  standard_plonk.bench.cpp
-  goblin.bench.cpp
-  eccvm.bench.cpp
   ultra_honk.bench.cpp
   ultra_honk_rounds.bench.cpp
   ultra_plonk.bench.cpp
@@ -12,7 +9,6 @@ set(BENCHMARK_SOURCES
 # Required libraries for benchmark suites
 set(LINKED_LIBRARIES
   ultra_honk
-  eccvm
   stdlib_sha256
   stdlib_keccak
   stdlib_merkle_tree
@@ -23,7 +19,7 @@ set(LINKED_LIBRARIES
 # Add executable and custom target for each suite, e.g. ultra_honk_bench
 foreach(BENCHMARK_SOURCE ${BENCHMARK_SOURCES})
   get_filename_component(BENCHMARK_NAME ${BENCHMARK_SOURCE} NAME_WE) # extract name without extension
-  add_executable(${BENCHMARK_NAME}_bench main.bench.cpp ${BENCHMARK_SOURCE} benchmark_utilities.hpp)
+  add_executable(${BENCHMARK_NAME}_bench main.bench.cpp ${BENCHMARK_SOURCE})
   target_link_libraries(${BENCHMARK_NAME}_bench ${LINKED_LIBRARIES})
   add_custom_target(run_${BENCHMARK_NAME} COMMAND ${BENCHMARK_NAME} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 endforeach()

barretenberg/cpp/src/barretenberg/benchmark/honk_bench/compare_honk_to_plonk_ultra.sh → barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/compare_honk_to_plonk_ultra.sh

@@ -30,7 +30,7 @@ PLONK_BENCH_RESULTS="$BENCH_RESULTS_DIR/plonk_bench.json"
 cd ..
 cmake --preset bench > /dev/null && cmake --build --preset bench --target ultra_honk_bench
 cd build-bench
-HONK_BENCH_RESULTS="$BENCH_RESULTS_DIR/honk_bench.json"
+HONK_BENCH_RESULTS="$BENCH_RESULTS_DIR/ultra_bench.json"
 ./bin/ultra_honk_bench --benchmark_format=json > $HONK_BENCH_RESULTS
 
 # Call compare.py on the results (json) to get high level statistics.

barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/main.bench.cpp

@@ -0,0 +1,3 @@
+#include <benchmark/benchmark.h>
+
+BENCHMARK_MAIN();

barretenberg/cpp/src/barretenberg/benchmark/honk_bench/ultra_honk.bench.cpp → barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/ultra_honk.bench.cpp

@@ -1,6 +1,6 @@
 #include <benchmark/benchmark.h>
 
-#include "barretenberg/benchmark/honk_bench/benchmark_utilities.hpp"
+#include "barretenberg/benchmark/ultra_bench/benchmark_utilities.hpp"
 #include "barretenberg/proof_system/circuit_builder/ultra_circuit_builder.hpp"
 #include "barretenberg/ultra_honk/ultra_composer.hpp"
 

barretenberg/cpp/src/barretenberg/benchmark/honk_bench/ultra_honk_rounds.bench.cpp → barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/ultra_honk_rounds.bench.cpp

@@ -1,6 +1,6 @@
 #include <benchmark/benchmark.h>
 
-#include "barretenberg/benchmark/honk_bench/benchmark_utilities.hpp"
+#include "barretenberg/benchmark/ultra_bench/benchmark_utilities.hpp"
 #include "barretenberg/proof_system/circuit_builder/ultra_circuit_builder.hpp"
 #include "barretenberg/ultra_honk/ultra_composer.hpp"
 #include "barretenberg/ultra_honk/ultra_prover.hpp"

barretenberg/cpp/src/barretenberg/benchmark/honk_bench/ultra_plonk.bench.cpp → barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/ultra_plonk.bench.cpp

@@ -1,4 +1,4 @@
-#include "barretenberg/benchmark/honk_bench/benchmark_utilities.hpp"
+#include "barretenberg/benchmark/ultra_bench/benchmark_utilities.hpp"
 #include "barretenberg/plonk/composer/ultra_composer.hpp"
 #include "barretenberg/proof_system/circuit_builder/ultra_circuit_builder.hpp"
 

barretenberg/cpp/src/barretenberg/benchmark/honk_bench/ultra_plonk_rounds.bench.cpp → barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/ultra_plonk_rounds.bench.cpp

@@ -1,6 +1,6 @@
 #include <benchmark/benchmark.h>
 
-#include "barretenberg/benchmark/honk_bench/benchmark_utilities.hpp"
+#include "barretenberg/benchmark/ultra_bench/benchmark_utilities.hpp"
 #include "barretenberg/proof_system/circuit_builder/ultra_circuit_builder.hpp"
 
 using namespace benchmark;

barretenberg/cpp/src/barretenberg/benchmark/widgets_bench/widget.bench.cpp

@@ -1,4 +1,4 @@
-#include "barretenberg/benchmark/honk_bench/benchmark_utilities.hpp"
+#include "barretenberg/benchmark/ultra_bench/benchmark_utilities.hpp"
 #include "barretenberg/flavor/goblin_ultra.hpp"
 #include "barretenberg/flavor/ultra.hpp"
 #include "barretenberg/plonk/composer/standard_composer.hpp"