UltraHonk Benchmarks

barretenberg.code-workspace

@@ -87,7 +87,7 @@
         // We should disable automatic inclusion of headers unless we decide to follow "WhyIWYU".
         "clangd.arguments": [
             "-header-insertion=never"
-          ]
+        ],
         //
         // CMake
         //
@@ -114,8 +114,8 @@
         // Ensures tests are run from the `build` directory
         // which ensures SRS can be read
         "testMate.cpp.test.workingDirectory": "${workspaceFolder}/cpp/build",
-        // Filter all binaries that are not tests
-        "testMate.cpp.test.executables": "${workspaceFolder}/cpp/build/bin/*{test,Test,TEST}*",
+        // Filter all binaries that are not tests or benchmarks
+        "testMate.cpp.test.executables": "${workspaceFolder}/cpp/{build}/bin/*{test,Test,TEST,bench}*",
         //
         // Other
         //

cpp/src/barretenberg/benchmark/honk_bench/CMakeLists.txt

@@ -1,8 +1,11 @@
-add_executable(honk_bench honk.bench.cpp)
+add_executable(honk_bench main.bench.cpp honk.bench.cpp ultra_honk.bench.cpp)
 
 target_link_libraries(
   honk_bench
   stdlib_primitives
+  stdlib_sha256
+  stdlib_keccak
+  stdlib_merkle_tree
   env
   benchmark::benchmark
 )

cpp/src/barretenberg/benchmark/honk_bench/compare_honk_bench.sh

@@ -24,6 +24,7 @@ echo -e '\nConfiguring and building honk_bench in master branch..'
 git checkout master > /dev/null
 rm -rf $BUILD_DIR
 cmake --preset bench > /dev/null && cmake --build --preset bench --target honk_bench > /dev/null
+cd build-bench
 MASTER_HONK_BENCH_RESULTS="$BENCH_RESULTS_DIR/honk_bench_results_master.json"
 echo -e '\nRunning honk_bench in master..'
 bin/honk_bench --benchmark_format=json > $MASTER_HONK_BENCH_RESULTS
@@ -33,6 +34,7 @@ echo -e '\nConfiguring and building honk_bench in current feature branch..'
 git checkout -
 rm -rf $BUILD_DIR
 cmake --preset bench > /dev/null && cmake --build --preset bench --target honk_bench > /dev/null
+cd build-bench
 BRANCH_HONK_BENCH_RESULTS="$BENCH_RESULTS_DIR/honk_bench_results_branch.json"
 echo -e '\nRunning honk_bench in feature branch..'
 bin/honk_bench --benchmark_format=json > $BRANCH_HONK_BENCH_RESULTS

cpp/src/barretenberg/benchmark/honk_bench/honk.bench.cpp

@@ -8,12 +8,16 @@
 
 using namespace benchmark;
 
+namespace standard_honk_bench {
+
+using Composer = proof_system::honk::StandardHonkComposer;
+
 constexpr size_t MIN_LOG_NUM_GATES = 16;
 constexpr size_t MAX_LOG_NUM_GATES = 16;
 // To get good statistics, number of Repetitions must be sufficient. ~30 Repetitions gives good results.
-constexpr size_t NUM_REPETITIONS = 30;
+constexpr size_t NUM_REPETITIONS = 5;
 
-void generate_test_plonk_circuit(auto& composer, size_t num_gates)
+void generate_test_circuit(auto& composer, size_t num_gates)
 {
     plonk::stdlib::field_t a(plonk::stdlib::witness_t(&composer, barretenberg::fr::random_element()));
     plonk::stdlib::field_t b(plonk::stdlib::witness_t(&composer, barretenberg::fr::random_element()));
@@ -29,69 +33,70 @@ void generate_test_plonk_circuit(auto& composer, size_t num_gates)
 /**
  * @brief Benchmark: Creation of a Standard Honk prover
  */
-void create_prover_bench(State& state) noexcept
+void create_prover_standard(State& state) noexcept
 {
     for (auto _ : state) {
         state.PauseTiming();
         auto num_gates = 1 << (size_t)state.range(0);
-        auto composer = proof_system::honk::StandardHonkComposer(static_cast<size_t>(num_gates));
-        generate_test_plonk_circuit(composer, static_cast<size_t>(num_gates));
+        auto composer = Composer(static_cast<size_t>(num_gates));
+        generate_test_circuit(composer, static_cast<size_t>(num_gates));
         state.ResumeTiming();
 
         composer.create_prover();
     }
 }
-BENCHMARK(create_prover_bench)->DenseRange(MIN_LOG_NUM_GATES, MAX_LOG_NUM_GATES, 1)->Repetitions(NUM_REPETITIONS);
+BENCHMARK(create_prover_standard)->DenseRange(MIN_LOG_NUM_GATES, MAX_LOG_NUM_GATES, 1)->Repetitions(NUM_REPETITIONS);
 
 /**
- * @brief Benchmark: Creation of a Standard Honk verifier
+ * @brief Benchmark: Construction of a Standard Honk proof
  */
-void create_verifier_bench(State& state) noexcept
+void construct_proof_standard(State& state) noexcept
 {
+    auto num_gates = 1 << (size_t)state.range(0);
     for (auto _ : state) {
         state.PauseTiming();
-        auto num_gates = 1 << (size_t)state.range(0);
-        auto composer = proof_system::honk::StandardHonkComposer(static_cast<size_t>(num_gates));
-        generate_test_plonk_circuit(composer, static_cast<size_t>(num_gates));
+        auto composer = Composer(static_cast<size_t>(num_gates));
+        generate_test_circuit(composer, static_cast<size_t>(num_gates));
+        auto ext_prover = composer.create_prover();
         state.ResumeTiming();
 
-        composer.create_verifier();
+        auto proof = ext_prover.construct_proof();
     }
+    state.SetComplexityN(num_gates); // Set up for computation of constant C where prover ~ C*N
 }
-BENCHMARK(create_verifier_bench)->DenseRange(MIN_LOG_NUM_GATES, MAX_LOG_NUM_GATES, 1)->Repetitions(NUM_REPETITIONS);
+BENCHMARK(construct_proof_standard)
+    ->DenseRange(MIN_LOG_NUM_GATES, MAX_LOG_NUM_GATES, 1)
+    ->Repetitions(NUM_REPETITIONS)
+    ->Complexity(oN);
 
 /**
- * @brief Benchmark: Construction of a Standard Honk proof
+ * @brief Benchmark: Creation of a Standard Honk verifier
  */
-void construct_proof_bench(State& state) noexcept
+void create_verifier_standard(State& state) noexcept
 {
-    auto num_gates = 1 << (size_t)state.range(0);
     for (auto _ : state) {
         state.PauseTiming();
-        auto composer = proof_system::honk::StandardHonkComposer(static_cast<size_t>(num_gates));
-        generate_test_plonk_circuit(composer, static_cast<size_t>(num_gates));
-        auto ext_prover = composer.create_prover();
+        auto num_gates = 1 << (size_t)state.range(0);
+        auto composer = Composer(static_cast<size_t>(num_gates));
+        generate_test_circuit(composer, static_cast<size_t>(num_gates));
         state.ResumeTiming();
 
-        auto proof = ext_prover.construct_proof();
+        composer.create_verifier();
     }
-    state.SetComplexityN(num_gates); // Set up for computation of constant C where prover ~ C*N
 }
-BENCHMARK(construct_proof_bench)
-    ->DenseRange(MIN_LOG_NUM_GATES, MAX_LOG_NUM_GATES, 1)
-    ->Repetitions(NUM_REPETITIONS)
-    ->Complexity(oN);
+// BENCHMARK(create_verifier_standard)->DenseRange(MIN_LOG_NUM_GATES, MAX_LOG_NUM_GATES,
+// 1)->Repetitions(NUM_REPETITIONS);
 
 /**
  * @brief Benchmark: Verification of a Standard Honk proof
  */
-void verify_proof_bench(State& state) noexcept
+void verify_proof_standard(State& state) noexcept
 {
     for (auto _ : state) {
         state.PauseTiming();
         auto num_gates = (size_t)state.range(0);
-        auto composer = proof_system::honk::StandardHonkComposer(static_cast<size_t>(num_gates));
-        generate_test_plonk_circuit(composer, static_cast<size_t>(num_gates));
+        auto composer = Composer(static_cast<size_t>(num_gates));
+        generate_test_circuit(composer, static_cast<size_t>(num_gates));
         auto prover = composer.create_prover();
         auto proof = prover.construct_proof();
         auto verifier = composer.create_verifier();
@@ -100,10 +105,5 @@ void verify_proof_bench(State& state) noexcept
         verifier.verify_proof(proof);
     }
 }
-// Note: enforcing Iterations == 1 for now. Otherwise proof construction will occur many times and this bench will take
-// a long time. (This is because the time limit for benchmarks does not include the time-excluded setup, and
-// verification itself is pretty fast).
-// Note: disabling this bench for now since it is not of primary interest
-// BENCHMARK(verify_proof_bench)->DenseRange(MIN_LOG_NUM_GATES, MAX_LOG_NUM_GATES, 1)->Iterations(1);
-
-BENCHMARK_MAIN();
+// BENCHMARK(verify_proof_standard)->DenseRange(MIN_LOG_NUM_GATES, MAX_LOG_NUM_GATES, 1)->Iterations(1);
+} // namespace standard_honk_bench

cpp/src/barretenberg/benchmark/honk_bench/main.bench.cpp

@@ -0,0 +1,3 @@
+#include <benchmark/benchmark.h>
+
+BENCHMARK_MAIN();

cpp/src/barretenberg/benchmark/honk_bench/ultra_honk.bench.cpp

@@ -0,0 +1,169 @@
+#include "barretenberg/crypto/ecdsa/ecdsa.hpp"
+#include "barretenberg/ecc/curves/bn254/fr.hpp"
+#include "barretenberg/honk/proof_system/ultra_prover.hpp"
+#include "barretenberg/honk/proof_system/ultra_verifier.hpp"
+#include <benchmark/benchmark.h>
+#include <cstddef>
+#include "barretenberg/honk/composer/ultra_honk_composer.hpp"
+#include "barretenberg/stdlib/encryption/ecdsa/ecdsa.hpp"
+#include "barretenberg/stdlib/hash/keccak/keccak.hpp"
+#include "barretenberg/stdlib/primitives/curves/secp256k1.hpp"
+#include "barretenberg/stdlib/primitives/packed_byte_array/packed_byte_array.hpp"
+#include "barretenberg/stdlib/hash/sha256/sha256.hpp"
+#include "barretenberg/stdlib/primitives/bool/bool.hpp"
+#include "barretenberg/stdlib/primitives/field/field.hpp"
+#include "barretenberg/stdlib/primitives/witness/witness.hpp"
+#include "barretenberg/stdlib/merkle_tree/merkle_tree.hpp"
+#include "barretenberg/stdlib/merkle_tree/membership.hpp"
+#include "barretenberg/stdlib/merkle_tree/memory_store.hpp"
+#include "barretenberg/stdlib/merkle_tree/memory_tree.hpp"
+
+using namespace benchmark;
+
+namespace ultra_honk_bench {
+
+using Composer = proof_system::honk::UltraHonkComposer;
+
+// Number of times to perform operation of interest in the benchmark circuits, e.g. # of hashes to perform
+constexpr size_t MIN_NUM_ITERATIONS = 10;
+constexpr size_t MAX_NUM_ITERATIONS = 10;
+// Number of times to repeat each benchmark
+constexpr size_t NUM_REPETITIONS = 1;
+
+/**
+ * @brief Generate test circuit with specified number of sha256 hashes
+ *
+ * @param composer
+ * @param num_iterations
+ */
+void generate_sha256_test_circuit(Composer& composer, size_t num_iterations)
+{
+    std::string in;
+    in.resize(32);
+    for (size_t i = 0; i < 32; ++i) {
+        in[i] = 0;
+    }
+    proof_system::plonk::stdlib::packed_byte_array<Composer> input(&composer, in);
+    for (size_t i = 0; i < num_iterations; i++) {
+        input = proof_system::plonk::stdlib::sha256<Composer>(input);
+    }
+}
+
+/**
+ * @brief Generate test circuit with specified number of keccak hashes
+ *
+ * @param composer
+ * @param num_iterations
+ */
+void generate_keccak_test_circuit(Composer& composer, size_t num_iterations)
+{
+    std::string in = "abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz01";
+
+    proof_system::plonk::stdlib::byte_array<Composer> input(&composer, in);
+    for (size_t i = 0; i < num_iterations; i++) {
+        input = proof_system::plonk::stdlib::keccak<Composer>::hash(input);
+    }
+}
+
+/**
+ * @brief Generate test circuit with specified number of ecdsa verifications
+ *
+ * @param composer
+ * @param num_iterations
+ */
+void generate_ecdsa_verification_test_circuit(Composer& composer, size_t num_iterations)
+{
+    using curve = proof_system::plonk::stdlib::secp256k1<Composer>;
+
+    std::string message_string = "Instructions unclear, ask again later.";
+
+    crypto::ecdsa::key_pair<curve::fr, curve::g1> account;
+    account.private_key = curve::fr::random_element();
+    account.public_key = curve::g1::one * account.private_key;
+
+    crypto::ecdsa::signature signature =
+        crypto::ecdsa::construct_signature<Sha256Hasher, curve::fq, curve::fr, curve::g1>(message_string, account);
+
+    bool first_result = crypto::ecdsa::verify_signature<Sha256Hasher, curve::fq, curve::fr, curve::g1>(
+        message_string, account.public_key, signature);
+
+    std::vector<uint8_t> rr(signature.r.begin(), signature.r.end());
+    std::vector<uint8_t> ss(signature.s.begin(), signature.s.end());
+    uint8_t vv = signature.v;
+
+    curve::g1_bigfr_ct public_key = curve::g1_bigfr_ct::from_witness(&composer, account.public_key);
+
+    proof_system::plonk::stdlib::ecdsa::signature<Composer> sig{ curve::byte_array_ct(&composer, rr),
+                                                                 curve::byte_array_ct(&composer, ss),
+                                                                 proof_system::plonk::stdlib::uint8<Composer>(&composer,
+                                                                                                              vv) };
+
+    curve::byte_array_ct message(&composer, message_string);
+
+    for (size_t i = 0; i < num_iterations; i++) {
+        proof_system::plonk::stdlib::ecdsa::
+            verify_signature<Composer, curve, curve::fq_ct, curve::bigfr_ct, curve::g1_bigfr_ct>(
+                message, public_key, sig);
+    }
+}
+
+/**
+ * @brief Generate test circuit with specified number of merkle membership checks
+ *
+ * @param composer
+ * @param num_iterations
+ * @todo (luke): should we consider deeper tree? non-zero leaf values? variable index?
+ */
+void generate_merkle_membership_test_circuit(Composer& composer, size_t num_iterations)
+{
+    using namespace proof_system::plonk::stdlib;
+    using field_ct = field_t<Composer>;
+    using witness_ct = witness_t<Composer>;
+    using witness_ct = witness_t<Composer>;
+    using MemStore = merkle_tree::MemoryStore;
+    using MerkleTree_ct = merkle_tree::MerkleTree<MemStore>;
+
+    MemStore store;
+    auto db = MerkleTree_ct(store, 3);
+
+    // Check that the leaf at index 0 has value 0.
+    auto zero = field_ct(witness_ct(&composer, fr::zero())).decompose_into_bits();
+    field_ct root = witness_ct(&composer, db.root());
+
+    for (size_t i = 0; i < num_iterations; i++) {
+        merkle_tree::check_membership(
+            root, merkle_tree::create_witness_hash_path(composer, db.get_hash_path(0)), field_ct(0), zero);
+    }
+}
+
+/**
+ * @brief Benchmark: Construction of a Ultra Honk proof for a circuit determined by the provided text circuit function
+ */
+void construct_proof_ultra(State& state, void (*test_circuit_function)(Composer&, size_t)) noexcept
+{
+    auto num_iterations = static_cast<size_t>(state.range(0));
+    for (auto _ : state) {
+        state.PauseTiming();
+        auto composer = Composer();
+        test_circuit_function(composer, num_iterations);
+        auto ext_prover = composer.create_prover();
+        state.ResumeTiming();
+
+        auto proof = ext_prover.construct_proof();
+    }
+}
+
+BENCHMARK_CAPTURE(construct_proof_ultra, sha256, &generate_sha256_test_circuit)
+    ->DenseRange(MIN_NUM_ITERATIONS, MAX_NUM_ITERATIONS)
+    ->Repetitions(NUM_REPETITIONS);
+BENCHMARK_CAPTURE(construct_proof_ultra, keccak, &generate_keccak_test_circuit)
+    ->DenseRange(MIN_NUM_ITERATIONS, MAX_NUM_ITERATIONS)
+    ->Repetitions(NUM_REPETITIONS);
+BENCHMARK_CAPTURE(construct_proof_ultra, ecdsa_verification, &generate_ecdsa_verification_test_circuit)
+    ->DenseRange(MIN_NUM_ITERATIONS, MAX_NUM_ITERATIONS)
+    ->Repetitions(NUM_REPETITIONS);
+BENCHMARK_CAPTURE(construct_proof_ultra, merkle_membership, &generate_merkle_membership_test_circuit)
+    ->DenseRange(MIN_NUM_ITERATIONS, MAX_NUM_ITERATIONS)
+    ->Repetitions(NUM_REPETITIONS);
+
+} // namespace ultra_honk_bench