diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 0000000000..9d70efdcb4 --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,69 @@ +name: Barretenberg Benchmarks + +# Only run on push to master +on: + push: + branches: + - master + +# This will cancel previous runs when a branch or PR is updated +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref || github.run_id }} + cancel-in-progress: true + +jobs: + bberg-bench: + name: Barretenberg Benchmarks + runs-on: ubuntu-latest # run in linux environment + + steps: + + - name: Checkout barretenberg + uses: actions/checkout@v3 + with: + ref: ${{ github.head_ref }} # checkout HEAD of triggering branch + token: ${{ secrets.GITHUB_TOKEN }} + + # Only run in Linux environment for now + - name: Setup Linux environment + run: | + sudo apt update + sudo apt install libomp-dev cmake ninja-build + + - name: Download SRS elements + working-directory: cpp/srs_db + run: ./download_ignition.sh 3 # only download first 4 transcript files + + - name: Build Honk benchmarks + working-directory: cpp + run: | + cmake --preset bench + cmake --build --preset bench --target ultra_honk_bench + + - name: Run Honk benchmarks + working-directory: cpp/build-bench + run: | + bin/ultra_honk_bench --benchmark_format=json > ../src/barretenberg/benchmark/honk_bench/bench_results.json + + # Utilize github-action-benchmark to automatically update the plots at + # https://aztecprotocol.github.io/barretenberg/dev/bench/ with new benchmark data. + # This also creates an alert if benchmarks exceed the threshold specified below. + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + name: C++ Benchmark + tool: 'googlecpp' + output-file-path: cpp/src/barretenberg/benchmark/honk_bench/bench_results.json + # Access token to deploy GitHub Pages branch + github-token: ${{ secrets.GITHUB_TOKEN }} + # Push and deploy GitHub pages branch automatically + auto-push: true + # Enable Job Summary for PRs + summary-always: true + # Show alert with commit comment on detecting possible performance regression + alert-threshold: '120%' # alert if bench result is 1.2x worse + comment-on-alert: true + fail-on-alert: false + alert-comment-cc-users: '@ledwards2225' + + \ No newline at end of file diff --git a/cpp/src/barretenberg/benchmark/honk_bench/CMakeLists.txt b/cpp/src/barretenberg/benchmark/honk_bench/CMakeLists.txt index a521b94cdc..d9a995e7cf 100644 --- a/cpp/src/barretenberg/benchmark/honk_bench/CMakeLists.txt +++ b/cpp/src/barretenberg/benchmark/honk_bench/CMakeLists.txt @@ -1,18 +1,23 @@ -add_executable(honk_bench main.bench.cpp honk.bench.cpp ultra_honk.bench.cpp) +# Each source represents a separate benchmark suite +set(BENCHMARK_SOURCES +standard_honk.bench.cpp +standard_plonk.bench.cpp +ultra_honk.bench.cpp +ultra_plonk.bench.cpp +) -target_link_libraries( - honk_bench - stdlib_primitives - common +# Required libraries for benchmark suites +set(LINKED_LIBRARIES stdlib_sha256 stdlib_keccak stdlib_merkle_tree - env benchmark::benchmark ) -add_custom_target( - run_honk_bench - COMMAND honk_bench - WORKING_DIRECTORY ${CMAKE_BINARY_DIR} -) \ No newline at end of file +# Add executable and custom target for each suite, e.g. standard_honk_bench +foreach(BENCHMARK_SOURCE ${BENCHMARK_SOURCES}) + get_filename_component(BENCHMARK_NAME ${BENCHMARK_SOURCE} NAME_WE) # extract name without extension + add_executable(${BENCHMARK_NAME}_bench main.bench.cpp ${BENCHMARK_SOURCE} benchmark_utilities.hpp) + target_link_libraries(${BENCHMARK_NAME}_bench ${LINKED_LIBRARIES}) + add_custom_target(run_${BENCHMARK_NAME} COMMAND ${BENCHMARK_NAME} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +endforeach() \ No newline at end of file diff --git a/cpp/src/barretenberg/benchmark/honk_bench/benchmark_utilities.hpp b/cpp/src/barretenberg/benchmark/honk_bench/benchmark_utilities.hpp new file mode 100644 index 0000000000..81a12b129e --- /dev/null +++ b/cpp/src/barretenberg/benchmark/honk_bench/benchmark_utilities.hpp @@ -0,0 +1,231 @@ +#include "barretenberg/crypto/ecdsa/ecdsa.hpp" +#include "barretenberg/ecc/curves/bn254/fr.hpp" +#include "barretenberg/honk/proof_system/ultra_prover.hpp" +#include "barretenberg/honk/proof_system/ultra_verifier.hpp" +#include +#include +#include "barretenberg/honk/composer/standard_honk_composer.hpp" +#include "barretenberg/plonk/composer/standard_plonk_composer.hpp" +#include "barretenberg/stdlib/encryption/ecdsa/ecdsa.hpp" +#include "barretenberg/stdlib/hash/keccak/keccak.hpp" +#include "barretenberg/stdlib/primitives/curves/secp256k1.hpp" +#include "barretenberg/stdlib/primitives/packed_byte_array/packed_byte_array.hpp" +#include "barretenberg/stdlib/hash/sha256/sha256.hpp" +#include "barretenberg/stdlib/primitives/bool/bool.hpp" +#include "barretenberg/stdlib/primitives/field/field.hpp" +#include "barretenberg/stdlib/primitives/witness/witness.hpp" +#include "barretenberg/stdlib/merkle_tree/merkle_tree.hpp" +#include "barretenberg/stdlib/merkle_tree/membership.hpp" +#include "barretenberg/stdlib/merkle_tree/memory_store.hpp" +#include "barretenberg/stdlib/merkle_tree/memory_tree.hpp" + +using namespace benchmark; + +namespace bench_utils { + +struct BenchParams { + // Num iterations of the operation of interest in a test circuit, e.g. num sha256 hashes + static constexpr size_t MIN_NUM_ITERATIONS = 10; + static constexpr size_t MAX_NUM_ITERATIONS = 10; + + // Log num gates; for simple circuits only, e.g. standard arithmetic circuit + static constexpr size_t MIN_LOG_NUM_GATES = 16; + static constexpr size_t MAX_LOG_NUM_GATES = 16; + + static constexpr size_t NUM_REPETITIONS = 1; +}; + +/** + * @brief Generate test circuit with basic arithmetic operations + * + * @param composer + * @param num_iterations + */ +template void generate_basic_arithmetic_circuit(Composer& composer, size_t num_gates) +{ + plonk::stdlib::field_t a(plonk::stdlib::witness_t(&composer, barretenberg::fr::random_element())); + plonk::stdlib::field_t b(plonk::stdlib::witness_t(&composer, barretenberg::fr::random_element())); + plonk::stdlib::field_t c(&composer); + for (size_t i = 0; i < (num_gates / 4) - 4; ++i) { + c = a + b; + c = a * c; + a = b * b; + b = c * c; + } +} + +/** + * @brief Generate test circuit with specified number of sha256 hashes + * + * @param composer + * @param num_iterations + */ +template void generate_sha256_test_circuit(Composer& composer, size_t num_iterations) +{ + std::string in; + in.resize(32); + for (size_t i = 0; i < 32; ++i) { + in[i] = 0; + } + proof_system::plonk::stdlib::packed_byte_array input(&composer, in); + for (size_t i = 0; i < num_iterations; i++) { + input = proof_system::plonk::stdlib::sha256(input); + } +} + +/** + * @brief Generate test circuit with specified number of keccak hashes + * + * @param composer + * @param num_iterations + */ +template void generate_keccak_test_circuit(Composer& composer, size_t num_iterations) +{ + std::string in = "abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz01"; + + proof_system::plonk::stdlib::byte_array input(&composer, in); + for (size_t i = 0; i < num_iterations; i++) { + input = proof_system::plonk::stdlib::keccak::hash(input); + } +} + +/** + * @brief Generate test circuit with specified number of ecdsa verifications + * + * @param composer + * @param num_iterations + */ +template void generate_ecdsa_verification_test_circuit(Composer& composer, size_t num_iterations) +{ + using curve = proof_system::plonk::stdlib::secp256k1; + using fr = typename curve::fr; + using fq = typename curve::fq; + using g1 = typename curve::g1; + + std::string message_string = "Instructions unclear, ask again later."; + + crypto::ecdsa::key_pair account; + for (size_t i = 0; i < num_iterations; i++) { + // Generate unique signature for each iteration + account.private_key = curve::fr::random_element(); + account.public_key = curve::g1::one * account.private_key; + + crypto::ecdsa::signature signature = + crypto::ecdsa::construct_signature(message_string, account); + + bool first_result = + crypto::ecdsa::verify_signature(message_string, account.public_key, signature); + + std::vector rr(signature.r.begin(), signature.r.end()); + std::vector ss(signature.s.begin(), signature.s.end()); + uint8_t vv = signature.v; + + typename curve::g1_bigfr_ct public_key = curve::g1_bigfr_ct::from_witness(&composer, account.public_key); + + proof_system::plonk::stdlib::ecdsa::signature sig{ typename curve::byte_array_ct(&composer, rr), + typename curve::byte_array_ct(&composer, ss), + proof_system::plonk::stdlib::uint8( + &composer, vv) }; + + typename curve::byte_array_ct message(&composer, message_string); + + // Verify ecdsa signature + proof_system::plonk::stdlib::ecdsa::verify_signature(message, public_key, sig); + } +} + +/** + * @brief Generate test circuit with specified number of merkle membership checks + * + * @param composer + * @param num_iterations + */ +template void generate_merkle_membership_test_circuit(Composer& composer, size_t num_iterations) +{ + using namespace proof_system::plonk::stdlib; + using field_ct = field_t; + using witness_ct = witness_t; + using witness_ct = witness_t; + using MemStore = merkle_tree::MemoryStore; + using MerkleTree_ct = merkle_tree::MerkleTree; + + MemStore store; + const size_t tree_depth = 7; + auto merkle_tree = MerkleTree_ct(store, tree_depth); + + for (size_t i = 0; i < num_iterations; i++) { + // For each iteration update and check the membership of a different value + size_t idx = i; + size_t value = i * 2; + merkle_tree.update_element(idx, value); + + field_ct root_ct = witness_ct(&composer, merkle_tree.root()); + auto idx_ct = field_ct(witness_ct(&composer, fr(idx))).decompose_into_bits(); + auto value_ct = field_ct(value); + + merkle_tree::check_membership( + root_ct, merkle_tree::create_witness_hash_path(composer, merkle_tree.get_hash_path(idx)), value_ct, idx_ct); + } +} + +/** + * @brief Performs proof constuction for benchmarks based on a provided circuit function + * + * @details This function assumes state.range refers to num_gates which is the size of the underlying circuit + * + * @tparam Composer + * @param state + * @param test_circuit_function + */ +template +void construct_proof_with_specified_num_gates(State& state, void (*test_circuit_function)(Composer&, size_t)) noexcept +{ + barretenberg::srs::init_crs_factory("../srs_db/ignition"); + auto num_gates = static_cast(1 << (size_t)state.range(0)); + for (auto _ : state) { + // Constuct circuit and prover; don't include this part in measurement + state.PauseTiming(); + auto composer = Composer(); + test_circuit_function(composer, num_gates); + auto ext_prover = composer.create_prover(); + state.ResumeTiming(); + + // Construct proof + auto proof = ext_prover.construct_proof(); + } +} + +/** + * @brief Performs proof constuction for benchmarks based on a provided circuit function + * + * @details This function assumes state.range refers to num_iterations which is the number of times to perform a given + * basic operation in the circuit, e.g. number of hashes + * + * @tparam Composer + * @param state + * @param test_circuit_function + */ +template +void construct_proof_with_specified_num_iterations(State& state, + void (*test_circuit_function)(Composer&, size_t)) noexcept +{ + barretenberg::srs::init_crs_factory("../srs_db/ignition"); + auto num_iterations = static_cast(state.range(0)); + for (auto _ : state) { + // Constuct circuit and prover; don't include this part in measurement + state.PauseTiming(); + auto composer = Composer(); + test_circuit_function(composer, num_iterations); + auto ext_prover = composer.create_prover(); + state.ResumeTiming(); + + // Construct proof + auto proof = ext_prover.construct_proof(); + } +} + +} // namespace bench_utils \ No newline at end of file diff --git a/cpp/src/barretenberg/benchmark/honk_bench/compare_honk_bench.sh b/cpp/src/barretenberg/benchmark/honk_bench/compare_honk_bench.sh deleted file mode 100755 index 95cdbb8ed6..0000000000 --- a/cpp/src/barretenberg/benchmark/honk_bench/compare_honk_bench.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -# This script is used to compare the results of honk_bench between baseline (master) and -# the branch from which the script is run. Simply check out the branch of interest, ensure -# it is up to date with local master, and run the script. - -echo -e '\nComparing Honk benchmarks between master and current branch:' -# Set some directories -BASE_DIR="$HOME/barretenberg/cpp" -BUILD_DIR="$BASE_DIR/build-bench" -BENCH_RESULTS_DIR="$BASE_DIR/tmp_bench_results" -BENCH_TOOLS_DIR="$BUILD_DIR/_deps/benchmark-src/tools" - -# Install requirements (numpy + scipy) for comparison script if necessary. -# Note: By default, installation will occur in $HOME/.local/bin. -pip3 install -r $BUILD_DIR/_deps/benchmark-src/requirements.txt - -# Create temporary directory for honk_bench results (json) -cd $BASE_DIR -mkdir $BENCH_RESULTS_DIR - -# Checkout master, run honk_bench, save results in json format -echo -e '\nConfiguring and building honk_bench in master branch..' -git checkout master > /dev/null -rm -rf $BUILD_DIR -cmake --preset bench > /dev/null && cmake --build --preset bench --target honk_bench > /dev/null -cd build-bench -MASTER_HONK_BENCH_RESULTS="$BENCH_RESULTS_DIR/honk_bench_results_master.json" -echo -e '\nRunning honk_bench in master..' -bin/honk_bench --benchmark_format=json > $MASTER_HONK_BENCH_RESULTS - -# Checkout working branch (-), run honk_bench, save results in json format -echo -e '\nConfiguring and building honk_bench in current feature branch..' -git checkout - -rm -rf $BUILD_DIR -cmake --preset bench > /dev/null && cmake --build --preset bench --target honk_bench > /dev/null -cd build-bench -BRANCH_HONK_BENCH_RESULTS="$BENCH_RESULTS_DIR/honk_bench_results_branch.json" -echo -e '\nRunning honk_bench in feature branch..' -bin/honk_bench --benchmark_format=json > $BRANCH_HONK_BENCH_RESULTS - -# Call compare.py on the results (json) to get high level statistics. -# See docs at https://github.com/google/benchmark/blob/main/docs/tools.md for more details. -$BENCH_TOOLS_DIR/compare.py benchmarks $MASTER_HONK_BENCH_RESULTS $BRANCH_HONK_BENCH_RESULTS - -# Delete the temporary results directory and its contents -rm -r $BENCH_RESULTS_DIR \ No newline at end of file diff --git a/cpp/src/barretenberg/benchmark/honk_bench/compare_honk_branch_vs_baseline.sh b/cpp/src/barretenberg/benchmark/honk_bench/compare_honk_branch_vs_baseline.sh new file mode 100755 index 0000000000..71bf3ee5b9 --- /dev/null +++ b/cpp/src/barretenberg/benchmark/honk_bench/compare_honk_branch_vs_baseline.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +# This script is used to compare the honk benchmarks between baseline (default: master) and +# the branch from which the script is run. Simply check out the branch of interest, ensure +# it is up to date with local master, and run the script. + +# Specify the benchmark suite and the "baseline" branch against which to compare +BENCH_TARGET="ultra_honk_bench" +BASELINE_BRANCH="master" + +echo -e "\nComparing $BENCH_TARGET between $BASELINE_BRANCH and current branch:" +# Set some directories +BASE_DIR="$HOME/barretenberg/cpp" +BUILD_DIR="$BASE_DIR/build-bench" # matches build dir specified in bench preset +BENCH_RESULTS_DIR="$BASE_DIR/tmp_bench_results" +BENCH_TOOLS_DIR="$BUILD_DIR/_deps/benchmark-src/tools" + +# Install requirements (numpy + scipy) for comparison script if necessary. +# Note: By default, installation will occur in $HOME/.local/bin. +pip3 install --user -r $BUILD_DIR/_deps/benchmark-src/requirements.txt + +# Create temporary directory for benchmark results (json) +cd $BASE_DIR +mkdir $BENCH_RESULTS_DIR + +# Build and run bench in current branch +echo -e "\nConfiguring and building $BENCH_TARGET in current feature branch..\n" +rm -rf $BUILD_DIR +cmake --preset bench > /dev/null && cmake --build --preset bench --target $BENCH_TARGET +cd build-bench +BRANCH_RESULTS="$BENCH_RESULTS_DIR/results_branch.json" +echo -e "\nRunning $BENCH_TARGET in feature branch.." +bin/$BENCH_TARGET --benchmark_format=json > $BRANCH_RESULTS + +# Checkout baseline branch, run benchmarks, save results in json format +echo -e "\nConfiguring and building $BENCH_TARGET in $BASELINE_BRANCH branch..\n" +git checkout master > /dev/null +rm -rf $BUILD_DIR +cmake --preset bench > /dev/null && cmake --build --preset bench --target $BENCH_TARGET +cd build-bench +BASELINE_RESULTS="$BENCH_RESULTS_DIR/results_baseline.json" +echo -e "\nRunning $BENCH_TARGET in master.." +bin/$BENCH_TARGET --benchmark_format=json > $BASELINE_RESULTS + +# Call compare.py on the results (json) to get high level statistics. +# See docs at https://github.com/google/benchmark/blob/main/docs/tools.md for more details. +$BENCH_TOOLS_DIR/compare.py benchmarks $BASELINE_RESULTS $BRANCH_RESULTS + +# Return to branch from which the script was called +git checkout - + +# Delete the temporary results directory and its contents +rm -r $BENCH_RESULTS_DIR diff --git a/cpp/src/barretenberg/benchmark/honk_bench/compare_honk_to_plonk_standard.sh b/cpp/src/barretenberg/benchmark/honk_bench/compare_honk_to_plonk_standard.sh new file mode 100755 index 0000000000..ad7ecc1f22 --- /dev/null +++ b/cpp/src/barretenberg/benchmark/honk_bench/compare_honk_to_plonk_standard.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# This script is used to compare the results of honk_bench between baseline (master) and +# the branch from which the script is run. Simply check out the branch of interest, ensure +# it is up to date with local master, and run the script. + +echo -e '\nComparing Standard Plonk/Honk benchmarks.' +# Set some directories +BASE_DIR="$HOME/barretenberg/cpp" +BUILD_DIR="$BASE_DIR/build-bench" +BENCH_RESULTS_DIR="$BASE_DIR/tmp_bench_results" +BENCH_TOOLS_DIR="$BUILD_DIR/_deps/benchmark-src/tools" + +# Install requirements (numpy + scipy) for comparison script if necessary. +# Note: By default, installation will occur in $HOME/.local/bin. +pip3 install --user -r $BUILD_DIR/_deps/benchmark-src/requirements.txt + +# Create temporary directory for honk_bench results (json) +cd $BASE_DIR +mkdir $BENCH_RESULTS_DIR + +# +echo -e '\nBuilding and running Standard benchmarks..' +# rm -rf $BUILD_DIR +cmake --preset bench > /dev/null && cmake --build --preset bench --target standard_plonk_bench +cd build-bench +PLONK_BENCH_RESULTS="$BENCH_RESULTS_DIR/plonk_bench.json" +./bin/standard_plonk_bench --benchmark_format=json > $PLONK_BENCH_RESULTS + +cd .. +cmake --preset bench > /dev/null && cmake --build --preset bench --target standard_honk_bench +cd build-bench +HONK_BENCH_RESULTS="$BENCH_RESULTS_DIR/honk_bench.json" +./bin/standard_honk_bench --benchmark_format=json > $HONK_BENCH_RESULTS + +# Call compare.py on the results (json) to get high level statistics. +# See docs at https://github.com/google/benchmark/blob/main/docs/tools.md for more details. +$BENCH_TOOLS_DIR/compare.py benchmarks $PLONK_BENCH_RESULTS $HONK_BENCH_RESULTS + +# # Delete the temporary results directory and its contents +rm -r $BENCH_RESULTS_DIR \ No newline at end of file diff --git a/cpp/src/barretenberg/benchmark/honk_bench/compare_honk_to_plonk_ultra.sh b/cpp/src/barretenberg/benchmark/honk_bench/compare_honk_to_plonk_ultra.sh new file mode 100755 index 0000000000..0e5625e930 --- /dev/null +++ b/cpp/src/barretenberg/benchmark/honk_bench/compare_honk_to_plonk_ultra.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# This script is used to compare the results of honk_bench between baseline (master) and +# the branch from which the script is run. Simply check out the branch of interest, ensure +# it is up to date with local master, and run the script. + +echo -e '\nComparing Ultra Plonk/Honk benchmarks.' +# Set some directories +BASE_DIR="$HOME/barretenberg/cpp" +BUILD_DIR="$BASE_DIR/build-bench" +BENCH_RESULTS_DIR="$BASE_DIR/tmp_bench_results" +BENCH_TOOLS_DIR="$BUILD_DIR/_deps/benchmark-src/tools" + +# Install requirements (numpy + scipy) for comparison script if necessary. +# Note: By default, installation will occur in $HOME/.local/bin. +pip3 install --user -r $BUILD_DIR/_deps/benchmark-src/requirements.txt + +# Create temporary directory for honk_bench results (json) +cd $BASE_DIR +mkdir $BENCH_RESULTS_DIR + +# +echo -e '\nBuilding and running Standard benchmarks..' +# rm -rf $BUILD_DIR +cmake --preset bench > /dev/null && cmake --build --preset bench --target ultra_plonk_bench +cd build-bench +PLONK_BENCH_RESULTS="$BENCH_RESULTS_DIR/plonk_bench.json" +./bin/ultra_plonk_bench --benchmark_format=json > $PLONK_BENCH_RESULTS + +cd .. +cmake --preset bench > /dev/null && cmake --build --preset bench --target ultra_honk_bench +cd build-bench +HONK_BENCH_RESULTS="$BENCH_RESULTS_DIR/honk_bench.json" +./bin/ultra_honk_bench --benchmark_format=json > $HONK_BENCH_RESULTS + +# Call compare.py on the results (json) to get high level statistics. +# See docs at https://github.com/google/benchmark/blob/main/docs/tools.md for more details. +$BENCH_TOOLS_DIR/compare.py benchmarks $PLONK_BENCH_RESULTS $HONK_BENCH_RESULTS + +# # Delete the temporary results directory and its contents +rm -r $BENCH_RESULTS_DIR \ No newline at end of file diff --git a/cpp/src/barretenberg/benchmark/honk_bench/standard_honk.bench.cpp b/cpp/src/barretenberg/benchmark/honk_bench/standard_honk.bench.cpp new file mode 100644 index 0000000000..c88d80292d --- /dev/null +++ b/cpp/src/barretenberg/benchmark/honk_bench/standard_honk.bench.cpp @@ -0,0 +1,28 @@ +#include "barretenberg/benchmark/honk_bench/benchmark_utilities.hpp" + +using namespace benchmark; + +namespace standard_honk_bench { + +using StandardHonk = proof_system::honk::StandardHonkComposer; + +// Log number of gates for test circuit +constexpr size_t MIN_LOG_NUM_GATES = bench_utils::BenchParams::MIN_LOG_NUM_GATES; +constexpr size_t MAX_LOG_NUM_GATES = bench_utils::BenchParams::MAX_LOG_NUM_GATES; +// Number of times to repeat each benchmark +constexpr size_t NUM_REPETITIONS = bench_utils::BenchParams::NUM_REPETITIONS; + +/** + * @brief Benchmark: Construction of a Standard proof for a circuit determined by the provided circuit function + */ +void construct_proof_standard(State& state, void (*test_circuit_function)(StandardHonk&, size_t)) noexcept +{ + bench_utils::construct_proof_with_specified_num_gates(state, test_circuit_function); +} + +BENCHMARK_CAPTURE(construct_proof_standard, arithmetic, &bench_utils::generate_basic_arithmetic_circuit) + ->DenseRange(MIN_LOG_NUM_GATES, MAX_LOG_NUM_GATES) + ->Repetitions(NUM_REPETITIONS) + ->Unit(::benchmark::kSecond); + +} // namespace standard_honk_bench \ No newline at end of file diff --git a/cpp/src/barretenberg/benchmark/honk_bench/standard_plonk.bench.cpp b/cpp/src/barretenberg/benchmark/honk_bench/standard_plonk.bench.cpp new file mode 100644 index 0000000000..106fd00f19 --- /dev/null +++ b/cpp/src/barretenberg/benchmark/honk_bench/standard_plonk.bench.cpp @@ -0,0 +1,28 @@ +#include "barretenberg/benchmark/honk_bench/benchmark_utilities.hpp" + +using namespace benchmark; + +namespace standard_plonk_bench { + +using StandardPlonk = proof_system::plonk::StandardPlonkComposer; + +// Log number of gates for test circuit +constexpr size_t MIN_LOG_NUM_GATES = bench_utils::BenchParams::MIN_LOG_NUM_GATES; +constexpr size_t MAX_LOG_NUM_GATES = bench_utils::BenchParams::MAX_LOG_NUM_GATES; +// Number of times to repeat each benchmark +constexpr size_t NUM_REPETITIONS = bench_utils::BenchParams::NUM_REPETITIONS; + +/** + * @brief Benchmark: Construction of a Standard proof for a circuit determined by the provided circuit function + */ +void construct_proof_standard(State& state, void (*test_circuit_function)(StandardPlonk&, size_t)) noexcept +{ + bench_utils::construct_proof_with_specified_num_gates(state, test_circuit_function); +} + +BENCHMARK_CAPTURE(construct_proof_standard, arithmetic, &bench_utils::generate_basic_arithmetic_circuit) + ->DenseRange(MIN_LOG_NUM_GATES, MAX_LOG_NUM_GATES) + ->Repetitions(NUM_REPETITIONS) + ->Unit(::benchmark::kSecond); + +} // namespace standard_plonk_bench \ No newline at end of file diff --git a/cpp/src/barretenberg/benchmark/honk_bench/ultra_honk.bench.cpp b/cpp/src/barretenberg/benchmark/honk_bench/ultra_honk.bench.cpp index 8a6f7933e4..e834d83ff9 100644 --- a/cpp/src/barretenberg/benchmark/honk_bench/ultra_honk.bench.cpp +++ b/cpp/src/barretenberg/benchmark/honk_bench/ultra_honk.bench.cpp @@ -1,168 +1,45 @@ -#include "barretenberg/crypto/ecdsa/ecdsa.hpp" -#include "barretenberg/ecc/curves/bn254/fr.hpp" -#include -#include -#include "barretenberg/stdlib/primitives/composers/composers_fwd.hpp" -#include "barretenberg/stdlib/primitives/composers/composers.hpp" -#include "barretenberg/stdlib/encryption/ecdsa/ecdsa.hpp" -#include "barretenberg/stdlib/hash/keccak/keccak.hpp" -#include "barretenberg/stdlib/primitives/curves/secp256k1.hpp" -#include "barretenberg/stdlib/primitives/packed_byte_array/packed_byte_array.hpp" -#include "barretenberg/stdlib/hash/sha256/sha256.hpp" -#include "barretenberg/stdlib/primitives/bool/bool.hpp" -#include "barretenberg/stdlib/primitives/field/field.hpp" -#include "barretenberg/stdlib/primitives/witness/witness.hpp" -#include "barretenberg/stdlib/merkle_tree/merkle_tree.hpp" -#include "barretenberg/stdlib/merkle_tree/membership.hpp" -#include "barretenberg/stdlib/merkle_tree/memory_store.hpp" -#include "barretenberg/stdlib/merkle_tree/memory_tree.hpp" +#include "barretenberg/benchmark/honk_bench/benchmark_utilities.hpp" using namespace benchmark; namespace ultra_honk_bench { -using Composer = proof_system::honk::UltraHonkComposer; +using UltraHonk = proof_system::honk::UltraHonkComposer; // Number of times to perform operation of interest in the benchmark circuits, e.g. # of hashes to perform -constexpr size_t MIN_NUM_ITERATIONS = 10; -constexpr size_t MAX_NUM_ITERATIONS = 10; +constexpr size_t MIN_NUM_ITERATIONS = bench_utils::BenchParams::MIN_NUM_ITERATIONS; +constexpr size_t MAX_NUM_ITERATIONS = bench_utils::BenchParams::MAX_NUM_ITERATIONS; // Number of times to repeat each benchmark -constexpr size_t NUM_REPETITIONS = 1; +constexpr size_t NUM_REPETITIONS = bench_utils::BenchParams::NUM_REPETITIONS; /** - * @brief Generate test circuit with specified number of sha256 hashes - * - * @param composer - * @param num_iterations + * @brief Benchmark: Construction of a Ultra Honk proof for a circuit determined by the provided circuit function */ -void generate_sha256_test_circuit(Composer& composer, size_t num_iterations) +void construct_proof_ultra(State& state, void (*test_circuit_function)(UltraHonk&, size_t)) noexcept { - std::string in; - in.resize(32); - for (size_t i = 0; i < 32; ++i) { - in[i] = 0; - } - proof_system::plonk::stdlib::packed_byte_array input(&composer, in); - for (size_t i = 0; i < num_iterations; i++) { - input = proof_system::plonk::stdlib::sha256(input); - } + bench_utils::construct_proof_with_specified_num_iterations(state, test_circuit_function); } -/** - * @brief Generate test circuit with specified number of keccak hashes - * - * @param composer - * @param num_iterations - */ -void generate_keccak_test_circuit(Composer& composer, size_t num_iterations) -{ - std::string in = "abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz01"; - - proof_system::plonk::stdlib::byte_array input(&composer, in); - for (size_t i = 0; i < num_iterations; i++) { - input = proof_system::plonk::stdlib::keccak::hash(input); - } -} - -/** - * @brief Generate test circuit with specified number of ecdsa verifications - * - * @param composer - * @param num_iterations - */ -void generate_ecdsa_verification_test_circuit(Composer& composer, size_t num_iterations) -{ - using curve = proof_system::plonk::stdlib::secp256k1; - - std::string message_string = "Instructions unclear, ask again later."; - - crypto::ecdsa::key_pair account; - account.private_key = curve::fr::random_element(); - account.public_key = curve::g1::one * account.private_key; - - crypto::ecdsa::signature signature = - crypto::ecdsa::construct_signature(message_string, account); - - bool first_result = crypto::ecdsa::verify_signature( - message_string, account.public_key, signature); - - std::vector rr(signature.r.begin(), signature.r.end()); - std::vector ss(signature.s.begin(), signature.s.end()); - uint8_t vv = signature.v; - - curve::g1_bigfr_ct public_key = curve::g1_bigfr_ct::from_witness(&composer, account.public_key); - - proof_system::plonk::stdlib::ecdsa::signature sig{ curve::byte_array_ct(&composer, rr), - curve::byte_array_ct(&composer, ss), - proof_system::plonk::stdlib::uint8(&composer, - vv) }; - - curve::byte_array_ct message(&composer, message_string); - - for (size_t i = 0; i < num_iterations; i++) { - proof_system::plonk::stdlib::ecdsa:: - verify_signature( - message, public_key, sig); - } -} - -/** - * @brief Generate test circuit with specified number of merkle membership checks - * - * @param composer - * @param num_iterations - * @todo (luke): should we consider deeper tree? non-zero leaf values? variable index? - */ -void generate_merkle_membership_test_circuit(Composer& composer, size_t num_iterations) -{ - using namespace proof_system::plonk::stdlib; - using field_ct = field_t; - using witness_ct = witness_t; - using witness_ct = witness_t; - using MemStore = merkle_tree::MemoryStore; - using MerkleTree_ct = merkle_tree::MerkleTree; - - MemStore store; - auto db = MerkleTree_ct(store, 3); - - // Check that the leaf at index 0 has value 0. - auto zero = field_ct(witness_ct(&composer, fr::zero())).decompose_into_bits(); - field_ct root = witness_ct(&composer, db.root()); - - for (size_t i = 0; i < num_iterations; i++) { - merkle_tree::check_membership( - root, merkle_tree::create_witness_hash_path(composer, db.get_hash_path(0)), field_ct(0), zero); - } -} - -/** - * @brief Benchmark: Construction of a Ultra Honk proof for a circuit determined by the provided text circuit function - */ -void construct_proof_ultra(State& state, void (*test_circuit_function)(Composer&, size_t)) noexcept -{ - auto num_iterations = static_cast(state.range(0)); - for (auto _ : state) { - state.PauseTiming(); - auto composer = Composer(); - test_circuit_function(composer, num_iterations); - auto ext_prover = composer.create_prover(); - state.ResumeTiming(); - - auto proof = ext_prover.construct_proof(); - } -} - -BENCHMARK_CAPTURE(construct_proof_ultra, sha256, &generate_sha256_test_circuit) +// Define benchmarks +BENCHMARK_CAPTURE(construct_proof_ultra, sha256, &bench_utils::generate_sha256_test_circuit) ->DenseRange(MIN_NUM_ITERATIONS, MAX_NUM_ITERATIONS) - ->Repetitions(NUM_REPETITIONS); -BENCHMARK_CAPTURE(construct_proof_ultra, keccak, &generate_keccak_test_circuit) + ->Repetitions(NUM_REPETITIONS) + ->Unit(::benchmark::kSecond); +BENCHMARK_CAPTURE(construct_proof_ultra, keccak, &bench_utils::generate_keccak_test_circuit) ->DenseRange(MIN_NUM_ITERATIONS, MAX_NUM_ITERATIONS) - ->Repetitions(NUM_REPETITIONS); -BENCHMARK_CAPTURE(construct_proof_ultra, ecdsa_verification, &generate_ecdsa_verification_test_circuit) + ->Repetitions(NUM_REPETITIONS) + ->Unit(::benchmark::kSecond); +BENCHMARK_CAPTURE(construct_proof_ultra, + ecdsa_verification, + &bench_utils::generate_ecdsa_verification_test_circuit) ->DenseRange(MIN_NUM_ITERATIONS, MAX_NUM_ITERATIONS) - ->Repetitions(NUM_REPETITIONS); -BENCHMARK_CAPTURE(construct_proof_ultra, merkle_membership, &generate_merkle_membership_test_circuit) + ->Repetitions(NUM_REPETITIONS) + ->Unit(::benchmark::kSecond); +BENCHMARK_CAPTURE(construct_proof_ultra, + merkle_membership, + &bench_utils::generate_merkle_membership_test_circuit) ->DenseRange(MIN_NUM_ITERATIONS, MAX_NUM_ITERATIONS) - ->Repetitions(NUM_REPETITIONS); + ->Repetitions(NUM_REPETITIONS) + ->Unit(::benchmark::kSecond); } // namespace ultra_honk_bench \ No newline at end of file diff --git a/cpp/src/barretenberg/benchmark/honk_bench/ultra_plonk.bench.cpp b/cpp/src/barretenberg/benchmark/honk_bench/ultra_plonk.bench.cpp new file mode 100644 index 0000000000..064d5ef204 --- /dev/null +++ b/cpp/src/barretenberg/benchmark/honk_bench/ultra_plonk.bench.cpp @@ -0,0 +1,44 @@ +#include "barretenberg/benchmark/honk_bench/benchmark_utilities.hpp" + +using namespace benchmark; + +namespace ultra_plonk_bench { + +using UltraPlonk = proof_system::plonk::UltraPlonkComposer; + +// Number of times to perform operation of interest in the benchmark circuits, e.g. # of hashes to perform +constexpr size_t MIN_NUM_ITERATIONS = bench_utils::BenchParams::MIN_NUM_ITERATIONS; +constexpr size_t MAX_NUM_ITERATIONS = bench_utils::BenchParams::MAX_NUM_ITERATIONS; +// Number of times to repeat each benchmark +constexpr size_t NUM_REPETITIONS = bench_utils::BenchParams::NUM_REPETITIONS; + +/** + * @brief Benchmark: Construction of a Ultra Honk proof for a circuit determined by the provided circuit function + */ +void construct_proof_ultra(State& state, void (*test_circuit_function)(UltraPlonk&, size_t)) noexcept +{ + bench_utils::construct_proof_with_specified_num_iterations(state, test_circuit_function); +} + +BENCHMARK_CAPTURE(construct_proof_ultra, sha256, &bench_utils::generate_sha256_test_circuit) + ->DenseRange(MIN_NUM_ITERATIONS, MAX_NUM_ITERATIONS) + ->Repetitions(NUM_REPETITIONS) + ->Unit(::benchmark::kSecond); +BENCHMARK_CAPTURE(construct_proof_ultra, keccak, &bench_utils::generate_keccak_test_circuit) + ->DenseRange(MIN_NUM_ITERATIONS, MAX_NUM_ITERATIONS) + ->Repetitions(NUM_REPETITIONS) + ->Unit(::benchmark::kSecond); +BENCHMARK_CAPTURE(construct_proof_ultra, + ecdsa_verification, + &bench_utils::generate_ecdsa_verification_test_circuit) + ->DenseRange(MIN_NUM_ITERATIONS, MAX_NUM_ITERATIONS) + ->Repetitions(NUM_REPETITIONS) + ->Unit(::benchmark::kSecond); +BENCHMARK_CAPTURE(construct_proof_ultra, + merkle_membership, + &bench_utils::generate_merkle_membership_test_circuit) + ->DenseRange(MIN_NUM_ITERATIONS, MAX_NUM_ITERATIONS) + ->Repetitions(NUM_REPETITIONS) + ->Unit(::benchmark::kSecond); + +} // namespace ultra_plonk_bench \ No newline at end of file