Skip to content

Commit

Permalink
feat: Manual ClientIVC breakdown (#4778)
Browse files Browse the repository at this point in the history
Add timers to more functions used in client IVC scheme. Add scripts to
get benchmarks and analyze them, giving a coarse profile of client IVC
execution in x86.

Small cleanup: rename ivc_bench module to client_ivc_bench for
uniformity and because we will have other IVC schemes.

Results on commit c2ab55e
```
Benchmarking lock created at ~/BENCHMARK_IN_PROGRESS.
client_ivc_bench                                                                                                                                            100%   16MB  49.3MB/s   00:00    
2024-02-27T14:44:25+00:00
Running ./client_ivc_bench
Run on (16 X 3597.59 MHz CPU s)
CPU Caches:
  L1 Data 32 KiB (x8)
  L1 Instruction 32 KiB (x8)
  L2 Unified 1024 KiB (x8)
  L3 Unified 36608 KiB (x1)
Load Average: 0.00, 0.00, 0.00
--------------------------------------------------------------------------------
Benchmark                      Time             CPU   Iterations UserCounters...
--------------------------------------------------------------------------------
ClientIVCBench/Full/6      39503 ms        37113 ms            1 Decider::construct_proof=1 Decider::construct_proof(t)=759.097M ECCVMComposer::create_prover=1 ECCVMComposer::create_prover(t)=3.77773G ECCVMProver::construct_proof=1 ECCVMProver::construct_proof(t)=1.76856G Goblin::merge=12 Goblin::merge(t)=153.269M GoblinTranslatorCircuitBuilder::constructor=1 GoblinTranslatorCircuitBuilder::constructor(t)=61.9766M GoblinTranslatorComposer::create_prover=1 GoblinTranslatorComposer::create_prover(t)=114.006M GoblinTranslatorProver::construct_proof=1 GoblinTranslatorProver::construct_proof(t)=939.454M ProtogalaxyProver::fold_instances=11 ProtogalaxyProver::fold_instances(t)=18.0888G UltraComposer::create_prover_instance=12 UltraComposer::create_prover_instance(t)=6.31421G batch_mul_with_endomorphism=30 batch_mul_with_endomorphism(t)=563.904M commit=804 commit(t)=8.10774G compute_combiner=11 compute_combiner(t)=7.92258G compute_perturbator=10 compute_perturbator(t)=1.52886G construct_mock_folding_kernel=5 construct_mock_folding_kernel(t)=2.33398G construct_mock_function_circuit=7 construct_mock_function_circuit(t)=5.16551G
Benchmarking lock deleted.
client_ivc_bench.json                                                                                                                                       100% 3063    85.5KB/s   00:00    
label                                           ms   % total
construct_mock_function_circuit(t)            5166    13.08%
construct_mock_folding_kernel(t)              2334     5.91%
UltraComposer::create_prover_instance(t)      6314    15.98%
ProtogalaxyProver::fold_instances(t)         18089    45.79%
Decider::construct_proof(t)                    759     1.92%
ECCVMComposer::create_prover(t)               3778     9.56%
GoblinTranslatorComposer::create_prover(t)     114     0.29%
ECCVMProver::construct_proof(t)               1769     4.48%
GoblinTranslatorProver::construct_proof(t)     939     2.38%
Goblin::merge(t)                               153     0.39%
```
  • Loading branch information
codygunton authored Feb 27, 2024
1 parent 61067a5 commit b4cfc89
Show file tree
Hide file tree
Showing 16 changed files with 98 additions and 17 deletions.
44 changes: 44 additions & 0 deletions barretenberg/cpp/scripts/analyze_client_ivc_bench.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
import json
from pathlib import Path

PREFIX = Path("build-op-count-time")
IVC_BENCH_JSON = Path("client_ivc_bench.json")
BENCHMARK = "ClientIVCBench/Full/6"

# Single out an independent set of functions accounting for most of BENCHMARK's real_time
to_keep = [
"construct_mock_function_circuit(t)",
"construct_mock_folding_kernel(t)",
"UltraComposer::create_prover_instance(t)",
"ProtogalaxyProver::fold_instances(t)",
"Decider::construct_proof(t)",
"ECCVMComposer::create_prover(t)",
"GoblinTranslatorComposer::create_prover(t)",
"ECCVMProver::construct_proof(t)",
"GoblinTranslatorProver::construct_proof(t)",
"Goblin::merge(t)"
]
with open(PREFIX/IVC_BENCH_JSON, "r") as read_file:
read_result = json.load(read_file)
for _bench in read_result["benchmarks"]:
if _bench["name"] == BENCHMARK:
bench = _bench
bench_components = dict(filter(lambda x: x[0] in to_keep, bench.items()))

# For each kept time, get the proportion over all kept times.
sum_of_kept_times_ms = sum(float(time)
for _, time in bench_components.items())/1e6
MAX_LABEL_LENGTH = max(len(label) for label in to_keep)
column = {"function": "function", "ms": "ms", "%": "% sum"}
print(
f"{column['function']:<{MAX_LABEL_LENGTH}}{column['ms']:>8} {column['%']:>8}")
for key in to_keep:
time_ms = bench[key]/1e6
print(f"{key:<{MAX_LABEL_LENGTH}}{time_ms:>8.0f} {time_ms/sum_of_kept_times_ms:>8.2%}")

# Validate that kept times account for most of the total measured time.
total_time_ms = bench["real_time"]
totals = '\nTotal time accounted for: {:.0f}ms/{:.0f}ms = {:.2%}'
totals = totals.format(
sum_of_kept_times_ms, total_time_ms, sum_of_kept_times_ms/total_time_ms)
print(totals)
25 changes: 25 additions & 0 deletions barretenberg/cpp/scripts/benchmark_client_ivc.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
#!/usr/bin/env bash
set -eu

TARGET="client_ivc_bench"
FILTER="ClientIVCBench/Full/6$"
BUILD_DIR=build-op-count-time

# Move above script dir.
cd $(dirname $0)/..

# Measure the benchmarks with ops time counting
./scripts/benchmark_remote.sh client_ivc_bench\
"./client_ivc_bench --benchmark_filter=$FILTER\
--benchmark_out=$TARGET.json\
--benchmark_out_format=json"\
op-count-time\
build-op-count-time

# Retrieve output from benching instance
cd $BUILD_DIR
scp $BB_SSH_KEY $BB_SSH_INSTANCE:$BB_SSH_CPP_PATH/build/$TARGET.json .

# Analyze the results
cd $(dirname $0)/..
python3 ./scripts/analyze_client_ivc_bench.py
2 changes: 1 addition & 1 deletion barretenberg/cpp/src/barretenberg/benchmark/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ add_subdirectory(basics_bench)
add_subdirectory(decrypt_bench)
add_subdirectory(goblin_bench)
add_subdirectory(ipa_bench)
add_subdirectory(ivc_bench)
add_subdirectory(client_ivc_bench)
add_subdirectory(pippenger_bench)
add_subdirectory(plonk_bench)
add_subdirectory(protogalaxy_bench)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
barretenberg_module(client_ivc_bench client_ivc stdlib_recursion stdlib_sha256 crypto_merkle_tree stdlib_primitives)
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@

#include <benchmark/benchmark.h>

#include "barretenberg/benchmark/ultra_bench/mock_proofs.hpp"
#include "barretenberg/client_ivc/client_ivc.hpp"
#include "barretenberg/common/op_count_google_bench.hpp"
#include "barretenberg/goblin/goblin.hpp"
#include "barretenberg/goblin/mock_circuits.hpp"
#include "barretenberg/proof_system/circuit_builder/ultra_circuit_builder.hpp"
#include "barretenberg/ultra_honk/ultra_composer.hpp"
Expand All @@ -18,7 +16,7 @@ namespace {
* @brief Benchmark suite for the aztec client PG-Goblin IVC scheme
*
*/
class IvcBench : public benchmark::Fixture {
class ClientIVCBench : public benchmark::Fixture {
public:
using Builder = GoblinUltraCircuitBuilder;

Expand Down Expand Up @@ -74,7 +72,7 @@ class IvcBench : public benchmark::Fixture {
* @brief Benchmark the prover work for the full PG-Goblin IVC protocol
*
*/
BENCHMARK_DEFINE_F(IvcBench, Full)(benchmark::State& state)
BENCHMARK_DEFINE_F(ClientIVCBench, Full)(benchmark::State& state)
{
ClientIVC ivc;

Expand All @@ -92,7 +90,7 @@ BENCHMARK_DEFINE_F(IvcBench, Full)(benchmark::State& state)
* @brief Benchmark only the accumulation rounds
*
*/
BENCHMARK_DEFINE_F(IvcBench, Accumulate)(benchmark::State& state)
BENCHMARK_DEFINE_F(ClientIVCBench, Accumulate)(benchmark::State& state)
{
ClientIVC ivc;

Expand All @@ -107,7 +105,7 @@ BENCHMARK_DEFINE_F(IvcBench, Accumulate)(benchmark::State& state)
* @brief Benchmark only the Decider component
*
*/
BENCHMARK_DEFINE_F(IvcBench, Decide)(benchmark::State& state)
BENCHMARK_DEFINE_F(ClientIVCBench, Decide)(benchmark::State& state)
{
ClientIVC ivc;

Expand All @@ -125,7 +123,7 @@ BENCHMARK_DEFINE_F(IvcBench, Decide)(benchmark::State& state)
* @brief Benchmark only the ECCVM component
*
*/
BENCHMARK_DEFINE_F(IvcBench, ECCVM)(benchmark::State& state)
BENCHMARK_DEFINE_F(ClientIVCBench, ECCVM)(benchmark::State& state)
{
ClientIVC ivc;

Expand All @@ -143,7 +141,7 @@ BENCHMARK_DEFINE_F(IvcBench, ECCVM)(benchmark::State& state)
* @brief Benchmark only the Translator component
*
*/
BENCHMARK_DEFINE_F(IvcBench, Translator)(benchmark::State& state)
BENCHMARK_DEFINE_F(ClientIVCBench, Translator)(benchmark::State& state)
{
ClientIVC ivc;

Expand All @@ -159,7 +157,7 @@ BENCHMARK_DEFINE_F(IvcBench, Translator)(benchmark::State& state)
}

#define ARGS \
Arg(IvcBench::NUM_ITERATIONS_MEDIUM_COMPLEXITY) \
Arg(ClientIVCBench::NUM_ITERATIONS_MEDIUM_COMPLEXITY) \
->Arg(1 << 0) \
->Arg(1 << 1) \
->Arg(1 << 2) \
Expand All @@ -168,11 +166,11 @@ BENCHMARK_DEFINE_F(IvcBench, Translator)(benchmark::State& state)
->Arg(1 << 5) \
->Arg(1 << 6)

BENCHMARK_REGISTER_F(IvcBench, Full)->Unit(benchmark::kMillisecond)->ARGS;
BENCHMARK_REGISTER_F(IvcBench, Accumulate)->Unit(benchmark::kMillisecond)->ARGS;
BENCHMARK_REGISTER_F(IvcBench, Decide)->Unit(benchmark::kMillisecond)->ARGS;
BENCHMARK_REGISTER_F(IvcBench, ECCVM)->Unit(benchmark::kMillisecond)->ARGS;
BENCHMARK_REGISTER_F(IvcBench, Translator)->Unit(benchmark::kMillisecond)->ARGS;
BENCHMARK_REGISTER_F(ClientIVCBench, Full)->Unit(benchmark::kMillisecond)->ARGS;
BENCHMARK_REGISTER_F(ClientIVCBench, Accumulate)->Unit(benchmark::kMillisecond)->ARGS;
BENCHMARK_REGISTER_F(ClientIVCBench, Decide)->Unit(benchmark::kMillisecond)->ARGS;
BENCHMARK_REGISTER_F(ClientIVCBench, ECCVM)->Unit(benchmark::kMillisecond)->ARGS;
BENCHMARK_REGISTER_F(ClientIVCBench, Translator)->Unit(benchmark::kMillisecond)->ARGS;

} // namespace

Expand Down

This file was deleted.

2 changes: 2 additions & 0 deletions barretenberg/cpp/src/barretenberg/eccvm/eccvm_composer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ template <IsECCVMFlavor Flavor>
ECCVMProver_<Flavor> ECCVMComposer_<Flavor>::create_prover(CircuitConstructor& circuit_constructor,
const std::shared_ptr<Transcript>& transcript)
{
BB_OP_COUNT_TIME_NAME("ECCVMComposer::create_prover");

compute_proving_key(circuit_constructor);
compute_witness(circuit_constructor);
compute_commitment_key(proving_key->circuit_size);
Expand Down
2 changes: 2 additions & 0 deletions barretenberg/cpp/src/barretenberg/eccvm/eccvm_prover.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -282,6 +282,8 @@ template <IsECCVMFlavor Flavor> HonkProof& ECCVMProver_<Flavor>::export_proof()

template <IsECCVMFlavor Flavor> HonkProof& ECCVMProver_<Flavor>::construct_proof()
{
BB_OP_COUNT_TIME_NAME("ECCVMProver::construct_proof");

execute_preamble_round();

execute_wire_commitments_round();
Expand Down
1 change: 1 addition & 0 deletions barretenberg/cpp/src/barretenberg/goblin/goblin.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,7 @@ class Goblin {
*/
void merge(GoblinUltraCircuitBuilder& circuit_builder)
{
BB_OP_COUNT_TIME_NAME("Goblin::merge");
// Complete the circuit logic by recursively verifying previous merge proof if it exists
if (merge_proof_exists) {
RecursiveMergeVerifier merge_verifier{ &circuit_builder };
Expand Down
1 change: 0 additions & 1 deletion barretenberg/cpp/src/barretenberg/polynomials/pow.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,6 @@ template <typename FF> struct PowPolynomial {
*/
BB_PROFILE void compute_values()
{
BB_OP_COUNT_TIME();
size_t pow_size = 1 << betas.size();
pow_betas = std::vector<FF>(pow_size);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -364,6 +364,7 @@ class GoblinTranslatorCircuitBuilder : public CircuitBuilderBase<bb::fr> {
std::shared_ptr<ECCOpQueue> op_queue)
: GoblinTranslatorCircuitBuilder(batching_challenge_v_, evaluation_input_x_)
{
BB_OP_COUNT_TIME_NAME("GoblinTranslatorCircuitBuilder::constructor");
feed_ecc_op_queue_into_circuit(op_queue);
}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
#include "decider_prover.hpp"
#include "barretenberg/common/op_count.hpp"
#include "barretenberg/sumcheck/sumcheck.hpp"

namespace bb {
Expand Down Expand Up @@ -102,6 +103,8 @@ template <IsUltraFlavor Flavor> HonkProof& DeciderProver_<Flavor>::export_proof(

template <IsUltraFlavor Flavor> HonkProof& DeciderProver_<Flavor>::construct_proof()
{
BB_OP_COUNT_TIME_NAME("Decider::construct_proof");

// Add ϕ, \vec{β*}, e* to transcript
execute_preamble_round();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -346,6 +346,7 @@ template <class ProverInstances> void ProtoGalaxyProver_<ProverInstances>::accum
template <class ProverInstances>
FoldingResult<typename ProverInstances::Flavor> ProtoGalaxyProver_<ProverInstances>::fold_instances()
{
BB_OP_COUNT_TIME_NAME("ProtogalaxyProver::fold_instances");
preparation_round();
perturbator_round();
combiner_quotient_round();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,7 @@ void GoblinTranslatorComposer::compute_witness(CircuitBuilder& circuit_builder)
GoblinTranslatorProver GoblinTranslatorComposer::create_prover(CircuitBuilder& circuit_builder,
const std::shared_ptr<Transcript>& transcript)
{
BB_OP_COUNT_TIME_NAME("GoblinTranslatorComposer::create_prover");

// Compute total number of gates, dyadic circuit size, etc.
compute_circuit_size_parameters(circuit_builder);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,8 @@ HonkProof& GoblinTranslatorProver::export_proof()

HonkProof& GoblinTranslatorProver::construct_proof()
{
BB_OP_COUNT_TIME_NAME("GoblinTranslatorProver::construct_proof");

// Add circuit size public input size and public inputs to transcript.
execute_preamble_round();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,8 @@ void UltraComposer_<Flavor>::compute_verification_key(const std::shared_ptr<Prov
template <IsUltraFlavor Flavor>
std::shared_ptr<ProverInstance_<Flavor>> UltraComposer_<Flavor>::create_instance(CircuitBuilder& circuit)
{
BB_OP_COUNT_TIME_NAME("UltraComposer::create_prover_instance");

circuit.add_gates_to_ensure_all_polys_are_non_zero();
circuit.finalize_circuit();
auto instance = std::make_shared<Instance>(circuit);
Expand Down

0 comments on commit b4cfc89

Please sign in to comment.