feat: mega zk features (#9774)

Please read [contributing guidelines](CONTRIBUTING.md) and remove this
line.

barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/mega_zk_honk.bench.cpp

@@ -0,0 +1,51 @@
+#include <benchmark/benchmark.h>
+
+#include "barretenberg/benchmark/ultra_bench/mock_circuits.hpp"
+#include "barretenberg/stdlib_circuit_builders/mega_circuit_builder.hpp"
+
+using namespace benchmark;
+using namespace bb;
+
+/**
+ * @brief Benchmark: Construction of a Ultra Honk proof for a circuit determined by the provided circuit function
+ */
+static void construct_proof_megahonk_zk(State& state,
+                                        void (*test_circuit_function)(MegaCircuitBuilder&, size_t)) noexcept
+{
+    size_t num_iterations = 10; // 10x the circuit
+    bb::mock_circuits::construct_proof_with_specified_num_iterations<MegaZKProver>(
+        state, test_circuit_function, num_iterations);
+}
+
+/**
+ * @brief Benchmark: Construction of a Ultra Plonk proof with 2**n gates
+ */
+static void construct_proof_megahonk_power_of_2_zk(State& state) noexcept
+{
+    auto log2_of_gates = static_cast<size_t>(state.range(0));
+    bb::mock_circuits::construct_proof_with_specified_num_iterations<MegaZKProver>(
+        state, &bb::mock_circuits::generate_basic_arithmetic_circuit<MegaCircuitBuilder>, log2_of_gates);
+}
+
+// Define benchmarks
+
+// This exists due to an issue where get_row was blowing up in time
+BENCHMARK_CAPTURE(construct_proof_megahonk_zk, sha256, &stdlib::generate_sha256_test_circuit<MegaCircuitBuilder>)
+    ->Unit(kMillisecond);
+BENCHMARK_CAPTURE(construct_proof_megahonk_zk, keccak, &stdlib::generate_keccak_test_circuit<MegaCircuitBuilder>)
+    ->Unit(kMillisecond);
+BENCHMARK_CAPTURE(construct_proof_megahonk_zk,
+                  ecdsa_verification,
+                  &stdlib::generate_ecdsa_verification_test_circuit<MegaCircuitBuilder>)
+    ->Unit(kMillisecond);
+BENCHMARK_CAPTURE(construct_proof_megahonk_zk,
+                  merkle_membership,
+                  &stdlib::generate_merkle_membership_test_circuit<MegaCircuitBuilder>)
+    ->Unit(kMillisecond);
+
+BENCHMARK(construct_proof_megahonk_power_of_2_zk)
+    // 2**15 gates to 2**20 gates
+    ->DenseRange(15, 20)
+    ->Unit(kMillisecond);
+
+BENCHMARK_MAIN();

barretenberg/cpp/src/barretenberg/commitment_schemes/kzg/kzg.test.cpp

@@ -318,6 +318,8 @@ TYPED_TEST(KZGTest, ShpleminiKzgWithShiftAndConcatenation)
                                                        mle_opening_point,
                                                        this->vk()->get_g1_identity(),
                                                        verifier_transcript,
+                                                       /* libra commitments = */ {},
+                                                       /* libra evaluations = */ {},
                                                        to_vector_of_ref_vectors(concatenation_groups_commitments),
                                                        RefVector(c_evaluations));
     const auto pairing_points = KZG::reduce_verify_batch_opening_claim(batch_opening_claim, verifier_transcript);

barretenberg/cpp/src/barretenberg/commitment_schemes/shplonk/shplemini.hpp

@@ -27,10 +27,10 @@ template <typename Curve> class ShpleminiProver_ {
                               std::span<FF> multilinear_challenge,
                               const std::shared_ptr<CommitmentKey<Curve>>& commitment_key,
                               const std::shared_ptr<Transcript>& transcript,
-                              RefSpan<Polynomial> concatenated_polynomials = {},
-                              const std::vector<RefVector<Polynomial>>& groups_to_be_concatenated = {},
                               const std::vector<bb::Univariate<FF, LENGTH>>& libra_univariates = {},
-                              const std::vector<FF>& libra_evaluations = {})
+                              const std::vector<FF>& libra_evaluations = {},
+                              RefSpan<Polynomial> concatenated_polynomials = {},
+                              const std::vector<RefVector<Polynomial>>& groups_to_be_concatenated = {})
     {
         std::vector<OpeningClaim> opening_claims = GeminiProver::prove(circuit_size,
                                                                        f_polynomials,
@@ -129,10 +129,10 @@ template <typename Curve> class ShpleminiVerifier_ {
         const std::vector<Fr>& multivariate_challenge,
         const Commitment& g1_identity,
         const std::shared_ptr<Transcript>& transcript,
-        const std::vector<RefVector<Commitment>>& concatenation_group_commitments = {},
-        RefSpan<Fr> concatenated_evaluations = {},
         RefSpan<Commitment> libra_univariate_commitments = {},
-        const std::vector<Fr>& libra_univariate_evaluations = {})
+        const std::vector<Fr>& libra_univariate_evaluations = {},
+        const std::vector<RefVector<Commitment>>& concatenation_group_commitments = {},
+        RefSpan<Fr> concatenated_evaluations = {})
 
     {
 

barretenberg/cpp/src/barretenberg/commitment_schemes/shplonk/shplemini.test.cpp

@@ -302,8 +302,6 @@ TYPED_TEST(ShpleminiTest, ShpleminiWithMaskingLibraUnivariates)
                                                 mle_opening_point,
                                                 this->ck(),
                                                 prover_transcript,
-                                                /* concatenated_polynomials = */ {},
-                                                /* groups_to_be_concatenated = */ {},
                                                 libra_univariates,
                                                 libra_evaluations);
     if constexpr (std::is_same_v<TypeParam, curve::Grumpkin>) {
@@ -318,19 +316,16 @@ TYPED_TEST(ShpleminiTest, ShpleminiWithMaskingLibraUnivariates)
 
     // Gemini verifier output:
     // - claim: d+1 commitments to Fold_{r}^(0), Fold_{-r}^(0), Fold^(l), d+1 evaluations a_0_pos, a_l, l = 0:d-1
-    auto batch_opening_claim =
-        ShpleminiVerifier::compute_batch_opening_claim(n,
-                                                       RefVector(unshifted_commitments),
-                                                       RefVector(shifted_commitments),
-                                                       RefArray{ eval1, eval2, eval3, eval4 },
-                                                       RefArray{ eval2_shift, eval3_shift },
-                                                       mle_opening_point,
-                                                       this->vk()->get_g1_identity(),
-                                                       verifier_transcript,
-                                                       /* concatenation_group_commitments = */ {},
-                                                       /* concatenated_evaluations = */ {},
-                                                       RefVector(libra_commitments),
-                                                       libra_evaluations);
+    auto batch_opening_claim = ShpleminiVerifier::compute_batch_opening_claim(n,
+                                                                              RefVector(unshifted_commitments),
+                                                                              RefVector(shifted_commitments),
+                                                                              RefArray{ eval1, eval2, eval3, eval4 },
+                                                                              RefArray{ eval2_shift, eval3_shift },
+                                                                              mle_opening_point,
+                                                                              this->vk()->get_g1_identity(),
+                                                                              verifier_transcript,
+                                                                              RefVector(libra_commitments),
+                                                                              libra_evaluations);
 
     if constexpr (std::is_same_v<TypeParam, curve::Grumpkin>) {
         auto result = IPA::reduce_verify_batch_opening_claim(batch_opening_claim, this->vk(), verifier_transcript);

barretenberg/cpp/src/barretenberg/eccvm/eccvm_prover.cpp

@@ -100,7 +100,9 @@ void ECCVMProver::execute_relation_check_rounds()
     for (size_t idx = 0; idx < CONST_PROOF_SIZE_LOG_N; idx++) {
         gate_challenges[idx] = transcript->template get_challenge<FF>("Sumcheck:gate_challenge_" + std::to_string(idx));
     }
-    zk_sumcheck_data = ZKSumcheckData<Flavor>(key->log_circuit_size, transcript, key);
+
+    auto commitment_key = std::make_shared<CommitmentKey>(Flavor::BATCHED_RELATION_PARTIAL_LENGTH);
+    zk_sumcheck_data = ZKSumcheckData<Flavor>(key->log_circuit_size, transcript, commitment_key);
 
     sumcheck_output = sumcheck.prove(key->polynomials, relation_parameters, alpha, gate_challenges, zk_sumcheck_data);
 }
@@ -127,8 +129,6 @@ void ECCVMProver::execute_pcs_rounds()
                          sumcheck_output.challenge,
                          key->commitment_key,
                          transcript,
-                         /* concatenated_polynomials = */ {},
-                         /* groups_to_be_concatenated = */ {},
                          zk_sumcheck_data.libra_univariates_monomial,
                          sumcheck_output.claimed_libra_evaluations);
 

barretenberg/cpp/src/barretenberg/eccvm/eccvm_verifier.cpp

@@ -78,8 +78,6 @@ bool ECCVMVerifier::verify_proof(const HonkProof& proof)
                                                multivariate_challenge,
                                                key->pcs_verification_key->get_g1_identity(),
                                                transcript,
-                                               /* concatenation_group_commitments = */ {},
-                                               /* concatenated_evaluations = */ {},
                                                RefVector(libra_commitments),
                                                libra_evaluations);
 

barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.cpp

@@ -1,7 +1,7 @@
 #include "execution_trace.hpp"
 #include "barretenberg/flavor/plonk_flavors.hpp"
 #include "barretenberg/plonk/proof_system/proving_key/proving_key.hpp"
-#include "barretenberg/stdlib_circuit_builders/mega_flavor.hpp"
+#include "barretenberg/stdlib_circuit_builders/mega_zk_flavor.hpp"
 #include "barretenberg/stdlib_circuit_builders/ultra_flavor.hpp"
 #include "barretenberg/stdlib_circuit_builders/ultra_keccak_flavor.hpp"
 namespace bb {
@@ -171,6 +171,7 @@ void ExecutionTrace_<Flavor>::add_ecc_op_wires_to_proving_key(Builder& builder,
 template class ExecutionTrace_<UltraFlavor>;
 template class ExecutionTrace_<UltraKeccakFlavor>;
 template class ExecutionTrace_<MegaFlavor>;
+template class ExecutionTrace_<MegaZKFlavor>;
 template class ExecutionTrace_<plonk::flavor::Standard>;
 template class ExecutionTrace_<plonk::flavor::Ultra>;
 

barretenberg/cpp/src/barretenberg/flavor/flavor.hpp

@@ -326,10 +326,12 @@ class UltraFlavorWithZK;
 class ECCVMFlavor;
 class UltraKeccakFlavor;
 class MegaFlavor;
+class MegaZKFlavor;
 class TranslatorFlavor;
 class AvmFlavor;
 template <typename BuilderType> class UltraRecursiveFlavor_;
 template <typename BuilderType> class MegaRecursiveFlavor_;
+template <typename BuilderType> class MegaZKRecursiveFlavor_;
 template <typename BuilderType> class TranslatorRecursiveFlavor_;
 template <typename BuilderType> class ECCVMRecursiveFlavor_;
 template <typename BuilderType> class AvmRecursiveFlavor_;
@@ -361,15 +363,18 @@ template <typename T>
 concept IsUltraPlonkOrHonk = IsAnyOf<T, plonk::flavor::Ultra, UltraFlavor, UltraKeccakFlavor, UltraFlavorWithZK, MegaFlavor>;
 
 template <typename T>
-concept IsHonkFlavor = IsAnyOf<T, UltraFlavor, UltraKeccakFlavor, UltraFlavorWithZK, MegaFlavor>;
+concept IsHonkFlavor = IsAnyOf<T, UltraFlavor, UltraKeccakFlavor, UltraFlavorWithZK, MegaFlavor, MegaZKFlavor>;
 
 template <typename T>
-concept IsUltraFlavor = IsAnyOf<T, UltraFlavor, UltraKeccakFlavor, UltraFlavorWithZK, MegaFlavor>;
+concept IsUltraFlavor = IsAnyOf<T, UltraFlavor, UltraKeccakFlavor, UltraFlavorWithZK, MegaFlavor, MegaZKFlavor>;
 
 template <typename T>
-concept IsGoblinFlavor = IsAnyOf<T, MegaFlavor,
+concept IsGoblinFlavor = IsAnyOf<T, MegaFlavor, MegaZKFlavor,
                                     MegaRecursiveFlavor_<UltraCircuitBuilder>,
-                                    MegaRecursiveFlavor_<MegaCircuitBuilder>, MegaRecursiveFlavor_<CircuitSimulatorBN254>>;
+                                    MegaRecursiveFlavor_<MegaCircuitBuilder>,
+MegaRecursiveFlavor_<CircuitSimulatorBN254>,
+MegaZKRecursiveFlavor_<MegaCircuitBuilder>,
+MegaZKRecursiveFlavor_<UltraCircuitBuilder>>;
 
 template <typename T>
 concept HasDataBus = IsGoblinFlavor<T>;
@@ -381,6 +386,8 @@ concept IsRecursiveFlavor = IsAnyOf<T, UltraRecursiveFlavor_<UltraCircuitBuilder
                                        MegaRecursiveFlavor_<UltraCircuitBuilder>,
                                        MegaRecursiveFlavor_<MegaCircuitBuilder>,
 MegaRecursiveFlavor_<CircuitSimulatorBN254>,
+MegaZKRecursiveFlavor_<MegaCircuitBuilder>,
+MegaZKRecursiveFlavor_<UltraCircuitBuilder>,
 TranslatorRecursiveFlavor_<UltraCircuitBuilder>,
 TranslatorRecursiveFlavor_<MegaCircuitBuilder>,
 TranslatorRecursiveFlavor_<CircuitSimulatorBN254>,
@@ -397,11 +404,15 @@ template <typename T> concept IsFoldingFlavor = IsAnyOf<T, UltraFlavor,
                                                            UltraKeccakFlavor,
                                                            UltraFlavorWithZK,
                                                            MegaFlavor,
+                                                           MegaZKFlavor,
                                                            UltraRecursiveFlavor_<UltraCircuitBuilder>,
                                                            UltraRecursiveFlavor_<MegaCircuitBuilder>,
                                                            UltraRecursiveFlavor_<CircuitSimulatorBN254>,
                                                            MegaRecursiveFlavor_<UltraCircuitBuilder>,
-                                                           MegaRecursiveFlavor_<MegaCircuitBuilder>, MegaRecursiveFlavor_<CircuitSimulatorBN254>>;
+                                                           MegaRecursiveFlavor_<MegaCircuitBuilder>,
+MegaRecursiveFlavor_<CircuitSimulatorBN254>,
+MegaZKRecursiveFlavor_<MegaCircuitBuilder>,
+MegaZKRecursiveFlavor_<UltraCircuitBuilder>>;
 template <typename T>
 concept FlavorHasZK =  T::HasZK;
 

barretenberg/cpp/src/barretenberg/stdlib/eccvm_verifier/eccvm_recursive_verifier.cpp

@@ -92,8 +92,6 @@ template <typename Flavor> void ECCVMRecursiveVerifier_<Flavor>::verify_proof(co
                                                multivariate_challenge,
                                                key->pcs_verification_key->get_g1_identity(),
                                                transcript,
-                                               /* concatenation_group_commitments = */ {},
-                                               /* concatenated_evaluations = */ {},
                                                RefVector(libra_commitments),
                                                libra_evaluations);
     // Reduce the accumulator to a single opening claim

barretenberg/cpp/src/barretenberg/stdlib/honk_verifier/oink_recursive_verifier.cpp

@@ -2,7 +2,9 @@
 
 #include "barretenberg/numeric/bitop/get_msb.hpp"
 #include "barretenberg/plonk_honk_shared/library/grand_product_delta.hpp"
-#include "barretenberg/transcript/transcript.hpp"
+#include "barretenberg/stdlib_circuit_builders/mega_recursive_flavor.hpp"
+#include "barretenberg/stdlib_circuit_builders/mega_zk_recursive_flavor.hpp"
+#include "barretenberg/stdlib_circuit_builders/ultra_recursive_flavor.hpp"
 #include <utility>
 
 namespace bb::stdlib::recursion::honk {
@@ -130,6 +132,8 @@ template class OinkRecursiveVerifier_<bb::UltraRecursiveFlavor_<UltraCircuitBuil
 template class OinkRecursiveVerifier_<bb::UltraRecursiveFlavor_<MegaCircuitBuilder>>;
 template class OinkRecursiveVerifier_<bb::MegaRecursiveFlavor_<UltraCircuitBuilder>>;
 template class OinkRecursiveVerifier_<bb::MegaRecursiveFlavor_<MegaCircuitBuilder>>;
+template class OinkRecursiveVerifier_<bb::MegaZKRecursiveFlavor_<MegaCircuitBuilder>>;
+template class OinkRecursiveVerifier_<bb::MegaZKRecursiveFlavor_<UltraCircuitBuilder>>;
 template class OinkRecursiveVerifier_<bb::UltraRecursiveFlavor_<CircuitSimulatorBN254>>;
 template class OinkRecursiveVerifier_<bb::MegaRecursiveFlavor_<CircuitSimulatorBN254>>;
 } // namespace bb::stdlib::recursion::honk

barretenberg/cpp/src/barretenberg/stdlib/honk_verifier/oink_recursive_verifier.hpp

@@ -1,8 +1,6 @@
 #pragma once
 #include "barretenberg/stdlib/protogalaxy_verifier/recursive_decider_verification_key.hpp"
 #include "barretenberg/stdlib/transcript/transcript.hpp"
-#include "barretenberg/stdlib_circuit_builders/mega_recursive_flavor.hpp"
-#include "barretenberg/stdlib_circuit_builders/ultra_recursive_flavor.hpp"
 
 namespace bb::stdlib::recursion::honk {
 

barretenberg/cpp/src/barretenberg/stdlib/honk_verifier/ultra_recursive_verifier.cpp

@@ -89,18 +89,36 @@ UltraRecursiveVerifier_<Flavor>::AggregationObject UltraRecursiveVerifier_<Flavo
     const size_t log_circuit_size = numeric::get_msb(static_cast<uint32_t>(key->circuit_size));
     auto sumcheck = Sumcheck(log_circuit_size, transcript);
 
-    auto [multivariate_challenge, claimed_evaluations, sumcheck_verified] =
+    // Receive commitments to Libra masking polynomials
+    std::vector<Commitment> libra_commitments = {};
+    if constexpr (Flavor::HasZK) {
+        for (size_t idx = 0; idx < log_circuit_size; idx++) {
+            Commitment libra_commitment =
+                transcript->template receive_from_prover<Commitment>("Libra:commitment_" + std::to_string(idx));
+            libra_commitments.push_back(libra_commitment);
+        };
+    }
+    SumcheckOutput<Flavor> sumcheck_output =
         sumcheck.verify(verification_key->relation_parameters, verification_key->alphas, gate_challenges);
 
+    // For MegaZKFlavor: the sumcheck output contains claimed evaluations of the Libra polynomials
+    std::vector<FF> libra_evaluations = {};
+    if constexpr (Flavor::HasZK) {
+        libra_evaluations = std::move(sumcheck_output.claimed_libra_evaluations);
+    }
+
     // Execute Shplemini to produce a batch opening claim subsequently verified by a univariate PCS
-    auto opening_claim = Shplemini::compute_batch_opening_claim(key->circuit_size,
-                                                                commitments.get_unshifted(),
-                                                                commitments.get_to_be_shifted(),
-                                                                claimed_evaluations.get_unshifted(),
-                                                                claimed_evaluations.get_shifted(),
-                                                                multivariate_challenge,
-                                                                Commitment::one(builder),
-                                                                transcript);
+    const BatchOpeningClaim<Curve> opening_claim =
+        Shplemini::compute_batch_opening_claim(key->circuit_size,
+                                               commitments.get_unshifted(),
+                                               commitments.get_to_be_shifted(),
+                                               sumcheck_output.claimed_evaluations.get_unshifted(),
+                                               sumcheck_output.claimed_evaluations.get_shifted(),
+                                               sumcheck_output.challenge,
+                                               Commitment::one(builder),
+                                               transcript,
+                                               RefVector(libra_commitments),
+                                               libra_evaluations);
     auto pairing_points = PCS::reduce_verify_batch_opening_claim(opening_claim, transcript);
 
     pairing_points[0] = pairing_points[0].normalize();
@@ -114,6 +132,8 @@ template class UltraRecursiveVerifier_<bb::UltraRecursiveFlavor_<UltraCircuitBui
 template class UltraRecursiveVerifier_<bb::UltraRecursiveFlavor_<MegaCircuitBuilder>>;
 template class UltraRecursiveVerifier_<bb::MegaRecursiveFlavor_<UltraCircuitBuilder>>;
 template class UltraRecursiveVerifier_<bb::MegaRecursiveFlavor_<MegaCircuitBuilder>>;
+template class UltraRecursiveVerifier_<bb::MegaZKRecursiveFlavor_<MegaCircuitBuilder>>;
+template class UltraRecursiveVerifier_<bb::MegaZKRecursiveFlavor_<UltraCircuitBuilder>>;
 template class UltraRecursiveVerifier_<bb::UltraRecursiveFlavor_<CircuitSimulatorBN254>>;
 template class UltraRecursiveVerifier_<bb::MegaRecursiveFlavor_<CircuitSimulatorBN254>>;
 } // namespace bb::stdlib::recursion::honk