feat: Protogalaxy Decider and complete folding tests (AztecProtocol#3657

)

This PR introduces the Protogalaxy decider, now having a complete proof
of concept of the folding protocol for Ultra instances. To enable this,
sumcheck has been modified to work with the same power polynomial as in
protogalaxy paper and can be initialised with a custom `target_sum`.
Moreover, we now use different batching challenges for each subrelation
in sumcheck, essential to ensure a reasonable degree of the combiner
polynomial.

Additional: Minor bug fixed in the computation of \vec{\beta*}. 

Resolves AztecProtocol/barretenberg#772.
Resolves AztecProtocol/barretenberg#691.

.circleci/config.yml

@@ -257,17 +257,6 @@ jobs:
           command: |
             barretenberg/cpp/scripts/ci/upload_benchmarks_to_s3.sh
 
-  barretenberg-honk-tests:
-    docker:
-      - image: aztecprotocol/alpine-build-image
-    resource_class: small
-    steps:
-      - *checkout
-      - *setup_env
-      - run:
-          name: "Test"
-          command: cond_spot_run_test barretenberg-x86_64-linux-clang-assert 32 ./scripts/run_tests 1 honk_tests
-
   barretenberg-proof-system-tests:
     docker:
       - image: aztecprotocol/alpine-build-image
@@ -1135,7 +1124,6 @@ workflows:
             - barretenberg-x86_64-linux-clang
           <<: *defaults
       - barretenberg-proof-system-tests: *bb_test
-      - barretenberg-honk-tests: *bb_test
       - barretenberg-dsl-tests: *bb_test
       - barretenberg-tests: *bb_test
       - barretenberg-stdlib-tests: *bb_test

barretenberg/cpp/CMakePresets.json

@@ -351,8 +351,7 @@
         "acvm_backend.wasm",
         "barretenberg",
         "wasi",
-        "env",
-        "honk_tests"
+        "env"
       ]
     },
     {

barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp

@@ -7,9 +7,6 @@
  * simplify the codebase.
  */
 
-#include "barretenberg/ecc/curves/bn254/bn254.hpp"
-#include "barretenberg/ecc/curves/bn254/pairing.hpp"
-#include "barretenberg/ecc/curves/grumpkin/grumpkin.hpp"
 #include "barretenberg/ecc/scalar_multiplication/scalar_multiplication.hpp"
 #include "barretenberg/numeric/bitop/pow.hpp"
 #include "barretenberg/polynomials/polynomial.hpp"

barretenberg/cpp/src/barretenberg/common/thread_utils.hpp

@@ -1,3 +1,4 @@
+#pragma once
 #include "thread.hpp"
 
 namespace barretenberg::thread_utils {

barretenberg/cpp/src/barretenberg/eccvm/eccvm_prover.cpp

@@ -4,7 +4,6 @@
 #include "barretenberg/common/ref_array.hpp"
 #include "barretenberg/honk/proof_system/logderivative_library.hpp"
 #include "barretenberg/honk/proof_system/permutation_library.hpp"
-#include "barretenberg/honk/proof_system/power_polynomial.hpp"
 #include "barretenberg/polynomials/polynomial.hpp"
 #include "barretenberg/proof_system/library/grand_product_library.hpp"
 #include "barretenberg/relations/lookup_relation.hpp"
@@ -111,8 +110,12 @@ template <ECCVMFlavor Flavor> void ECCVMProver_<Flavor>::execute_relation_check_
     using Sumcheck = sumcheck::SumcheckProver<Flavor>;
 
     auto sumcheck = Sumcheck(key->circuit_size, transcript);
-    FF alpha = transcript->get_challenge("alpha");
-    sumcheck_output = sumcheck.prove(prover_polynomials, relation_parameters, alpha);
+    FF alpha = transcript->get_challenge("Sumcheck:alpha");
+    std::vector<FF> gate_challenges(numeric::get_msb(key->circuit_size));
+    for (size_t idx = 0; idx < gate_challenges.size(); idx++) {
+        gate_challenges[idx] = transcript->get_challenge("Sumcheck:gate_challenge_" + std::to_string(idx));
+    }
+    sumcheck_output = sumcheck.prove(prover_polynomials, relation_parameters, alpha, gate_challenges);
 }
 
 /**

barretenberg/cpp/src/barretenberg/eccvm/eccvm_transcript.test.cpp

@@ -126,10 +126,13 @@ template <typename Flavor> class ECCVMTranscriptTests : public ::testing::Test {
         round++;
         manifest_expected.add_entry(round, "LOOKUP_INVERSES", size_G);
         manifest_expected.add_entry(round, "Z_PERM", size_G);
-        manifest_expected.add_challenge(round, "alpha");
+        manifest_expected.add_challenge(round, "Sumcheck:alpha");
 
-        round++;
-        manifest_expected.add_challenge(round, "Sumcheck:zeta");
+        for (size_t i = 0; i < log_n; i++) {
+            round++;
+            std::string label = "Sumcheck:gate_challenge_" + std::to_string(i);
+            manifest_expected.add_challenge(round, label);
+        }
 
         for (size_t i = 0; i < log_n; ++i) {
             round++;

barretenberg/cpp/src/barretenberg/eccvm/eccvm_verifier.cpp

@@ -1,7 +1,6 @@
 #include "./eccvm_verifier.hpp"
 #include "barretenberg/commitment_schemes/gemini/gemini.hpp"
 #include "barretenberg/commitment_schemes/shplonk/shplonk.hpp"
-#include "barretenberg/honk/proof_system/power_polynomial.hpp"
 #include "barretenberg/numeric/bitop/get_msb.hpp"
 #include "barretenberg/transcript/transcript.hpp"
 
@@ -158,10 +157,16 @@ template <typename Flavor> bool ECCVMVerifier_<Flavor>::verify_proof(const plonk
     commitments.z_perm = receive_commitment(commitment_labels.z_perm);
 
     // Execute Sumcheck Verifier
-    auto sumcheck = SumcheckVerifier<Flavor>(circuit_size);
-    FF alpha = transcript->get_challenge("alpha");
+    const size_t log_circuit_size = numeric::get_msb(circuit_size);
+    auto sumcheck = SumcheckVerifier<Flavor>(log_circuit_size, transcript);
+    FF alpha = transcript->get_challenge("Sumcheck:alpha");
+    std::vector<FF> gate_challenges(numeric::get_msb(key->circuit_size));
+    for (size_t idx = 0; idx < gate_challenges.size(); idx++) {
+        gate_challenges[idx] = transcript->get_challenge("Sumcheck:gate_challenge_" + std::to_string(idx));
+    }
+
     auto [multivariate_challenge, purported_evaluations, sumcheck_verified] =
-        sumcheck.verify(relation_parameters, alpha, transcript);
+        sumcheck.verify(relation_parameters, alpha, gate_challenges);
 
     // If Sumcheck did not verify, return false
     if (sumcheck_verified.has_value() && !sumcheck_verified.value()) {

barretenberg/cpp/src/barretenberg/flavor/ecc_vm.hpp

@@ -45,6 +45,7 @@ template <typename CycleGroup_T, typename Curve_T, typename PCS_T> class ECCVMBa
     using CommitmentHandle = typename G1::affine_element;
     using CommitmentKey = pcs::CommitmentKey<Curve>;
     using VerifierCommitmentKey = pcs::VerifierCommitmentKey<Curve>;
+    using RelationSeparator = FF;
 
     static constexpr size_t NUM_WIRES = 74;
 

barretenberg/cpp/src/barretenberg/flavor/flavor.hpp

@@ -302,6 +302,10 @@ concept IsRecursiveFlavor = IsAnyOf<T, honk::flavor::UltraRecursive_<UltraCircui
 
 template <typename T> concept IsGrumpkinFlavor = IsAnyOf<T, honk::flavor::ECCVM>;
 
+template <typename T> concept IsFoldingFlavor = IsAnyOf<T, honk::flavor::Ultra, honk::flavor::GoblinUltra,honk::flavor::UltraRecursive_<UltraCircuitBuilder>, 
+                                       honk::flavor::UltraRecursive_<GoblinUltraCircuitBuilder>, 
+                                       honk::flavor::GoblinUltraRecursive>;
+
 template <typename T> concept UltraFlavor = IsAnyOf<T, honk::flavor::Ultra, honk::flavor::GoblinUltra>;
 
 template <typename T> concept ECCVMFlavor = IsAnyOf<T, honk::flavor::ECCVM>;

barretenberg/cpp/src/barretenberg/flavor/generated/AvmMini_flavor.hpp

@@ -17,8 +17,7 @@
 #include "barretenberg/relations/generated/AvmMini/mem_trace.hpp"
 #include "barretenberg/transcript/transcript.hpp"
 
-namespace proof_system::honk {
-namespace flavor {
+namespace proof_system::honk::flavor {
 
 class AvmMiniFlavor {
   public:
@@ -34,6 +33,7 @@ class AvmMiniFlavor {
     using CommitmentHandle = G1::affine_element;
     using CommitmentKey = pcs::CommitmentKey<Curve>;
     using VerifierCommitmentKey = pcs::VerifierCommitmentKey<Curve>;
+    using RelationSeparator = FF;
 
     static constexpr size_t NUM_PRECOMPUTED_ENTITIES = 2;
     static constexpr size_t NUM_WITNESS_ENTITIES = 38;
@@ -636,5 +636,4 @@ class AvmMiniFlavor {
     };
 };
 
-} // namespace flavor
-} // namespace proof_system::honk
+} // namespace proof_system::honk::flavor

barretenberg/cpp/src/barretenberg/flavor/generated/Toy_flavor.hpp

@@ -34,6 +34,7 @@ class ToyFlavor {
     using CommitmentHandle = G1::affine_element;
     using CommitmentKey = pcs::CommitmentKey<Curve>;
     using VerifierCommitmentKey = pcs::VerifierCommitmentKey<Curve>;
+    using RelationSeparator = FF;
 
     static constexpr size_t NUM_PRECOMPUTED_ENTITIES = 1;
     static constexpr size_t NUM_WITNESS_ENTITIES = 16;

barretenberg/cpp/src/barretenberg/flavor/goblin_translator.hpp

@@ -34,6 +34,7 @@ class GoblinTranslator {
     using BF = Curve::BaseField;
     using Polynomial = barretenberg::Polynomial<FF>;
     using PolynomialHandle = std::span<FF>;
+    using RelationSeparator = FF;
 
     // The size of the circuit which is filled with non-zero values for most polynomials. Most relations (everything
     // except for Permutation and GenPermSort) can be evaluated just on the first chunk

barretenberg/cpp/src/barretenberg/flavor/goblin_ultra.hpp

@@ -68,7 +68,6 @@ class GoblinUltra {
 
     static constexpr size_t MAX_PARTIAL_RELATION_LENGTH = compute_max_partial_relation_length<Relations>();
     static constexpr size_t MAX_TOTAL_RELATION_LENGTH = compute_max_total_relation_length<Relations>();
-    static constexpr size_t NUMBER_OF_SUBRELATIONS = compute_number_of_subrelations<Relations>();
 
     // BATCHED_RELATION_PARTIAL_LENGTH = algebraic degree of sumcheck relation *after* multiplying by the `pow_zeta`
     // random polynomial e.g. For \sum(x) [A(x) * B(x) + C(x)] * PowZeta(X), relation length = 2 and random relation
@@ -77,6 +76,12 @@ class GoblinUltra {
     static constexpr size_t BATCHED_RELATION_TOTAL_LENGTH = MAX_TOTAL_RELATION_LENGTH + 1;
     static constexpr size_t NUM_RELATIONS = std::tuple_size_v<Relations>;
 
+    // For instances of this flavour, used in folding, we need a unique sumcheck batching challenges for each
+    // subrelation. This
+    // is because using powers of alpha would increase the degree of Protogalaxy polynomial $G$ (the combiner) to much.
+    static constexpr size_t NUM_SUBRELATIONS = compute_number_of_subrelations<Relations>();
+    using RelationSeparator = std::array<FF, NUM_SUBRELATIONS - 1>;
+
     template <size_t NUM_INSTANCES>
     using ProtogalaxyTupleOfTuplesOfUnivariates =
         decltype(create_protogalaxy_tuple_of_tuples_of_univariates<Relations, NUM_INSTANCES>());
@@ -85,7 +90,6 @@ class GoblinUltra {
 
     // Whether or not the first row of the execution trace is reserved for 0s to enable shifts
     static constexpr bool has_zero_row = true;
-
     /**
      * @brief A base class labelling precomputed entities and (ordered) subsets of interest.
      * @details Used to build the proving key and verification key.
@@ -448,14 +452,58 @@ class GoblinUltra {
             this->lagrange_ecc_op = verification_key->lagrange_ecc_op;
             this->databus_id = verification_key->databus_id;
         }
+
+        VerifierCommitments_(const std::shared_ptr<VerificationKey>& verification_key,
+                             const WitnessCommitments& witness_commitments)
+        {
+            this->q_m = verification_key->q_m;
+            this->q_l = verification_key->q_l;
+            this->q_r = verification_key->q_r;
+            this->q_o = verification_key->q_o;
+            this->q_4 = verification_key->q_4;
+            this->q_c = verification_key->q_c;
+            this->q_arith = verification_key->q_arith;
+            this->q_sort = verification_key->q_sort;
+            this->q_elliptic = verification_key->q_elliptic;
+            this->q_aux = verification_key->q_aux;
+            this->q_lookup = verification_key->q_lookup;
+            this->q_busread = verification_key->q_busread;
+            this->q_poseidon2_external = verification_key->q_poseidon2_external;
+            this->q_poseidon2_internal = verification_key->q_poseidon2_internal;
+            this->sigma_1 = verification_key->sigma_1;
+            this->sigma_2 = verification_key->sigma_2;
+            this->sigma_3 = verification_key->sigma_3;
+            this->sigma_4 = verification_key->sigma_4;
+            this->id_1 = verification_key->id_1;
+            this->id_2 = verification_key->id_2;
+            this->id_3 = verification_key->id_3;
+            this->id_4 = verification_key->id_4;
+            this->table_1 = verification_key->table_1;
+            this->table_2 = verification_key->table_2;
+            this->table_3 = verification_key->table_3;
+            this->table_4 = verification_key->table_4;
+            this->lagrange_first = verification_key->lagrange_first;
+            this->lagrange_last = verification_key->lagrange_last;
+            this->lagrange_ecc_op = verification_key->lagrange_ecc_op;
+            this->databus_id = verification_key->databus_id;
+
+            this->w_l = witness_commitments.w_l;
+            this->w_r = witness_commitments.w_r;
+            this->w_o = witness_commitments.w_o;
+            this->sorted_accum = witness_commitments.sorted_accum;
+            this->w_4 = witness_commitments.w_4;
+            this->z_perm = witness_commitments.z_perm;
+            this->z_lookup = witness_commitments.z_lookup;
+            this->ecc_op_wire_1 = witness_commitments.ecc_op_wire_1;
+            this->ecc_op_wire_2 = witness_commitments.ecc_op_wire_2;
+            this->ecc_op_wire_3 = witness_commitments.ecc_op_wire_3;
+            this->calldata = witness_commitments.calldata;
+            this->calldata = witness_commitments.calldata_read_counts;
+            this->lookup_inverses = witness_commitments.lookup_inverses;
+        }
     };
     // Specialize for GoblinUltra (general case used in GoblinUltraRecursive).
     using VerifierCommitments = VerifierCommitments_<Commitment, VerificationKey>;
-    class FoldingParameters {
-      public:
-        std::vector<FF> gate_challenges;
-        FF target_sum;
-    };
 
     /**
      * @brief Derived class that defines proof structure for GoblinUltra proofs, as well as supporting functions.

barretenberg/cpp/src/barretenberg/flavor/goblin_ultra_recursive.hpp

@@ -74,6 +74,12 @@ class GoblinUltraRecursive {
     static constexpr size_t BATCHED_RELATION_PARTIAL_LENGTH = MAX_PARTIAL_RELATION_LENGTH + 1;
     static constexpr size_t NUM_RELATIONS = std::tuple_size<Relations>::value;
 
+    // For instances of this flavour, used in folding, we need a unique sumcheck batching challenge for each
+    // subrelation. This is because using powers of alpha would increase the degree of Protogalaxy polynomial $G$ (the
+    // combiner) to much.
+    static constexpr size_t NUM_SUBRELATIONS = compute_number_of_subrelations<Relations>();
+    using RelationSeparator = std::array<FF, NUM_SUBRELATIONS - 1>;
+
     // define the container for storing the univariate contribution from each relation in Sumcheck
     using SumcheckTupleOfTuplesOfUnivariates = decltype(create_sumcheck_tuple_of_tuples_of_univariates<Relations>());
     using TupleOfArraysOfValues = decltype(create_tuple_of_arrays_of_values<Relations>());

barretenberg/cpp/src/barretenberg/flavor/ultra.hpp

@@ -54,10 +54,14 @@ class Ultra {
                                  proof_system::AuxiliaryRelation<FF>>;
 
     static constexpr size_t MAX_PARTIAL_RELATION_LENGTH = compute_max_partial_relation_length<Relations>();
-    static constexpr size_t MAX_TOTAL_RELATION_LENGTH = compute_max_total_relation_length<Relations>();
     static_assert(MAX_PARTIAL_RELATION_LENGTH == 6);
+    static constexpr size_t MAX_TOTAL_RELATION_LENGTH = compute_max_total_relation_length<Relations>();
     static_assert(MAX_TOTAL_RELATION_LENGTH == 12);
-    static constexpr size_t NUMBER_OF_SUBRELATIONS = compute_number_of_subrelations<Relations>();
+    static constexpr size_t NUM_SUBRELATIONS = compute_number_of_subrelations<Relations>();
+    // For instances of this flavour, used in folding, we need a unique sumcheck batching challenge for each
+    // subrelation. This is because using powers of alpha would increase the degree of Protogalaxy polynomial $G$ (the
+    // combiner) too much.
+    using RelationSeparator = std::array<FF, NUM_SUBRELATIONS - 1>;
 
     // BATCHED_RELATION_PARTIAL_LENGTH = algebraic degree of sumcheck relation *after* multiplying by the `pow_zeta`
     // random polynomial e.g. For \sum(x) [A(x) * B(x) + C(x)] * PowZeta(X), relation length = 2 and random relation
@@ -75,6 +79,8 @@ class Ultra {
     // Whether or not the first row of the execution trace is reserved for 0s to enable shifts
     static constexpr bool has_zero_row = true;
 
+    static constexpr bool is_decider = true;
+
   private:
     /**
      * @brief A base class labelling precomputed entities and (ordered) subsets of interest.
@@ -400,6 +406,11 @@ class Ultra {
         };
     };
 
+    /**
+     * @brief A container encapsulating all the commitments that the verifier receives (to precomputed polynomials and
+     * witness polynomials).
+     *
+     */
     class VerifierCommitments : public AllEntities<Commitment> {
       public:
         VerifierCommitments(const std::shared_ptr<VerificationKey>& verification_key)
@@ -430,12 +441,44 @@ class Ultra {
             lagrange_first = verification_key->lagrange_first;
             lagrange_last = verification_key->lagrange_last;
         }
-    };
 
-    class FoldingParameters {
-      public:
-        std::vector<FF> gate_challenges;
-        FF target_sum;
+        VerifierCommitments(const std::shared_ptr<VerificationKey>& verification_key,
+                            const WitnessCommitments& witness_commitments)
+        {
+            q_m = verification_key->q_m;
+            q_c = verification_key->q_c;
+            q_l = verification_key->q_l;
+            q_r = verification_key->q_r;
+            q_o = verification_key->q_o;
+            q_4 = verification_key->q_4;
+            q_arith = verification_key->q_arith;
+            q_sort = verification_key->q_sort;
+            q_elliptic = verification_key->q_elliptic;
+            q_aux = verification_key->q_aux;
+            q_lookup = verification_key->q_lookup;
+            sigma_1 = verification_key->sigma_1;
+            sigma_2 = verification_key->sigma_2;
+            sigma_3 = verification_key->sigma_3;
+            sigma_4 = verification_key->sigma_4;
+            id_1 = verification_key->id_1;
+            id_2 = verification_key->id_2;
+            id_3 = verification_key->id_3;
+            id_4 = verification_key->id_4;
+            table_1 = verification_key->table_1;
+            table_2 = verification_key->table_2;
+            table_3 = verification_key->table_3;
+            table_4 = verification_key->table_4;
+            lagrange_first = verification_key->lagrange_first;
+            lagrange_last = verification_key->lagrange_last;
+
+            w_l = witness_commitments.w_l;
+            w_r = witness_commitments.w_r;
+            w_o = witness_commitments.w_o;
+            sorted_accum = witness_commitments.sorted_accum;
+            w_4 = witness_commitments.w_4;
+            z_perm = witness_commitments.z_perm;
+            z_lookup = witness_commitments.z_lookup;
+        }
     };
 
     /**

barretenberg/cpp/src/barretenberg/flavor/ultra_recursive.hpp

@@ -86,6 +86,12 @@ template <typename BuilderType> class UltraRecursive_ {
     static constexpr size_t BATCHED_RELATION_PARTIAL_LENGTH = MAX_PARTIAL_RELATION_LENGTH + 1;
     static constexpr size_t NUM_RELATIONS = std::tuple_size<Relations>::value;
 
+    // For instances of this flavour, used in folding, we need a unique sumcheck batching challenges for each
+    // subrelation to avoid increasing the degree of Protogalaxy polynomial $G$ (the
+    // combiner) too much.
+    static constexpr size_t NUM_SUBRELATIONS = compute_number_of_subrelations<Relations>();
+    using RelationSeparator = std::array<FF, NUM_SUBRELATIONS - 1>;
+
     // define the container for storing the univariate contribution from each relation in Sumcheck
     using SumcheckTupleOfTuplesOfUnivariates = decltype(create_sumcheck_tuple_of_tuples_of_univariates<Relations>());
     using TupleOfArraysOfValues = decltype(create_tuple_of_arrays_of_values<Relations>());