feat: complete folding prover and verifier for ultra instances (Aztec…

…Protocol#3419)

Closes AztecProtocol/barretenberg#781 
Closes AztecProtocol/barretenberg#689 
Closes AztecProtocol/barretenberg#690
Closes AztecProtocol/barretenberg#802

This PR provides the first full version of folding prover and verifiers
for Ultra instances and provides the first full folding test. The
protogalaxy interfaces have been slightly reworked so the prover now
returns a complete new relaxed `Instance` and a folding proof. While the
verifier receives the folding proof and returns a boolean dependent on
whether the pubic data from the relaxed instance, sent by the prover, is
the same as the data computed by the folding verifier.

barretenberg/cpp/src/barretenberg/flavor/goblin_ultra.hpp

@@ -194,11 +194,6 @@ class GoblinUltra {
         {
             return { this->ecc_op_wire_1, this->ecc_op_wire_2, this->ecc_op_wire_3, this->ecc_op_wire_4 };
         }
-        // The sorted concatenations of table and witness data needed for plookup.
-        RefVector<DataType> get_sorted_polynomials()
-        {
-            return { this->sorted_1, this->sorted_2, this->sorted_3, this->sorted_4 };
-        };
     };
 
     template <typename DataType> class ShiftedEntities {
@@ -290,11 +285,60 @@ class GoblinUltra {
                      this->calldata_read_counts,
                      this->lookup_inverses };
         };
+
+        RefVector<DataType> get_witness()
+        {
+            return { this->w_l,
+                     this->w_r,
+                     this->w_o,
+                     this->w_4,
+                     this->sorted_accum,
+                     this->z_perm,
+                     this->z_lookup,
+                     this->ecc_op_wire_1,
+                     this->ecc_op_wire_2,
+                     this->ecc_op_wire_3,
+                     this->ecc_op_wire_4,
+                     this->calldata,
+                     this->calldata_read_counts,
+                     this->lookup_inverses };
+        };
         RefVector<DataType> get_to_be_shifted()
         {
             return { this->table_1, this->table_2, this->table_3,      this->table_4, this->w_l,     this->w_r,
                      this->w_o,     this->w_4,     this->sorted_accum, this->z_perm,  this->z_lookup };
         };
+        RefVector<DataType> get_precomputed()
+        {
+            return { this->q_m,
+                     this->q_c,
+                     this->q_l,
+                     this->q_r,
+                     this->q_o,
+                     this->q_4,
+                     this->q_arith,
+                     this->q_sort,
+                     this->q_elliptic,
+                     this->q_aux,
+                     this->q_lookup,
+                     this->q_busread,
+                     this->sigma_1,
+                     this->sigma_2,
+                     this->sigma_3,
+                     this->sigma_4,
+                     this->id_1,
+                     this->id_2,
+                     this->id_3,
+                     this->id_4,
+                     this->table_1,
+                     this->table_2,
+                     this->table_3,
+                     this->table_4,
+                     this->lagrange_first,
+                     this->lagrange_last,
+                     this->lagrange_ecc_op,
+                     this->databus_id };
+        }
         RefVector<DataType> get_shifted() { return ShiftedEntities<DataType>::get_all(); };
     };
 
@@ -381,6 +425,29 @@ class GoblinUltra {
         }
     };
 
+    /**
+     * @brief An owning container of polynomials.
+     * @warning When this was introduced it broke some of our design principles.
+     *   - Execution trace builders don't handle "polynomials" because the interpretation of the execution trace
+     * columns as polynomials is a detail of the proving system, and trace builders are (sometimes in practice,
+     * always in principle) reusable for different proving protocols (e.g., Plonk and Honk).
+     *   - Polynomial storage is handled by key classes. Polynomials aren't moved, but are accessed elsewhere by
+     * std::spans.
+     *
+     *  We will consider revising this data model: TODO(https://github.com/AztecProtocol/barretenberg/issues/743)
+     */
+    class AllPolynomials : public AllEntities<Polynomial> {
+      public:
+        [[nodiscard]] AllValues get_row(const size_t row_idx) const
+        {
+            AllValues result;
+            for (auto [result_field, polynomial] : zip_view(result.get_all(), this->get_all())) {
+                result_field = polynomial[row_idx];
+            }
+            return result;
+        }
+    };
+
     /**
      * @brief A container for the witness commitments.
      */
@@ -488,7 +555,7 @@ class GoblinUltra {
     using VerifierCommitments = VerifierCommitments_<Commitment, VerificationKey>;
     class FoldingParameters {
       public:
-        std::vector<FF> gate_separation_challenges;
+        std::vector<FF> gate_challenges;
         FF target_sum;
     };
 

barretenberg/cpp/src/barretenberg/flavor/ultra.hpp

@@ -207,6 +207,18 @@ class Ultra {
 
             };
         };
+
+        RefVector<DataType> get_precomputed()
+        {
+            return { q_m,          q_c,   q_l,      q_r,     q_o,     q_4,     q_arith, q_sort,
+                     q_elliptic,   q_aux, q_lookup, sigma_1, sigma_2, sigma_3, sigma_4, id_1,
+                     id_2,         id_3,  id_4,     table_1, table_2, table_3, table_4, lagrange_first,
+                     lagrange_last
+
+            };
+        }
+
+        RefVector<DataType> get_witness() { return { w_l, w_r, w_o, w_4, sorted_accum, z_perm, z_lookup }; };
         RefVector<DataType> get_to_be_shifted()
         {
             return { table_1, table_2, table_3, table_4, w_l, w_r, w_o, w_4, sorted_accum, z_perm, z_lookup };
@@ -248,8 +260,8 @@ class Ultra {
     using VerificationKey = VerificationKey_<PrecomputedEntities<Commitment>>;
 
     /**
-     * @brief A field element for each entity of the flavor. These entities represent the prover polynomials evaluated
-     * at one point.
+     * @brief A field element for each entity of the flavor. These entities represent the prover polynomials
+     * evaluated at one point.
      */
     class AllValues : public AllEntities<FF> {
       public:
@@ -273,6 +285,29 @@ class Ultra {
         }
     };
 
+    /**
+     * @brief An owning container of polynomials.
+     * @warning When this was introduced it broke some of our design principles.
+     *   - Execution trace builders don't handle "polynomials" because the interpretation of the execution trace
+     * columns as polynomials is a detail of the proving system, and trace builders are (sometimes in practice,
+     * always in principle) reusable for different proving protocols (e.g., Plonk and Honk).
+     *   - Polynomial storage is handled by key classes. Polynomials aren't moved, but are accessed elsewhere by
+     * std::spans.
+     *
+     *  We will consider revising this data model: TODO(https://github.com/AztecProtocol/barretenberg/issues/743)
+     */
+    class AllPolynomials : public AllEntities<Polynomial> {
+      public:
+        [[nodiscard]] AllValues get_row(const size_t row_idx) const
+        {
+            AllValues result;
+            for (auto [result_field, polynomial] : zip_view(result.get_all(), this->get_all())) {
+                result_field = polynomial[row_idx];
+            }
+            return result;
+        }
+    };
+
     /**
      * @brief A container for storing the partially evaluated multivariates produced by sumcheck.
      */
@@ -323,32 +358,31 @@ class Ultra {
             z_lookup = "Z_LOOKUP";
             sorted_accum = "SORTED_ACCUM";
 
-            // The ones beginning with "__" are only used for debugging
-            q_c = "__Q_C";
-            q_l = "__Q_L";
-            q_r = "__Q_R";
-            q_o = "__Q_O";
-            q_4 = "__Q_4";
-            q_m = "__Q_M";
-            q_arith = "__Q_ARITH";
-            q_sort = "__Q_SORT";
-            q_elliptic = "__Q_ELLIPTIC";
-            q_aux = "__Q_AUX";
-            q_lookup = "__Q_LOOKUP";
-            sigma_1 = "__SIGMA_1";
-            sigma_2 = "__SIGMA_2";
-            sigma_3 = "__SIGMA_3";
-            sigma_4 = "__SIGMA_4";
-            id_1 = "__ID_1";
-            id_2 = "__ID_2";
-            id_3 = "__ID_3";
-            id_4 = "__ID_4";
-            table_1 = "__TABLE_1";
-            table_2 = "__TABLE_2";
-            table_3 = "__TABLE_3";
-            table_4 = "__TABLE_4";
-            lagrange_first = "__LAGRANGE_FIRST";
-            lagrange_last = "__LAGRANGE_LAST";
+            q_c = "Q_C";
+            q_l = "Q_L";
+            q_r = "Q_R";
+            q_o = "Q_O";
+            q_4 = "Q_4";
+            q_m = "Q_M";
+            q_arith = "Q_ARITH";
+            q_sort = "Q_SORT";
+            q_elliptic = "Q_ELLIPTIC";
+            q_aux = "Q_AUX";
+            q_lookup = "Q_LOOKUP";
+            sigma_1 = "SIGMA_1";
+            sigma_2 = "SIGMA_2";
+            sigma_3 = "SIGMA_3";
+            sigma_4 = "SIGMA_4";
+            id_1 = "ID_1";
+            id_2 = "ID_2";
+            id_3 = "ID_3";
+            id_4 = "ID_4";
+            table_1 = "TABLE_1";
+            table_2 = "TABLE_2";
+            table_3 = "TABLE_3";
+            table_4 = "TABLE_4";
+            lagrange_first = "LAGRANGE_FIRST";
+            lagrange_last = "LAGRANGE_LAST";
         };
     };
 
@@ -357,11 +391,11 @@ class Ultra {
         VerifierCommitments(const std::shared_ptr<VerificationKey>& verification_key)
         {
             q_m = verification_key->q_m;
+            q_c = verification_key->q_c;
             q_l = verification_key->q_l;
             q_r = verification_key->q_r;
             q_o = verification_key->q_o;
             q_4 = verification_key->q_4;
-            q_c = verification_key->q_c;
             q_arith = verification_key->q_arith;
             q_sort = verification_key->q_sort;
             q_elliptic = verification_key->q_elliptic;
@@ -386,7 +420,7 @@ class Ultra {
 
     class FoldingParameters {
       public:
-        std::vector<FF> gate_separation_challenges;
+        std::vector<FF> gate_challenges;
         FF target_sum;
     };
 

barretenberg/cpp/src/barretenberg/honk/utils/testing.hpp

@@ -9,52 +9,50 @@ namespace proof_system::honk {
  * function returns an array of data pointed to by the ProverPolynomials.
  */
 template <typename Flavor>
-std::pair<std::array<barretenberg::Polynomial<typename Flavor::FF>, Flavor::NUM_ALL_ENTITIES>,
-          typename Flavor::ProverPolynomials>
-get_sequential_prover_polynomials(const size_t log_circuit_size, const size_t starting_value)
+std::pair<typename Flavor::AllPolynomials, typename Flavor::ProverPolynomials> get_sequential_prover_polynomials(
+    const size_t log_circuit_size, const size_t starting_value)
 {
     using FF = typename Flavor::FF;
     using ProverPolynomials = typename Flavor::ProverPolynomials;
     using Polynomial = typename Flavor::Polynomial;
 
-    std::array<barretenberg::Polynomial<typename Flavor::FF>, Flavor::NUM_ALL_ENTITIES> storage;
+    typename Flavor::AllPolynomials storage;
     size_t circuit_size = 1 << log_circuit_size;
     size_t value_idx = starting_value;
-    for (auto& polynomial : storage) {
+    for (auto& polynomial : storage.get_all()) {
         polynomial = Polynomial(circuit_size);
         for (auto& value : polynomial) {
             value = FF(value_idx++);
         }
     }
 
     ProverPolynomials prover_polynomials;
-    for (auto [prover_poly, storage_poly] : zip_view(prover_polynomials.get_all(), storage)) {
+    for (auto [prover_poly, storage_poly] : zip_view(prover_polynomials.get_all(), storage.get_all())) {
         prover_poly = storage_poly;
     }
 
     return std::pair(std::move(storage), prover_polynomials);
 }
 
 template <typename Flavor>
-std::pair<std::array<barretenberg::Polynomial<typename Flavor::FF>, Flavor::NUM_ALL_ENTITIES>,
-          typename Flavor::ProverPolynomials>
-get_zero_prover_polynomials(const size_t log_circuit_size)
+std::pair<typename Flavor::AllPolynomials, typename Flavor::ProverPolynomials> get_zero_prover_polynomials(
+    const size_t log_circuit_size)
 {
     using FF = typename Flavor::FF;
     using ProverPolynomials = typename Flavor::ProverPolynomials;
     using Polynomial = typename Flavor::Polynomial;
 
-    std::array<barretenberg::Polynomial<typename Flavor::FF>, Flavor::NUM_ALL_ENTITIES> storage;
+    typename Flavor::AllPolynomials storage;
     size_t circuit_size = 1 << log_circuit_size;
-    for (auto& polynomial : storage) {
+    for (auto& polynomial : storage.get_all()) {
         polynomial = Polynomial(circuit_size);
         for (auto& value : polynomial) {
             value = FF(0);
         }
     }
 
     ProverPolynomials prover_polynomials;
-    for (auto [prover_poly, storage_poly] : zip_view(prover_polynomials.get_all(), storage)) {
+    for (auto [prover_poly, storage_poly] : zip_view(prover_polynomials.get_all(), storage.get_all())) {
         prover_poly = storage_poly;
     }
 

barretenberg/cpp/src/barretenberg/honk/utils/testing.test.cpp

@@ -10,8 +10,8 @@ TEST(HonkTestingUtils, ProverPolynomials)
     auto [storage, prover_polynomials] =
         proof_system::honk::get_sequential_prover_polynomials<Flavor>(/*log_circuit_size=*/2, /*starting_value=*/0);
     auto& first_polynomial = prover_polynomials.get_all()[0];
-    EXPECT_EQ(storage[0][0], first_polynomial[0]);
-    EXPECT_EQ(storage[0][1], first_polynomial[1]);
+    EXPECT_EQ(storage.get_all()[0][0], first_polynomial[0]);
+    EXPECT_EQ(storage.get_all()[0][1], first_polynomial[1]);
 };
 
 } // namespace barretenberg::test_testing_utils

barretenberg/cpp/src/barretenberg/protogalaxy/combiner.test.cpp

@@ -33,9 +33,11 @@ TEST(Protogalaxy, CombinerOn2Instances)
     };
 
     auto run_test = [&](bool is_random_input) {
+        // Combiner test on prover polynomisls containing random values, restricted to only the standard arithmetic
+        // relation.
         if (is_random_input) {
             std::vector<std::shared_ptr<ProverInstance>> instance_data(NUM_INSTANCES);
-            std::array<std::array<Polynomial, Flavor::NUM_ALL_ENTITIES>, NUM_INSTANCES> storage_arrays;
+            std::array<typename Flavor::AllPolynomials, NUM_INSTANCES> storage_arrays;
             ProtoGalaxyProver prover;
             std::vector<FF> pow_betas = { FF(1), FF(2) };
 
@@ -46,6 +48,7 @@ TEST(Protogalaxy, CombinerOn2Instances)
                 restrict_to_standard_arithmetic_relation(prover_polynomials);
                 storage_arrays[idx] = std::move(storage);
                 instance->prover_polynomials = prover_polynomials;
+                instance->instance_size = 2;
                 instance_data[idx] = instance;
             }
 
@@ -70,7 +73,7 @@ TEST(Protogalaxy, CombinerOn2Instances)
             EXPECT_EQ(result, expected_result);
         } else {
             std::vector<std::shared_ptr<ProverInstance>> instance_data(NUM_INSTANCES);
-            std::array<std::array<Polynomial, Flavor::NUM_ALL_ENTITIES>, NUM_INSTANCES> storage_arrays;
+            std::array<typename Flavor::AllPolynomials, NUM_INSTANCES> storage_arrays;
             ProtoGalaxyProver prover;
             std::vector<FF> pow_betas = { FF(1), FF(2) };
 
@@ -81,6 +84,7 @@ TEST(Protogalaxy, CombinerOn2Instances)
                 restrict_to_standard_arithmetic_relation(prover_polynomials);
                 storage_arrays[idx] = std::move(storage);
                 instance->prover_polynomials = prover_polynomials;
+                instance->instance_size = 2;
                 instance_data[idx] = instance;
             }
 
@@ -162,7 +166,7 @@ TEST(Protogalaxy, CombinerOn4Instances)
 
     auto run_test = [&]() {
         std::vector<std::shared_ptr<ProverInstance>> instance_data(NUM_INSTANCES);
-        std::array<std::array<Polynomial, Flavor::NUM_ALL_ENTITIES>, NUM_INSTANCES> storage_arrays;
+        std::array<typename Flavor::AllPolynomials, NUM_INSTANCES> storage_arrays;
         ProtoGalaxyProver prover;
         std::vector<FF> pow_betas = { FF(1), FF(2) };
 
@@ -172,6 +176,7 @@ TEST(Protogalaxy, CombinerOn4Instances)
                 /*log_circuit_size=*/1);
             storage_arrays[idx] = std::move(storage);
             instance->prover_polynomials = prover_polynomials;
+            instance->instance_size = 2;
             instance_data[idx] = instance;
         }
 

barretenberg/cpp/src/barretenberg/protogalaxy/combiner_example_gen.py

@@ -103,7 +103,6 @@ def compute_first_example():
                              row.q_l, row.q_r, row.q_o, row.q_c)
         accumulator += zeta_pow * relation_value
         zeta_pow *= zeta
-
     return accumulator