feat: conventional lookups using log-deriv

barretenberg/acir_tests/gen_inner_proof_inputs_ultra_honk.sh

@@ -3,7 +3,7 @@
 #   BIN: to specify a different binary to test with (e.g. bb.js or bb.js-dev).
 set -eu
 
-BIN=${BIN:-../cpp/build-debug/bin/bb}
+BIN=${BIN:-../cpp/build/bin/bb}
 CRS_PATH=~/.bb-crs
 BRANCH=master
 VERBOSE=${VERBOSE:-}

barretenberg/cpp/src/barretenberg/benchmark/relations_bench/relations.bench.cpp

@@ -65,7 +65,7 @@ BENCHMARK(execute_relation_for_pg_univariates<UltraFlavor, UltraArithmeticRelati
 BENCHMARK(execute_relation_for_pg_univariates<UltraFlavor, DeltaRangeConstraintRelation<Fr>>);
 BENCHMARK(execute_relation_for_pg_univariates<UltraFlavor, EllipticRelation<Fr>>);
 BENCHMARK(execute_relation_for_pg_univariates<UltraFlavor, AuxiliaryRelation<Fr>>);
-BENCHMARK(execute_relation_for_pg_univariates<UltraFlavor, LookupRelation<Fr>>);
+BENCHMARK(execute_relation_for_pg_univariates<UltraFlavor, LogDerivLookupRelation<Fr>>);
 BENCHMARK(execute_relation_for_pg_univariates<UltraFlavor, UltraPermutationRelation<Fr>>);
 
 // Goblin-Ultra only relations (PG prover combiner work)
@@ -79,7 +79,7 @@ BENCHMARK(execute_relation_for_univariates<UltraFlavor, UltraArithmeticRelation<
 BENCHMARK(execute_relation_for_univariates<UltraFlavor, DeltaRangeConstraintRelation<Fr>>);
 BENCHMARK(execute_relation_for_univariates<UltraFlavor, EllipticRelation<Fr>>);
 BENCHMARK(execute_relation_for_univariates<UltraFlavor, AuxiliaryRelation<Fr>>);
-BENCHMARK(execute_relation_for_univariates<UltraFlavor, LookupRelation<Fr>>);
+BENCHMARK(execute_relation_for_univariates<UltraFlavor, LogDerivLookupRelation<Fr>>);
 BENCHMARK(execute_relation_for_univariates<UltraFlavor, UltraPermutationRelation<Fr>>);
 
 // Goblin-Ultra only relations (Sumcheck prover work)
@@ -93,7 +93,7 @@ BENCHMARK(execute_relation_for_values<UltraFlavor, UltraArithmeticRelation<Fr>>)
 BENCHMARK(execute_relation_for_values<UltraFlavor, DeltaRangeConstraintRelation<Fr>>);
 BENCHMARK(execute_relation_for_values<UltraFlavor, EllipticRelation<Fr>>);
 BENCHMARK(execute_relation_for_values<UltraFlavor, AuxiliaryRelation<Fr>>);
-BENCHMARK(execute_relation_for_values<UltraFlavor, LookupRelation<Fr>>);
+BENCHMARK(execute_relation_for_values<UltraFlavor, LogDerivLookupRelation<Fr>>);
 BENCHMARK(execute_relation_for_values<UltraFlavor, UltraPermutationRelation<Fr>>);
 
 // Goblin-Ultra only relations (verifier work)

barretenberg/cpp/src/barretenberg/eccvm/eccvm_prover.cpp

@@ -6,7 +6,6 @@
 #include "barretenberg/honk/proof_system/permutation_library.hpp"
 #include "barretenberg/plonk_honk_shared/library/grand_product_library.hpp"
 #include "barretenberg/polynomials/polynomial.hpp"
-#include "barretenberg/relations/lookup_relation.hpp"
 #include "barretenberg/relations/permutation_relation.hpp"
 #include "barretenberg/sumcheck/sumcheck.hpp"
 

barretenberg/cpp/src/barretenberg/plonk/composer/composer_lib.hpp

@@ -48,4 +48,79 @@ std::shared_ptr<plonk::verification_key> compute_verification_key_common(
     // silencing for now but need to figure out where to extract type of VerifierCrs from :-/
     std::shared_ptr<bb::srs::factories::VerifierCrs<curve::BN254>> const& vrs);
 
+/**
+ * @brief Construct polynomials containing the sorted concatenation of the lookups and the lookup tables
+ *
+ * @tparam Flavor
+ * @param circuit
+ * @param dyadic_circuit_size
+ * @param additional_offset Additional space needed in polynomials to add randomness for zk (Plonk only)
+ * @return std::array<typename Flavor::Polynomial, 4>
+ */
+template <typename Flavor>
+std::array<typename Flavor::Polynomial, 4> construct_sorted_list_polynomials(typename Flavor::CircuitBuilder& circuit,
+                                                                             const size_t dyadic_circuit_size,
+                                                                             size_t additional_offset = 0)
+{
+    using Polynomial = typename Flavor::Polynomial;
+    std::array<Polynomial, 4> sorted_polynomials;
+    // Initialise the sorted concatenated list polynomials for the lookup argument
+    for (auto& s_i : sorted_polynomials) {
+        s_i = Polynomial(dyadic_circuit_size);
+    }
+
+    // The sorted list polynomials have (tables_size + lookups_size) populated entries. We define the index below so
+    // that these entries are written into the last indices of the polynomials. The values on the first
+    // dyadic_circuit_size - (tables_size + lookups_size) indices are automatically initialized to zero via the
+    // polynomial constructor.
+    size_t s_index = dyadic_circuit_size - (circuit.get_tables_size() + circuit.get_lookups_size()) - additional_offset;
+    ASSERT(s_index > 0); // We need at least 1 row of zeroes for the permutation argument
+
+    for (auto& table : circuit.lookup_tables) {
+        const fr table_index(table.table_index);
+        auto& lookup_gates = table.lookup_gates;
+        for (size_t i = 0; i < table.size(); ++i) {
+            if (table.use_twin_keys) {
+                lookup_gates.push_back({
+                    {
+                        table.column_1[i].from_montgomery_form().data[0],
+                        table.column_2[i].from_montgomery_form().data[0],
+                    },
+                    {
+                        table.column_3[i],
+                        0,
+                    },
+                });
+            } else {
+                lookup_gates.push_back({
+                    {
+                        table.column_1[i].from_montgomery_form().data[0],
+                        0,
+                    },
+                    {
+                        table.column_2[i],
+                        table.column_3[i],
+                    },
+                });
+            }
+        }
+
+#ifdef NO_TBB
+        std::sort(lookup_gates.begin(), lookup_gates.end());
+#else
+        std::sort(std::execution::par_unseq, lookup_gates.begin(), lookup_gates.end());
+#endif
+
+        for (const auto& entry : lookup_gates) {
+            const auto components = entry.to_table_components(table.use_twin_keys);
+            sorted_polynomials[0][s_index] = components[0];
+            sorted_polynomials[1][s_index] = components[1];
+            sorted_polynomials[2][s_index] = components[2];
+            sorted_polynomials[3][s_index] = table_index;
+            ++s_index;
+        }
+    }
+    return sorted_polynomials;
+}
+
 } // namespace bb::plonk

barretenberg/cpp/src/barretenberg/plonk_honk_shared/CMakeLists.txt

@@ -1 +1 @@
-barretenberg_module(plonk_honk_shared polynomials)
+barretenberg_module(plonk_honk_shared polynomials ultra_honk)

barretenberg/cpp/src/barretenberg/plonk_honk_shared/composer/composer_lib.hpp

@@ -2,6 +2,7 @@
 #include "barretenberg/common/ref_array.hpp"
 #include "barretenberg/flavor/flavor.hpp"
 #include "barretenberg/polynomials/polynomial_store.hpp"
+#include "barretenberg/stdlib_circuit_builders/plookup_tables/types.hpp"
 
 #include <memory>
 
@@ -20,6 +21,7 @@ void construct_lookup_table_polynomials(RefArray<typename Flavor::Polynomial, 4>
     //  ^^^^^^^^^  ^^^^^^^^  ^^^^^^^  ^nonzero to ensure uniqueness and to avoid infinity commitments
     //  |          table     randomness
     //  ignored, as used for regular constraints and padding to the next power of 2.
+    // TODO(https://github.com/AztecProtocol/barretenberg/issues/1033): construct tables and counts at top of trace
     ASSERT(dyadic_circuit_size > circuit.get_tables_size() + additional_offset);
     size_t offset = dyadic_circuit_size - circuit.get_tables_size() - additional_offset;
 
@@ -37,78 +39,40 @@ void construct_lookup_table_polynomials(RefArray<typename Flavor::Polynomial, 4>
 }
 
 /**
- * @brief Construct polynomials containing the sorted concatenation of the lookups and the lookup tables
- *
- * @tparam Flavor
- * @param circuit
- * @param dyadic_circuit_size
- * @param additional_offset Additional space needed in polynomials to add randomness for zk (Plonk only)
- * @return std::array<typename Flavor::Polynomial, 4>
+ * @brief Construct polynomial whose value at index i is the number of times the table entry at that index has been
+ * read.
+ * @details Read counts are needed for the log derivative lookup argument. The table polynomials are constructed as a
+ * concatenation of basic 3-column tables. Similarly, the read counts polynomial is constructed as the concatenation of
+ * read counts for the individual tables.
  */
 template <typename Flavor>
-std::array<typename Flavor::Polynomial, 4> construct_sorted_list_polynomials(typename Flavor::CircuitBuilder& circuit,
-                                                                             const size_t dyadic_circuit_size,
-                                                                             size_t additional_offset = 0)
+void construct_lookup_read_counts(typename Flavor::Polynomial& read_counts,
+                                  typename Flavor::Polynomial& read_tags,
+                                  typename Flavor::CircuitBuilder& circuit,
+                                  size_t dyadic_circuit_size)
 {
-    using Polynomial = typename Flavor::Polynomial;
-    std::array<Polynomial, 4> sorted_polynomials;
-    // Initialise the sorted concatenated list polynomials for the lookup argument
-    for (auto& s_i : sorted_polynomials) {
-        s_i = Polynomial(dyadic_circuit_size);
-    }
-
-    // The sorted list polynomials have (tables_size + lookups_size) populated entries. We define the index below so
-    // that these entries are written into the last indices of the polynomials. The values on the first
-    // dyadic_circuit_size - (tables_size + lookups_size) indices are automatically initialized to zero via the
-    // polynomial constructor.
-    size_t s_index = dyadic_circuit_size - (circuit.get_tables_size() + circuit.get_lookups_size()) - additional_offset;
-    ASSERT(s_index > 0); // We need at least 1 row of zeroes for the permutation argument
+    // TODO(https://github.com/AztecProtocol/barretenberg/issues/1033): construct tables and counts at top of trace
+    size_t offset = dyadic_circuit_size - circuit.get_tables_size();
 
+    size_t table_offset = offset; // offset of the present table in the table polynomials
+    // loop over all tables used in the circuit; each table contains data about the lookups made on it
     for (auto& table : circuit.lookup_tables) {
-        const fr table_index(table.table_index);
-        auto& lookup_gates = table.lookup_gates;
-        for (size_t i = 0; i < table.size(); ++i) {
-            if (table.use_twin_keys) {
-                lookup_gates.push_back({
-                    {
-                        table.column_1[i].from_montgomery_form().data[0],
-                        table.column_2[i].from_montgomery_form().data[0],
-                    },
-                    {
-                        table.column_3[i],
-                        0,
-                    },
-                });
-            } else {
-                lookup_gates.push_back({
-                    {
-                        table.column_1[i].from_montgomery_form().data[0],
-                        0,
-                    },
-                    {
-                        table.column_2[i],
-                        table.column_3[i],
-                    },
-                });
-            }
-        }
+        table.initialize_index_map();
+
+        for (auto& gate_data : table.lookup_gates) {
+            // convert lookup gate data to an array of three field elements, one for each of the 3 columns
+            auto table_entry = gate_data.to_table_components(table.use_twin_keys);
 
-#ifdef NO_TBB
-        std::sort(lookup_gates.begin(), lookup_gates.end());
-#else
-        std::sort(std::execution::par_unseq, lookup_gates.begin(), lookup_gates.end());
-#endif
+            // find the index of the entry in the table
+            auto index_in_table = table.index_map[table_entry];
 
-        for (const auto& entry : lookup_gates) {
-            const auto components = entry.to_sorted_list_components(table.use_twin_keys);
-            sorted_polynomials[0][s_index] = components[0];
-            sorted_polynomials[1][s_index] = components[1];
-            sorted_polynomials[2][s_index] = components[2];
-            sorted_polynomials[3][s_index] = table_index;
-            ++s_index;
+            // increment the read count at the corresponding index in the full polynomial
+            size_t index_in_poly = table_offset + index_in_table;
+            read_counts[index_in_poly]++;
+            read_tags[index_in_poly] = 1; // tag is 1 if entry has been read 1 or more times
         }
+        table_offset += table.size(); // set the offset of the next table within the polynomials
     }
-    return sorted_polynomials;
 }
 
 } // namespace bb

barretenberg/cpp/src/barretenberg/plonk_honk_shared/composer/composer_lib.test.cpp

@@ -1,21 +1,78 @@
 #include "barretenberg/plonk_honk_shared/composer/composer_lib.hpp"
-#include "barretenberg/common/slab_allocator.hpp"
-#include "barretenberg/plonk_honk_shared/types/circuit_type.hpp"
 #include "barretenberg/srs/factories/crs_factory.hpp"
+#include "barretenberg/stdlib_circuit_builders/ultra_circuit_builder.hpp"
 #include "barretenberg/stdlib_circuit_builders/ultra_flavor.hpp"
+
 #include <array>
 #include <gtest/gtest.h>
 
 using namespace bb;
 
 class ComposerLibTests : public ::testing::Test {
+  public:
+    using Flavor = UltraFlavor;
+    using FF = typename Flavor::FF;
+
   protected:
+    static void SetUpTestSuite() { bb::srs::init_crs_factory("../srs_db/ignition"); }
+};
+
+/**
+ * @brief A test to demonstrate that lookup read counts/tags are computed correctly for a simple 'hand-computable' case
+ * using the uint32 XOR table
+ *
+ */
+TEST_F(ComposerLibTests, LookupReadCounts)
+{
+    using Builder = UltraCircuitBuilder;
     using Flavor = UltraFlavor;
     using FF = typename Flavor::FF;
-    Flavor::CircuitBuilder circuit_constructor;
-    Flavor::ProvingKey proving_key = []() {
-        auto crs_factory = srs::factories::CrsFactory<bb::curve::BN254>();
-        auto crs = crs_factory.get_prover_crs(4);
-        return Flavor::ProvingKey(/*circuit_size=*/8, /*num_public_inputs=*/0);
-    }();
-};
+    using Polynomial = typename Flavor::Polynomial;
+    auto UINT32_XOR = plookup::MultiTableId::UINT32_XOR;
+
+    Builder builder;
+
+    // define some very simply inputs to XOR
+    FF left{ 1 };
+    FF right{ 5 };
+
+    auto left_idx = builder.add_variable(left);
+    auto right_idx = builder.add_variable(right);
+
+    // create a single lookup from the uint32 XOR table
+    auto accumulators = plookup::get_lookup_accumulators(UINT32_XOR, left, right, /*is_2_to_1_lookup*/ true);
+    builder.create_gates_from_plookup_accumulators(UINT32_XOR, accumulators, left_idx, right_idx);
+
+    EXPECT_EQ(builder.lookup_tables.size(), 1);       // we only used a single table
+    EXPECT_EQ(builder.lookup_tables[0].size(), 4096); // table has size 64*64 (6 bit operands)
+
+    size_t circuit_size = 8192;
+
+    Polynomial read_counts{ circuit_size };
+    Polynomial read_tags{ circuit_size };
+
+    construct_lookup_read_counts<Flavor>(read_counts, read_tags, builder, circuit_size);
+
+    // The table polys are constructed at the bottom of the trace, thus so to are the counts/tags
+    // TODO(https://github.com/AztecProtocol/barretenberg/issues/1033): construct tables and counts at top of trace
+    size_t offset = circuit_size - builder.get_tables_size();
+
+    // The uint32 XOR lookup table is constructed for 6 bit operands via double for loop that iterates through the left
+    // operand externally (0 to 63) then the right operand internally (0 to 63). Computing (1 XOR 5) will thus result in
+    // 1 lookup from the (1*64 + 5)th index in the table and 5 lookups from the (0*64 + 0)th index (for the remaining 5
+    // limbs that are all 0). The counts and tags at all other indices should be zero.
+    size_t idx = 0;
+    for (auto [count, tag] : zip_view(read_counts, read_tags)) {
+        if (idx == (0 + offset)) {
+            EXPECT_EQ(count, 5);
+            EXPECT_EQ(tag, 1);
+        } else if (idx == (69 + offset)) {
+            EXPECT_EQ(count, 1);
+            EXPECT_EQ(tag, 1);
+        } else {
+            EXPECT_EQ(count, 0);
+            EXPECT_EQ(tag, 0);
+        }
+        idx++;
+    }
+}