feat: Poseidon2 stdlib impl (AztecProtocol#3551)

Poseidon2 permutation and sponge function stdlib implementation that
follows native crypto/ implementation.
Adds hash_buffer function to native and stdlib poseidon2 implementations.
Updates CI tests with poseidon2 tests, stdlib_pedersen_hash tests.
Adds poseidon2 end gate.

Resolves AztecProtocol/barretenberg#776

barretenberg/cpp/scripts/bb-tests.sh

@@ -18,6 +18,7 @@ TESTS=(
   crypto_ecdsa_tests
   crypto_pedersen_commitment_tests
   crypto_pedersen_hash_tests
+  crypto_poseidon2_tests
   crypto_schnorr_tests
   crypto_sha256_tests
   dsl_tests

barretenberg/cpp/scripts/stdlib-tests

@@ -5,5 +5,7 @@ stdlib_blake3s_tests
 stdlib_ecdsa_tests
 stdlib_merkle_tree_tests
 stdlib_pedersen_commitment_tests
+stdlib_pedersen_hash_tests
+stdlib_poseidon2_tests
 stdlib_schnorr_tests
 stdlib_sha256_tests

barretenberg/cpp/src/barretenberg/crypto/pedersen_hash/pedersen.cpp

@@ -30,16 +30,14 @@ std::vector<typename Curve::BaseField> pedersen_hash_base<Curve>::convert_buffer
     };
 
     std::vector<Fq> elements;
-    for (size_t i = 0; i < num_elements; ++i) {
-        size_t bytes_to_slice = 0;
-        if (i == num_elements - 1) {
-            bytes_to_slice = num_bytes - (i * bytes_per_element);
-        } else {
-            bytes_to_slice = bytes_per_element;
-        }
+    for (size_t i = 0; i < num_elements - 1; ++i) {
+        size_t bytes_to_slice = bytes_per_element;
         Fq element = slice(input, i * bytes_per_element, bytes_to_slice);
         elements.emplace_back(element);
     }
+    size_t bytes_to_slice = num_bytes - ((num_elements - 1) * bytes_per_element);
+    Fq element = slice(input, (num_elements - 1) * bytes_per_element, bytes_to_slice);
+    elements.emplace_back(element);
     return elements;
 }
 

barretenberg/cpp/src/barretenberg/crypto/poseidon2/poseidon2.bench.cpp

@@ -10,9 +10,8 @@ grumpkin::fq poseidon_function(const size_t count)
     for (size_t i = 0; i < count; ++i) {
         inputs[i] = grumpkin::fq::random_element();
     }
-    std::span tmp(inputs);
     // hash count many field elements
-    inputs[0] = crypto::Poseidon2<crypto::Poseidon2Bn254ScalarFieldParams>::hash(tmp);
+    inputs[0] = crypto::Poseidon2<crypto::Poseidon2Bn254ScalarFieldParams>::hash(inputs);
     return inputs[0];
 }
 

barretenberg/cpp/src/barretenberg/crypto/poseidon2/poseidon2.cpp

@@ -0,0 +1,48 @@
+#include "poseidon2.hpp"
+
+namespace crypto {
+/**
+ * @brief Hashes a vector of field elements
+ */
+template <typename Params>
+typename Poseidon2<Params>::FF Poseidon2<Params>::hash(const std::vector<typename Poseidon2<Params>::FF>& input)
+{
+    auto input_span = input;
+    return Sponge::hash_fixed_length(input_span);
+}
+
+/**
+ * @brief Hashes vector of bytes by chunking it into 31 byte field elements and calling hash()
+ * @details Slice function cuts out the required number of bytes from the byte vector
+ */
+template <typename Params>
+typename Poseidon2<Params>::FF Poseidon2<Params>::hash_buffer(const std::vector<uint8_t>& input)
+{
+    const size_t num_bytes = input.size();
+    const size_t bytes_per_element = 31;
+    size_t num_elements = static_cast<size_t>(num_bytes % bytes_per_element != 0) + (num_bytes / bytes_per_element);
+
+    const auto slice = [](const std::vector<uint8_t>& data, const size_t start, const size_t slice_size) {
+        uint256_t result(0);
+        for (size_t i = 0; i < slice_size; ++i) {
+            result = (result << uint256_t(8));
+            result += uint256_t(data[i + start]);
+        }
+        return FF(result);
+    };
+
+    std::vector<FF> converted;
+    for (size_t i = 0; i < num_elements - 1; ++i) {
+        size_t bytes_to_slice = bytes_per_element;
+        FF element = slice(input, i * bytes_per_element, bytes_to_slice);
+        converted.emplace_back(element);
+    }
+    size_t bytes_to_slice = num_bytes - ((num_elements - 1) * bytes_per_element);
+    FF element = slice(input, (num_elements - 1) * bytes_per_element, bytes_to_slice);
+    converted.emplace_back(element);
+
+    return hash(converted);
+}
+
+template class Poseidon2<Poseidon2Bn254ScalarFieldParams>;
+} // namespace crypto

barretenberg/cpp/src/barretenberg/crypto/poseidon2/poseidon2.hpp

@@ -10,7 +10,19 @@ template <typename Params> class Poseidon2 {
   public:
     using FF = typename Params::FF;
 
+    // We choose our rate to be t-1 and capacity to be 1.
     using Sponge = FieldSponge<FF, Params::t - 1, 1, Params::t, Poseidon2Permutation<Params>>;
-    static FF hash(std::span<FF> input) { return Sponge::hash_fixed_length(input); }
+
+    /**
+     * @brief Hashes a vector of field elements
+     */
+    static FF hash(const std::vector<FF>& input);
+    /**
+     * @brief Hashes vector of bytes by chunking it into 31 byte field elements and calling hash()
+     * @details Slice function cuts out the required number of bytes from the byte vector
+     */
+    static FF hash_buffer(const std::vector<uint8_t>& input);
 };
+
+extern template class Poseidon2<Poseidon2Bn254ScalarFieldParams>;
 } // namespace crypto

barretenberg/cpp/src/barretenberg/crypto/poseidon2/poseidon2.test.cpp

@@ -10,7 +10,7 @@ auto& engine = numeric::random::get_debug_engine();
 }
 
 namespace poseidon2_tests {
-TEST(Poseidon2, BasicTests)
+TEST(Poseidon2, HashBasicTests)
 {
 
     barretenberg::fr a = barretenberg::fr::random_element(&engine);
@@ -32,17 +32,33 @@ TEST(Poseidon2, BasicTests)
 // N.B. these hardcoded values were extracted from the algorithm being tested. These are NOT independent test vectors!
 // TODO(@zac-williamson #3132): find independent test vectors we can compare against! (very hard to find given
 // flexibility of Poseidon's parametrisation)
-TEST(Poseidon2, ConsistencyCheck)
+TEST(Poseidon2, HashConsistencyCheck)
 {
     barretenberg::fr a(std::string("9a807b615c4d3e2fa0b1c2d3e4f56789fedcba9876543210abcdef0123456789"));
     barretenberg::fr b(std::string("9a807b615c4d3e2fa0b1c2d3e4f56789fedcba9876543210abcdef0123456789"));
     barretenberg::fr c(std::string("0x9a807b615c4d3e2fa0b1c2d3e4f56789fedcba9876543210abcdef0123456789"));
     barretenberg::fr d(std::string("0x9a807b615c4d3e2fa0b1c2d3e4f56789fedcba9876543210abcdef0123456789"));
 
-    std::array<barretenberg::fr, 4> input{ a, b, c, d };
+    std::vector<barretenberg::fr> input{ a, b, c, d };
     auto result = crypto::Poseidon2<crypto::Poseidon2Bn254ScalarFieldParams>::hash(input);
 
-    barretenberg::fr expected(std::string("0x150c19ae11b3290c137c7a4d760d9482a6581d731535f560c3601d6a766b0937"));
+    barretenberg::fr expected(std::string("0x2f43a0f83b51a6f5fc839dea0ecec74947637802a579fa9841930a25a0bcec11"));
+
+    EXPECT_EQ(result, expected);
+}
+
+TEST(Poseidon2, HashBufferConsistencyCheck)
+{
+    // 31 byte inputs because hash_buffer slicing is only injective with 31 bytes, as it slices 31 bytes for each field
+    // element
+    barretenberg::fr a(std::string("00000b615c4d3e2fa0b1c2d3e4f56789fedcba9876543210abcdef0123456789"));
+
+    auto input_vec = to_buffer(a);      // takes field element and converts it to 32 bytes
+    input_vec.erase(input_vec.begin()); // erase first byte since we want 31 bytes
+    std::vector<barretenberg::fr> input{ a };
+    auto expected = crypto::Poseidon2<crypto::Poseidon2Bn254ScalarFieldParams>::hash(input);
+
+    barretenberg::fr result = crypto::Poseidon2<crypto::Poseidon2Bn254ScalarFieldParams>::hash_buffer(input_vec);
 
     EXPECT_EQ(result, expected);
 }

barretenberg/cpp/src/barretenberg/crypto/poseidon2/poseidon2_permutation.hpp

@@ -7,7 +7,6 @@
 #include <array>
 #include <cstddef>
 #include <cstdint>
-#include <vector>
 
 namespace crypto {
 
@@ -123,6 +122,13 @@ template <typename Params> class Poseidon2Permutation {
         }
     }
 
+    /**
+     * @brief Native form of Poseidon2 permutation from https://eprint.iacr.org/2023/323.
+     * @details The permutation consists of one initial linear layer, then a set of external rounds, a set of internal
+     * rounds, and a set of external rounds.
+     * @param input
+     * @return constexpr State
+     */
     static constexpr State permutation(const State& input)
     {
         // deep copy
@@ -131,20 +137,23 @@ template <typename Params> class Poseidon2Permutation {
         // Apply 1st linear layer
         matrix_multiplication_external(current_state);
 
+        // First set of external rounds
         constexpr size_t rounds_f_beginning = rounds_f / 2;
         for (size_t i = 0; i < rounds_f_beginning; ++i) {
             add_round_constants(current_state, round_constants[i]);
             apply_sbox(current_state);
             matrix_multiplication_external(current_state);
         }
 
+        // Internal rounds
         const size_t p_end = rounds_f_beginning + rounds_p;
         for (size_t i = rounds_f_beginning; i < p_end; ++i) {
             current_state[0] += round_constants[i][0];
             apply_single_sbox(current_state[0]);
             matrix_multiplication_internal(current_state);
         }
 
+        // Remaining external rounds
         for (size_t i = p_end; i < NUM_ROUNDS; ++i) {
             add_round_constants(current_state, round_constants[i]);
             apply_sbox(current_state);

barretenberg/cpp/src/barretenberg/proof_system/arithmetization/gate_data.hpp

@@ -138,19 +138,29 @@ template <typename FF> struct databus_lookup_gate_ {
     uint32_t value;
 };
 
+/* External gate data for poseidon2 external round*/
 template <typename FF> struct poseidon2_external_gate_ {
     uint32_t a;
     uint32_t b;
     uint32_t c;
     uint32_t d;
-    uint32_t round_idx;
+    size_t round_idx;
 };
 
+/* Internal gate data for poseidon2 internal round*/
 template <typename FF> struct poseidon2_internal_gate_ {
     uint32_t a;
     uint32_t b;
     uint32_t c;
     uint32_t d;
-    uint32_t round_idx;
+    size_t round_idx;
+};
+
+/* Last gate for poseidon2, needed because poseidon2 gates compare against the shifted wires. */
+template <typename FF> struct poseidon2_end_gate_ {
+    uint32_t a;
+    uint32_t b;
+    uint32_t c;
+    uint32_t d;
 };
 } // namespace proof_system

barretenberg/cpp/src/barretenberg/proof_system/circuit_builder/goblin_ultra_circuit_builder.cpp

@@ -260,6 +260,9 @@ void GoblinUltraCircuitBuilder_<FF>::create_calldata_lookup_gate(const databus_l
     ++this->num_gates;
 }
 
+/**
+ * @brief Poseidon2 external round gate, activates the q_poseidon2_external selector and relation
+ */
 template <typename FF>
 void GoblinUltraCircuitBuilder_<FF>::create_poseidon2_external_gate(const poseidon2_external_gate_<FF>& in)
 {
@@ -284,6 +287,9 @@ void GoblinUltraCircuitBuilder_<FF>::create_poseidon2_external_gate(const poseid
     ++this->num_gates;
 }
 
+/**
+ * @brief Poseidon2 internal round gate, activates the q_poseidon2_internal selector and relation
+ */
 template <typename FF>
 void GoblinUltraCircuitBuilder_<FF>::create_poseidon2_internal_gate(const poseidon2_internal_gate_<FF>& in)
 {
@@ -308,6 +314,36 @@ void GoblinUltraCircuitBuilder_<FF>::create_poseidon2_internal_gate(const poseid
     ++this->num_gates;
 }
 
+/**
+ * @brief Poseidon2 end round gate, needed because poseidon2 rounds compare with shifted wires
+ * @details The Poseidon2 permutation is 64 rounds, but needs to be a block of 65 rows, since the result of applying a
+ * round of Poseidon2 is stored in the next row (the shifted row). As a result, we need this end row to compare with the
+ * result from the 64th round of Poseidon2. Note that it does not activate any selectors since it only serves as a
+ * comparison through the shifted wires.
+ */
+template <typename FF> void GoblinUltraCircuitBuilder_<FF>::create_poseidon2_end_gate(const poseidon2_end_gate_<FF>& in)
+{
+    this->w_l().emplace_back(in.a);
+    this->w_r().emplace_back(in.b);
+    this->w_o().emplace_back(in.c);
+    this->w_4().emplace_back(in.d);
+    this->q_m().emplace_back(0);
+    this->q_1().emplace_back(0);
+    this->q_2().emplace_back(0);
+    this->q_3().emplace_back(0);
+    this->q_c().emplace_back(0);
+    this->q_arith().emplace_back(0);
+    this->q_4().emplace_back(0);
+    this->q_sort().emplace_back(0);
+    this->q_lookup_type().emplace_back(0);
+    this->q_elliptic().emplace_back(0);
+    this->q_aux().emplace_back(0);
+    this->q_busread().emplace_back(0);
+    this->q_poseidon2_external().emplace_back(0);
+    this->q_poseidon2_internal().emplace_back(0);
+    ++this->num_gates;
+}
+
 template <typename FF>
 inline FF GoblinUltraCircuitBuilder_<FF>::compute_poseidon2_external_identity(FF q_poseidon2_external_value,
                                                                               FF q_1_value,

barretenberg/cpp/src/barretenberg/proof_system/circuit_builder/goblin_ultra_circuit_builder.hpp

@@ -175,6 +175,7 @@ template <typename FF> class GoblinUltraCircuitBuilder_ : public UltraCircuitBui
 
     void create_poseidon2_external_gate(const poseidon2_external_gate_<FF>& in);
     void create_poseidon2_internal_gate(const poseidon2_internal_gate_<FF>& in);
+    void create_poseidon2_end_gate(const poseidon2_end_gate_<FF>& in);
 
     FF compute_poseidon2_external_identity(FF q_poseidon2_external_value,
                                            FF q_1_value,

barretenberg/cpp/src/barretenberg/stdlib/hash/CMakeLists.txt

@@ -3,4 +3,5 @@ add_subdirectory(blake3s)
 add_subdirectory(pedersen)
 add_subdirectory(sha256)
 add_subdirectory(keccak)
-add_subdirectory(benchmarks)
+add_subdirectory(benchmarks)
+add_subdirectory(poseidon2)

barretenberg/cpp/src/barretenberg/stdlib/hash/pedersen/pedersen.cpp

@@ -6,7 +6,7 @@ using namespace barretenberg;
 using namespace proof_system;
 
 template <typename C>
-field_t<C> pedersen_hash<C>::hash(const std::vector<field_t>& inputs, const GeneratorContext context)
+field_t<C> pedersen_hash<C>::hash(const std::vector<field_ct>& inputs, const GeneratorContext context)
 {
     using cycle_scalar = typename cycle_group::cycle_scalar;
     using Curve = EmbeddedCurve;
@@ -15,7 +15,7 @@ field_t<C> pedersen_hash<C>::hash(const std::vector<field_t>& inputs, const Gene
 
     std::vector<cycle_scalar> scalars;
     std::vector<cycle_group> points;
-    scalars.emplace_back(cycle_scalar::create_from_bn254_scalar(field_t(inputs.size())));
+    scalars.emplace_back(cycle_scalar::create_from_bn254_scalar(field_ct(inputs.size())));
     points.emplace_back(crypto::pedersen_hash_base<Curve>::length_generator);
     for (size_t i = 0; i < inputs.size(); ++i) {
         scalars.emplace_back(cycle_scalar::create_from_bn254_scalar(inputs[i]));
@@ -28,7 +28,7 @@ field_t<C> pedersen_hash<C>::hash(const std::vector<field_t>& inputs, const Gene
 }
 
 template <typename C>
-field_t<C> pedersen_hash<C>::hash_skip_field_validation(const std::vector<field_t>& inputs,
+field_t<C> pedersen_hash<C>::hash_skip_field_validation(const std::vector<field_ct>& inputs,
                                                         const GeneratorContext context)
 {
     using cycle_scalar = typename cycle_group::cycle_scalar;
@@ -38,7 +38,7 @@ field_t<C> pedersen_hash<C>::hash_skip_field_validation(const std::vector<field_
 
     std::vector<cycle_scalar> scalars;
     std::vector<cycle_group> points;
-    scalars.emplace_back(cycle_scalar::create_from_bn254_scalar(field_t(inputs.size())));
+    scalars.emplace_back(cycle_scalar::create_from_bn254_scalar(field_ct(inputs.size())));
     points.emplace_back(crypto::pedersen_hash_base<Curve>::length_generator);
     for (size_t i = 0; i < inputs.size(); ++i) {
         // `true` param = skip primality test when performing a scalar mul
@@ -52,7 +52,7 @@ field_t<C> pedersen_hash<C>::hash_skip_field_validation(const std::vector<field_
 }
 
 /**
- * Hash a byte_array.
+ * @brief Hash a byte_array.
  *
  * TODO(@zac-williamson #2796) Once Poseidon is implemented, replace this method with a more canonical hash algorithm
  * (that is less efficient)
@@ -64,21 +64,18 @@ field_t<C> pedersen_hash<C>::hash_buffer(const stdlib::byte_array<C>& input, Gen
     const size_t bytes_per_element = 31;
     size_t num_elements = static_cast<size_t>(num_bytes % bytes_per_element != 0) + (num_bytes / bytes_per_element);
 
-    std::vector<field_t> elements;
+    std::vector<field_ct> elements;
     for (size_t i = 0; i < num_elements; ++i) {
         size_t bytes_to_slice = 0;
         if (i == num_elements - 1) {
             bytes_to_slice = num_bytes - (i * bytes_per_element);
         } else {
             bytes_to_slice = bytes_per_element;
         }
-        auto element = static_cast<field_t>(input.slice(i * bytes_per_element, bytes_to_slice));
+        auto element = static_cast<field_ct>(input.slice(i * bytes_per_element, bytes_to_slice));
         elements.emplace_back(element);
     }
-    for (auto& x : elements) {
-        std::cout << x << std::endl;
-    }
-    field_t hashed;
+    field_ct hashed;
     if (elements.size() < 2) {
         hashed = hash(elements, context);
     } else {