Split Pedersen Hash & Commitment Gadgets (#95)

* [SQUASHED] Pedersen refactor into hash and commitment.

Rename `crypto_pedersen` to `crypto_pedersen_hash`

Rename.

Pull generator data stuff out of pedersen crypto.

Large refactor of pedersen native and stdlib.

FIX everything.

Get rid of unnecessary pedersen hash test. Its tricky to get this working: you need hash interface for byte array which is what we would like to avoid for pedersen_hash

Fix cci.

Enable ultra with different interfaces for:
1. pedersen hash
2. pedersen commitment

Use lookup pedersen for merkle tree, fixed-base pedersen for commitments.

Merkle tree test fixes.

Circuit vk updates.

* post rebase fixes.

* js eg circuit size fix.

---------

Co-authored-by: Suyash Bagad <[email protected]>

cpp/scripts/bb-tests

@@ -2,7 +2,7 @@ crypto_aes128_tests
 crypto_blake2s_tests
 crypto_blake3s_tests
 crypto_ecdsa_tests
-crypto_pedersen_tests
+crypto_pedersen_commitment_tests
 crypto_schnorr_tests
 crypto_sha256_tests
 ecc_tests
@@ -18,7 +18,7 @@ stdlib_blake2s_tests
 stdlib_blake3s_tests
 stdlib_ecdsa_tests
 stdlib_merkle_tree_tests
-stdlib_pedersen_tests
+stdlib_pedersen_commitment_tests
 stdlib_schnorr_tests
 stdlib_sha256_tests
 transcript_tests

cpp/src/aztec/CMakeLists.txt

@@ -71,13 +71,15 @@ if(WASM)
         $<TARGET_OBJECTS:crypto_blake3s_objects>
         $<TARGET_OBJECTS:crypto_keccak_objects>
         $<TARGET_OBJECTS:crypto_schnorr_objects>
-        $<TARGET_OBJECTS:crypto_pedersen_objects>
+        $<TARGET_OBJECTS:crypto_generators_objects>
+        $<TARGET_OBJECTS:crypto_pedersen_commitment_objects>
         $<TARGET_OBJECTS:ecc_objects>
         $<TARGET_OBJECTS:polynomials_objects>
         $<TARGET_OBJECTS:plonk_objects>
         $<TARGET_OBJECTS:stdlib_primitives_objects>
         $<TARGET_OBJECTS:stdlib_schnorr_objects>
-        $<TARGET_OBJECTS:stdlib_pedersen_objects>
+        $<TARGET_OBJECTS:stdlib_pedersen_hash_objects>
+        $<TARGET_OBJECTS:stdlib_pedersen_commitment_objects>
         $<TARGET_OBJECTS:stdlib_blake2s_objects>
         $<TARGET_OBJECTS:stdlib_blake3s_objects>
     )
@@ -117,13 +119,15 @@ if(WASM)
         $<TARGET_OBJECTS:crypto_blake3s_objects>
         $<TARGET_OBJECTS:crypto_keccak_objects>
         $<TARGET_OBJECTS:crypto_schnorr_objects>
-        $<TARGET_OBJECTS:crypto_pedersen_objects>
+        $<TARGET_OBJECTS:crypto_pedersen_hash_objects>
+        $<TARGET_OBJECTS:crypto_pedersen_commitment_objects>
         $<TARGET_OBJECTS:ecc_objects>
         $<TARGET_OBJECTS:polynomials_objects>
         $<TARGET_OBJECTS:plonk_objects>
         $<TARGET_OBJECTS:stdlib_primitives_objects>
         $<TARGET_OBJECTS:stdlib_schnorr_objects>
-        $<TARGET_OBJECTS:stdlib_pedersen_objects>
+        $<TARGET_OBJECTS:stdlib_pedersen_hash_objects>
+        $<TARGET_OBJECTS:stdlib_pedersen_commitment_objects>
         $<TARGET_OBJECTS:stdlib_blake2s_objects>
         $<TARGET_OBJECTS:stdlib_blake3s_objects>
         $<TARGET_OBJECTS:stdlib_sha256_objects>
@@ -145,13 +149,15 @@ else()
         $<TARGET_OBJECTS:crypto_blake3s_objects>
         $<TARGET_OBJECTS:crypto_keccak_objects>
         $<TARGET_OBJECTS:crypto_schnorr_objects>
-        $<TARGET_OBJECTS:crypto_pedersen_objects>
+        $<TARGET_OBJECTS:crypto_pedersen_hash_objects>
+        $<TARGET_OBJECTS:crypto_pedersen_commitment_objects>
         $<TARGET_OBJECTS:ecc_objects>
         $<TARGET_OBJECTS:polynomials_objects>
         $<TARGET_OBJECTS:plonk_objects>
         $<TARGET_OBJECTS:stdlib_primitives_objects>
         $<TARGET_OBJECTS:stdlib_schnorr_objects>
-        $<TARGET_OBJECTS:stdlib_pedersen_objects>
+        $<TARGET_OBJECTS:stdlib_pedersen_hash_objects>
+        $<TARGET_OBJECTS:stdlib_pedersen_commitment_objects>
         $<TARGET_OBJECTS:stdlib_blake2s_objects>
         $<TARGET_OBJECTS:stdlib_blake3s_objects>
         $<TARGET_OBJECTS:stdlib_sha256_objects>

cpp/src/aztec/crypto/CMakeLists.txt

@@ -1,9 +1,11 @@
 add_subdirectory(hmac)
+add_subdirectory(generators)
 add_subdirectory(blake2s)
 add_subdirectory(blake3s)
 add_subdirectory(blake3s_full)
 add_subdirectory(keccak)
-add_subdirectory(pedersen)
+add_subdirectory(pedersen_commitment)
+add_subdirectory(pedersen_hash)
 add_subdirectory(schnorr)
 add_subdirectory(sha256)
 add_subdirectory(ecdsa)

cpp/src/aztec/crypto/generators/CMakeLists.txt

@@ -0,0 +1 @@
+barretenberg_module(crypto_generators ecc)

cpp/src/aztec/crypto/pedersen/fixed_base_scalar_mul.hpp → cpp/src/aztec/crypto/generators/fixed_base_scalar_mul.hpp

@@ -1,8 +1,9 @@
 #pragma once
 #include <ecc/curves/grumpkin/grumpkin.hpp>
+#include "./generator_data.hpp"
 
 namespace crypto {
-namespace pedersen {
+namespace generators {
 
 constexpr uint64_t WNAF_MASK = 0x7fffffffUL;
 
@@ -17,7 +18,7 @@ grumpkin::g1::element fixed_base_scalar_mul(const barretenberg::fr& in, const si
     constexpr size_t num_quads = ((num_quads_base << 1) + 1 < num_bits) ? num_quads_base + 1 : num_quads_base;
     constexpr size_t num_wnaf_bits = (num_quads << 1) + 1;
 
-    const crypto::pedersen::fixed_base_ladder* ladder = gen_data.get_ladder(num_bits);
+    const crypto::generators::fixed_base_ladder* ladder = gen_data.get_ladder(num_bits);
 
     uint64_t wnaf_entries[num_quads + 2] = { 0 };
     bool skew = false;
@@ -40,5 +41,5 @@ grumpkin::g1::element fixed_base_scalar_mul(const barretenberg::fr& in, const si
     return accumulator.normalize();
 }
 
-} // namespace pedersen
+} // namespace generators
 } // namespace crypto

cpp/src/aztec/crypto/pedersen/generator_data.cpp → cpp/src/aztec/crypto/generators/generator_data.cpp

@@ -1,7 +1,7 @@
 #include "./generator_data.hpp"
 
 namespace crypto {
-namespace pedersen {
+namespace generators {
 namespace {
 
 // The number of unique base points with default main index with precomputed ladders
@@ -276,5 +276,5 @@ const fixed_base_ladder* generator_data::get_hash_ladder(size_t num_bits) const
     return get_ladder_internal(hash_ladder, num_bits);
 }
 
-} // namespace pedersen
+} // namespace generators
 } // namespace crypto

cpp/src/aztec/crypto/pedersen/generator_data.hpp → cpp/src/aztec/crypto/generators/generator_data.hpp

@@ -4,7 +4,7 @@
 #include <ecc/curves/grumpkin/grumpkin.hpp>
 
 namespace crypto {
-namespace pedersen {
+namespace generators {
 
 struct generator_index_t {
     size_t index;
@@ -58,5 +58,5 @@ std::vector<std::unique_ptr<generator_data>> const& init_generator_data();
 const fixed_base_ladder* get_g1_ladder(const size_t num_bits);
 generator_data const& get_generator_data(generator_index_t index);
 
-} // namespace pedersen
+} // namespace generators
 } // namespace crypto

cpp/src/aztec/crypto/pedersen/pedersen.test.cpp → cpp/src/aztec/crypto/generators/generator_data.test.cpp

@@ -1,10 +1,11 @@
 #include <gtest/gtest.h>
 #include <common/streams.hpp>
-#include "./pedersen.hpp"
+#include "./fixed_base_scalar_mul.hpp"
+#include "./generator_data.hpp"
 
-using namespace crypto::pedersen;
+using namespace crypto::generators;
 
-TEST(pedersen, hash_ladder_structure)
+TEST(generators, hash_ladder_structure)
 {
     generator_index_t index = { 2, 0 };
     generator_data gen_data = get_generator_data(index);
@@ -60,7 +61,7 @@ TEST(pedersen, hash_ladder_structure)
     EXPECT_EQ(grumpkin::g1::element(hash_ladder[0].one), mult);
 }
 
-TEST(pedersen, fixed_base_scalar_mul)
+TEST(generators, fixed_base_scalar_mul)
 {
     uint256_t scalar(123, 0, 0, 0);
 

cpp/src/aztec/crypto/pedersen_commitment/CMakeLists.txt

@@ -0,0 +1 @@
+barretenberg_module(crypto_pedersen_commitment ecc crypto_generators crypto_pedersen_hash)

cpp/src/aztec/crypto/pedersen_commitment/c_bind.cpp

@@ -0,0 +1,37 @@
+#include "pedersen.hpp"
+#include <common/serialize.hpp>
+#include <common/timer.hpp>
+#include <common/mem.hpp>
+#include <common/streams.hpp>
+#define WASM_EXPORT __attribute__((visibility("default")))
+
+extern "C" {
+
+WASM_EXPORT void pedersen__init()
+{
+    crypto::generators::init_generator_data();
+}
+
+WASM_EXPORT void pedersen__compress(uint8_t const* inputs_buffer, uint8_t* output)
+{
+    std::vector<grumpkin::fq> to_compress;
+    read(inputs_buffer, to_compress);
+    auto r = crypto::pedersen_commitment::compress_native(to_compress);
+    barretenberg::fr::serialize_to_buffer(r, output);
+}
+
+WASM_EXPORT void pedersen__compress_with_hash_index(uint8_t const* inputs_buffer, uint8_t* output, uint32_t hash_index)
+{
+    std::vector<grumpkin::fq> to_compress;
+    read(inputs_buffer, to_compress);
+    auto r = crypto::pedersen_commitment::compress_native(to_compress, hash_index);
+    barretenberg::fr::serialize_to_buffer(r, output);
+}
+
+WASM_EXPORT void pedersen__buffer_to_field(uint8_t const* data, size_t length, uint8_t* r)
+{
+    std::vector<uint8_t> to_compress(data, data + length);
+    auto output = crypto::pedersen_commitment::compress_native(to_compress);
+    write(r, output);
+}
+}

cpp/src/aztec/crypto/pedersen/convert_buffer_to_field.hpp → cpp/src/aztec/crypto/pedersen_commitment/convert_buffer_to_field.hpp

@@ -3,7 +3,7 @@
 #include <ecc/curves/grumpkin/grumpkin.hpp>
 
 namespace crypto {
-namespace pedersen {
+namespace pedersen_commitment {
 
 inline std::vector<grumpkin::fq> convert_buffer_to_field(const std::vector<uint8_t>& input)
 {
@@ -33,5 +33,5 @@ inline std::vector<grumpkin::fq> convert_buffer_to_field(const std::vector<uint8
     }
     return elements;
 }
-} // namespace pedersen
+} // namespace pedersen_commitment
 } // namespace crypto

cpp/src/aztec/crypto/pedersen/pedersen.cpp → cpp/src/aztec/crypto/pedersen_commitment/pedersen.cpp

@@ -6,10 +6,12 @@
 #include <omp.h>
 #endif
 
+// using namespace crypto::generators;
+
 namespace crypto {
-namespace pedersen {
+namespace pedersen_commitment {
 
-grumpkin::g1::element hash_single(const barretenberg::fr& in, generator_index_t const& index)
+grumpkin::g1::element commit_single(const barretenberg::fr& in, generator_index_t const& index)
 {
     auto gen_data = get_generator_data(index);
     barretenberg::fr scalar_multiplier = in.from_montgomery_form();
@@ -19,7 +21,7 @@ grumpkin::g1::element hash_single(const barretenberg::fr& in, generator_index_t
     constexpr size_t num_quads = ((num_quads_base << 1) + 1 < num_bits) ? num_quads_base + 1 : num_quads_base;
     constexpr size_t num_wnaf_bits = (num_quads << 1) + 1;
 
-    const crypto::pedersen::fixed_base_ladder* ladder = gen_data.get_hash_ladder(num_bits);
+    const crypto::generators::fixed_base_ladder* ladder = gen_data.get_hash_ladder(num_bits);
 
     uint64_t wnaf_entries[num_quads + 2] = { 0 };
     bool skew = false;
@@ -56,7 +58,7 @@ grumpkin::g1::affine_element commit_native(const std::vector<grumpkin::fq>& inpu
 #endif
     for (size_t i = 0; i < inputs.size(); ++i) {
         generator_index_t index = { hash_index, i };
-        out[i] = hash_single(inputs[i], index);
+        out[i] = commit_single(inputs[i], index);
     }
 
     grumpkin::g1::element r = out[0];
@@ -89,5 +91,5 @@ grumpkin::fq compress_native(const std::vector<uint8_t>& input)
     return compress_native_buffer_to_field(input);
 }
 
-} // namespace pedersen
+} // namespace pedersen_commitment
 } // namespace crypto

cpp/src/aztec/crypto/pedersen/pedersen.hpp → cpp/src/aztec/crypto/pedersen_commitment/pedersen.hpp

@@ -1,13 +1,15 @@
 #pragma once
 #include <array>
 #include <ecc/curves/grumpkin/grumpkin.hpp>
-#include "./generator_data.hpp"
-#include "./fixed_base_scalar_mul.hpp"
+#include "../generators/generator_data.hpp"
+#include "../generators/fixed_base_scalar_mul.hpp"
+
+using namespace crypto::generators;
 
 namespace crypto {
-namespace pedersen {
+namespace pedersen_commitment {
 
-grumpkin::g1::element hash_single(const barretenberg::fr& in, generator_index_t const& index);
+grumpkin::g1::element commit_single(const barretenberg::fr& in, generator_index_t const& index);
 
 grumpkin::g1::affine_element commit_native(const std::vector<grumpkin::fq>& elements, const size_t hash_index = 0);
 
@@ -21,5 +23,5 @@ template <size_t T> grumpkin::fq compress_native(const std::array<grumpkin::fq,
 
 grumpkin::fq compress_native(const std::vector<uint8_t>& input);
 
-} // namespace pedersen
+} // namespace pedersen_commitment
 } // namespace crypto

cpp/src/aztec/crypto/pedersen_commitment/pedersen_lookup.cpp

@@ -0,0 +1,79 @@
+#include "./pedersen_lookup.hpp"
+#include "../pedersen_hash/pedersen_lookup.hpp"
+#include "./convert_buffer_to_field.hpp"
+
+#include <ecc/curves/grumpkin/grumpkin.hpp>
+
+using namespace crypto::pedersen_hash::lookup;
+
+namespace crypto::pedersen_hash::lookup {
+extern std::array<std::vector<grumpkin::g1::affine_element>, NUM_PEDERSEN_TABLES> pedersen_tables;
+extern std::vector<grumpkin::g1::affine_element> pedersen_iv_table;
+extern std::array<grumpkin::g1::affine_element, NUM_PEDERSEN_TABLES> generators;
+} // namespace crypto::pedersen_hash::lookup
+
+namespace crypto {
+namespace pedersen_commitment {
+namespace lookup {
+
+grumpkin::g1::element merkle_damgard_compress(const std::vector<grumpkin::fq>& inputs, const size_t iv)
+{
+    if (inputs.size() == 0) {
+        auto result = grumpkin::g1::affine_one;
+        result.self_set_infinity();
+        return result;
+    }
+    init();
+    const size_t num_inputs = inputs.size();
+
+    grumpkin::fq result = (pedersen_iv_table[iv]).x;
+    for (size_t i = 0; i < num_inputs; i++) {
+        result = hash_pair(result, inputs[i]);
+    }
+
+    return (hash_single(result, false) + hash_single(grumpkin::fq(num_inputs), true));
+}
+
+grumpkin::g1::affine_element commit_native(const std::vector<grumpkin::fq>& inputs, const size_t hash_index)
+{
+    return grumpkin::g1::affine_element(merkle_damgard_compress(inputs, hash_index));
+}
+
+grumpkin::fq compress_native(const std::vector<grumpkin::fq>& inputs, const size_t hash_index)
+{
+    return commit_native(inputs, hash_index).x;
+}
+
+grumpkin::fq compress_native_buffer_to_field(const std::vector<uint8_t>& input)
+{
+    const auto elements = convert_buffer_to_field(input);
+    grumpkin::fq result_fq = compress_native(elements);
+    return result_fq;
+}
+
+std::vector<uint8_t> compress_native(const std::vector<uint8_t>& input)
+{
+    const auto result_fq = compress_native_buffer_to_field(input);
+    uint256_t result_u256(result_fq);
+    const size_t num_bytes = input.size();
+
+    bool is_zero = true;
+    for (const auto byte : input) {
+        is_zero = is_zero && (byte == static_cast<uint8_t>(0));
+    }
+    if (is_zero) {
+        result_u256 = num_bytes;
+    }
+    std::vector<uint8_t> result_buffer;
+    result_buffer.reserve(32);
+    for (size_t i = 0; i < 32; ++i) {
+        const uint64_t shift = (31 - i) * 8;
+        uint256_t shifted = result_u256 >> uint256_t(shift);
+        result_buffer.push_back(static_cast<uint8_t>(shifted.data[0]));
+    }
+    return result_buffer;
+}
+
+} // namespace lookup
+} // namespace pedersen_commitment
+} // namespace crypto