Split Pedersen Hash & Commitment Gadgets (#95)

* [SQUASHED] Pedersen refactor into hash and commitment.

Rename `crypto_pedersen` to `crypto_pedersen_hash`

Rename.

Pull generator data stuff out of pedersen crypto.

Large refactor of pedersen native and stdlib.

FIX everything.

Get rid of unnecessary pedersen hash test. Its tricky to get this working: you need hash interface for byte array which is what we would like to avoid for pedersen_hash

Fix cci.

Enable ultra with different interfaces for:
1. pedersen hash
2. pedersen commitment

Use lookup pedersen for merkle tree, fixed-base pedersen for commitments.

Merkle tree test fixes.

Circuit vk updates.

* post rebase fixes.

* js eg circuit size fix.

---------

Co-authored-by: Suyash Bagad <[email protected]>

cpp/scripts/bb-tests

@@ -2,7 +2,7 @@ crypto_aes128_tests
 crypto_blake2s_tests
 crypto_blake3s_tests
 crypto_ecdsa_tests
-crypto_pedersen_tests
+crypto_pedersen_commitment_tests
 crypto_schnorr_tests
 crypto_sha256_tests
 ecc_tests
@@ -18,7 +18,7 @@ stdlib_blake2s_tests
 stdlib_blake3s_tests
 stdlib_ecdsa_tests
 stdlib_merkle_tree_tests
-stdlib_pedersen_tests
+stdlib_pedersen_commitment_tests
 stdlib_schnorr_tests
 stdlib_sha256_tests
 transcript_tests

cpp/src/CMakeLists.txt

@@ -76,15 +76,17 @@ if(WASM)
         $<TARGET_OBJECTS:crypto_blake3s_objects>
         $<TARGET_OBJECTS:crypto_keccak_objects>
         $<TARGET_OBJECTS:crypto_schnorr_objects>
-        $<TARGET_OBJECTS:crypto_pedersen_objects>
+        $<TARGET_OBJECTS:crypto_generators_objects>
+        $<TARGET_OBJECTS:crypto_pedersen_commitment_objects>
         $<TARGET_OBJECTS:ecc_objects>
         $<TARGET_OBJECTS:polynomials_objects>
         $<TARGET_OBJECTS:plonk_objects>
         $<TARGET_OBJECTS:honk_objects>
         $<TARGET_OBJECTS:proof_system_objects>
         $<TARGET_OBJECTS:stdlib_primitives_objects>
         $<TARGET_OBJECTS:stdlib_schnorr_objects>
-        $<TARGET_OBJECTS:stdlib_pedersen_objects>
+        $<TARGET_OBJECTS:stdlib_pedersen_hash_objects>
+        $<TARGET_OBJECTS:stdlib_pedersen_commitment_objects>
         $<TARGET_OBJECTS:stdlib_blake2s_objects>
         $<TARGET_OBJECTS:stdlib_blake3s_objects>
         $<TARGET_OBJECTS:stdlib_sha256_objects>
@@ -136,15 +138,17 @@ if(WASM)
         $<TARGET_OBJECTS:crypto_blake3s_objects>
         $<TARGET_OBJECTS:crypto_keccak_objects>
         $<TARGET_OBJECTS:crypto_schnorr_objects>
-        $<TARGET_OBJECTS:crypto_pedersen_objects>
+        $<TARGET_OBJECTS:crypto_pedersen_hash_objects>
+        $<TARGET_OBJECTS:crypto_pedersen_commitment_objects>
         $<TARGET_OBJECTS:ecc_objects>
         $<TARGET_OBJECTS:polynomials_objects>
         $<TARGET_OBJECTS:plonk_objects>
         $<TARGET_OBJECTS:honk_objects>
         $<TARGET_OBJECTS:proof_system_objects>
         $<TARGET_OBJECTS:stdlib_primitives_objects>
         $<TARGET_OBJECTS:stdlib_schnorr_objects>
-        $<TARGET_OBJECTS:stdlib_pedersen_objects>
+        $<TARGET_OBJECTS:stdlib_pedersen_hash_objects>
+        $<TARGET_OBJECTS:stdlib_pedersen_commitment_objects>
         $<TARGET_OBJECTS:stdlib_blake2s_objects>
         $<TARGET_OBJECTS:stdlib_blake3s_objects>
         $<TARGET_OBJECTS:stdlib_sha256_objects>
@@ -168,15 +172,17 @@ else()
         $<TARGET_OBJECTS:crypto_blake3s_objects>
         $<TARGET_OBJECTS:crypto_keccak_objects>
         $<TARGET_OBJECTS:crypto_schnorr_objects>
-        $<TARGET_OBJECTS:crypto_pedersen_objects>
+        $<TARGET_OBJECTS:crypto_pedersen_hash_objects>
+        $<TARGET_OBJECTS:crypto_pedersen_commitment_objects>
         $<TARGET_OBJECTS:ecc_objects>
         $<TARGET_OBJECTS:polynomials_objects>
         $<TARGET_OBJECTS:plonk_objects>
         $<TARGET_OBJECTS:honk_objects>
         $<TARGET_OBJECTS:proof_system_objects>
         $<TARGET_OBJECTS:stdlib_primitives_objects>
         $<TARGET_OBJECTS:stdlib_schnorr_objects>
-        $<TARGET_OBJECTS:stdlib_pedersen_objects>
+        $<TARGET_OBJECTS:stdlib_pedersen_hash_objects>
+        $<TARGET_OBJECTS:stdlib_pedersen_commitment_objects>
         $<TARGET_OBJECTS:stdlib_blake2s_objects>
         $<TARGET_OBJECTS:stdlib_blake3s_objects>
         $<TARGET_OBJECTS:stdlib_sha256_objects>

cpp/src/barretenberg/crypto/CMakeLists.txt

@@ -1,9 +1,11 @@
 add_subdirectory(hmac)
+add_subdirectory(generators)
 add_subdirectory(blake2s)
 add_subdirectory(blake3s)
 add_subdirectory(blake3s_full)
 add_subdirectory(keccak)
-add_subdirectory(pedersen)
+add_subdirectory(pedersen_commitment)
+add_subdirectory(pedersen_hash)
 add_subdirectory(schnorr)
 add_subdirectory(sha256)
 add_subdirectory(ecdsa)

cpp/src/barretenberg/crypto/generators/CMakeLists.txt

@@ -0,0 +1 @@
+barretenberg_module(crypto_generators ecc)

cpp/src/barretenberg/crypto/pedersen/fixed_base_scalar_mul.hpp → cpp/src/barretenberg/crypto/generators/fixed_base_scalar_mul.hpp

@@ -1,8 +1,9 @@
 #pragma once
 #include "barretenberg/ecc/curves/grumpkin/grumpkin.hpp"
+#include "./generator_data.hpp"
 
 namespace crypto {
-namespace pedersen {
+namespace generators {
 
 constexpr uint64_t WNAF_MASK = 0x7fffffffUL;
 
@@ -17,7 +18,7 @@ grumpkin::g1::element fixed_base_scalar_mul(const barretenberg::fr& in, const si
     constexpr size_t num_quads = ((num_quads_base << 1) + 1 < num_bits) ? num_quads_base + 1 : num_quads_base;
     constexpr size_t num_wnaf_bits = (num_quads << 1) + 1;
 
-    const crypto::pedersen::fixed_base_ladder* ladder = gen_data.get_ladder(num_bits);
+    const crypto::generators::fixed_base_ladder* ladder = gen_data.get_ladder(num_bits);
 
     uint64_t wnaf_entries[num_quads + 2] = { 0 };
     bool skew = false;
@@ -40,5 +41,5 @@ grumpkin::g1::element fixed_base_scalar_mul(const barretenberg::fr& in, const si
     return accumulator.normalize();
 }
 
-} // namespace pedersen
+} // namespace generators
 } // namespace crypto

cpp/src/barretenberg/crypto/pedersen/generator_data.cpp → cpp/src/barretenberg/crypto/generators/generator_data.cpp

@@ -1,7 +1,7 @@
 #include "./generator_data.hpp"
 
 namespace crypto {
-namespace pedersen {
+namespace generators {
 namespace {
 
 // The number of unique base points with default main index with precomputed ladders
@@ -12,8 +12,8 @@ constexpr size_t num_default_generators = 2048;
 #endif
 
 constexpr size_t hash_indices_generator_offset = 2048;
-constexpr size_t num_hash_indices = 16;
-constexpr size_t num_generators_per_hash_index = 8;
+constexpr size_t num_hash_indices = 32;
+constexpr size_t num_generators_per_hash_index = 64;
 constexpr size_t num_indexed_generators = num_hash_indices * num_generators_per_hash_index;
 constexpr size_t size_of_generator_data_array = hash_indices_generator_offset + num_indexed_generators;
 constexpr size_t num_generator_types = 3;
@@ -235,21 +235,26 @@ const fixed_base_ladder* get_g1_ladder(const size_t num_bits)
 /**
  * Generator indexing:
  *
+ * Number of default generators (index = 0): N = 2048
+ * Number of hash indices: H = 32
+ * Number of sub indices for a given hash index: h = 64.
+ * Number of types of generators needed per hash index: t = 3
+ *
  * Default generators:
- * 0: P_0  P_1  P_2  ...  P_{2047}
+ * 0: P_0  P_1  P_2  ...  P_{N'-1}
  *
- * Hash-index dependent generators:
- * 1:  P_{2048 + 0*8}   P_{2049}  ...  P_{2055}
- * 2:  P_{2048 + 1*8}   P_{2048 + 1*8 + 1}   ...  P_{2048 + 1*8 + 7}
- * 3:
+ * Hash-index dependent generators: (let N' = t * N)
+ * 1:  P_{N' + 0*h*t}   P_{N' + 0*h*t + 1*t}  ...  P_{N' + 0*h*t + (h-1)*t}
+ * 2:  P_{N' + 1*h*t}   P_{N' + 1*h*t + 1*t}  ...  P_{N' + 1*h*t + (h-1)*t}
+ * 2:  P_{N' + 2*h*t}   P_{N' + 2*h*t + 1*t}  ...  P_{N' + 2*h*t + (h-1)*t}
  * 4:
  * .
  * .
  * .
- * 31: P_{2048 + 30*8}  P_{2048 + 30*8 + 1}  ...  P_{2048 + 30*8 + 7}
- * 32: P_{2048 + 31*8}  P_{2048 + 31*8 + 1}  ...  P_{2048 + 31*8 + 7}
+ * H-1:  P_{N' + (H-2)*h*t}   P_{N' + (H-2)*h*t + 1*t}  ...  P_{N' + (H-2)*h*t + (h-1)*t}
+ * H  :  P_{N' + (H-1)*h*t}   P_{N' + (H-1)*h*t + 1*t}  ...  P_{N' + (H-1)*h*t + (h-1)*t}
  *
- * Total generators = 2048 + 32*8 = 2304
+ * Total generators = (N + H * h) * t = 2304
  */
 generator_data const& get_generator_data(generator_index_t index)
 {
@@ -276,5 +281,5 @@ const fixed_base_ladder* generator_data::get_hash_ladder(size_t num_bits) const
     return get_ladder_internal(hash_ladder, num_bits);
 }
 
-} // namespace pedersen
+} // namespace generators
 } // namespace crypto

cpp/src/barretenberg/crypto/pedersen/generator_data.hpp → cpp/src/barretenberg/crypto/generators/generator_data.hpp

@@ -4,7 +4,7 @@
 #include "barretenberg/ecc/curves/grumpkin/grumpkin.hpp"
 
 namespace crypto {
-namespace pedersen {
+namespace generators {
 
 struct generator_index_t {
     size_t index;
@@ -58,5 +58,5 @@ std::vector<std::unique_ptr<generator_data>> const& init_generator_data();
 const fixed_base_ladder* get_g1_ladder(const size_t num_bits);
 generator_data const& get_generator_data(generator_index_t index);
 
-} // namespace pedersen
+} // namespace generators
 } // namespace crypto

cpp/src/barretenberg/crypto/pedersen/pedersen.test.cpp → cpp/src/barretenberg/crypto/generators/generator_data.test.cpp

@@ -1,10 +1,11 @@
 #include <gtest/gtest.h>
 #include "barretenberg/common/streams.hpp"
-#include "./pedersen.hpp"
+#include "./fixed_base_scalar_mul.hpp"
+#include "./generator_data.hpp"
 
-using namespace crypto::pedersen;
+using namespace crypto::generators;
 
-TEST(pedersen, hash_ladder_structure)
+TEST(generators, hash_ladder_structure)
 {
     generator_index_t index = { 2, 0 };
     generator_data gen_data = get_generator_data(index);
@@ -60,7 +61,7 @@ TEST(pedersen, hash_ladder_structure)
     EXPECT_EQ(grumpkin::g1::element(hash_ladder[0].one), mult);
 }
 
-TEST(pedersen, fixed_base_scalar_mul)
+TEST(generators, fixed_base_scalar_mul)
 {
     uint256_t scalar(123, 0, 0, 0);
 

cpp/src/barretenberg/crypto/pedersen_commitment/CMakeLists.txt

@@ -0,0 +1 @@
+barretenberg_module(crypto_pedersen_commitment ecc crypto_generators crypto_pedersen_hash)

cpp/src/barretenberg/crypto/pedersen_commitment/c_bind.cpp

@@ -0,0 +1,47 @@
+#include "c_bind.hpp"
+#include "pedersen.hpp"
+#include "barretenberg/common/serialize.hpp"
+#include "barretenberg/common/timer.hpp"
+#include "barretenberg/common/mem.hpp"
+#include "barretenberg/common/streams.hpp"
+#define WASM_EXPORT __attribute__((visibility("default")))
+
+extern "C" {
+
+WASM_EXPORT void pedersen__init()
+{
+    crypto::generators::init_generator_data();
+}
+
+WASM_EXPORT void pedersen__compress(uint8_t const* inputs_buffer, uint8_t* output)
+{
+    std::vector<grumpkin::fq> to_compress;
+    read(inputs_buffer, to_compress);
+    auto r = crypto::pedersen_commitment::compress_native(to_compress);
+    barretenberg::fr::serialize_to_buffer(r, output);
+}
+
+WASM_EXPORT void pedersen__compress_with_hash_index(uint8_t const* inputs_buffer, uint8_t* output, uint32_t hash_index)
+{
+    std::vector<grumpkin::fq> to_compress;
+    read(inputs_buffer, to_compress);
+    auto r = crypto::pedersen_commitment::compress_native(to_compress, hash_index);
+    barretenberg::fr::serialize_to_buffer(r, output);
+}
+
+WASM_EXPORT void pedersen__commit(uint8_t const* inputs_buffer, uint8_t* output)
+{
+    std::vector<grumpkin::fq> to_compress;
+    read(inputs_buffer, to_compress);
+    grumpkin::g1::affine_element pedersen_hash = crypto::pedersen_commitment::commit_native(to_compress);
+
+    write(output, pedersen_hash);
+}
+
+WASM_EXPORT void pedersen__buffer_to_field(uint8_t const* data, size_t length, uint8_t* r)
+{
+    std::vector<uint8_t> to_compress(data, data + length);
+    auto output = crypto::pedersen_commitment::compress_native(to_compress);
+    write(r, output);
+}
+}

cpp/src/barretenberg/crypto/pedersen/c_bind.hpp → cpp/src/barretenberg/crypto/pedersen_commitment/c_bind.hpp

@@ -1,3 +1,4 @@
+#pragma once
 #include "barretenberg/common/serialize.hpp"
 #include "barretenberg/common/timer.hpp"
 #include "barretenberg/common/mem.hpp"
@@ -8,22 +9,11 @@ extern "C" {
 
 WASM_EXPORT void pedersen__init();
 
-WASM_EXPORT void pedersen__compress_fields(uint8_t const* left, uint8_t const* right, uint8_t* result);
-
 WASM_EXPORT void pedersen__compress(uint8_t const* inputs_buffer, uint8_t* output);
 
 WASM_EXPORT void pedersen__compress_with_hash_index(uint8_t const* inputs_buffer, uint8_t* output, uint32_t hash_index);
 
 WASM_EXPORT void pedersen__commit(uint8_t const* inputs_buffer, uint8_t* output);
 
 WASM_EXPORT void pedersen__buffer_to_field(uint8_t const* data, size_t length, uint8_t* r);
-
-/**
- * Given a buffer containing 32 byte pedersen leaves, return a new buffer containing the leaves and all pairs of
- * nodes that define a merkle tree.
- * e.g.
- * input:  [1][2][3][4]
- * output: [1][2][3][4][compress(1,2)][compress(3,4)][compress(5,6)]
- */
-WASM_EXPORT uint8_t* pedersen__hash_to_tree(uint8_t const* data);
-}
+}

cpp/src/barretenberg/crypto/pedersen/convert_buffer_to_field.hpp → cpp/src/barretenberg/crypto/pedersen_commitment/convert_buffer_to_field.hpp

@@ -3,7 +3,7 @@
 #include "barretenberg/ecc/curves/grumpkin/grumpkin.hpp"
 
 namespace crypto {
-namespace pedersen {
+namespace pedersen_commitment {
 
 inline std::vector<grumpkin::fq> convert_buffer_to_field(const std::vector<uint8_t>& input)
 {
@@ -33,5 +33,5 @@ inline std::vector<grumpkin::fq> convert_buffer_to_field(const std::vector<uint8
     }
     return elements;
 }
-} // namespace pedersen
-} // namespace crypto
+} // namespace pedersen_commitment
+} // namespace crypto