From 8590e15f9624f5a7fb9b771d873d05d2022c7482 Mon Sep 17 00:00:00 2001 From: Suyash Bagad Date: Wed, 29 Mar 2023 18:12:38 +0100 Subject: [PATCH] Add Indexed Merkle Tree (#281) * add basic indexed merkle tree. * membership check for nullifier tree and cleanup. * test works for a simple case. * fix test. * fix. * Add comment. --- .../barretenberg/stdlib/merkle_tree/hash.hpp | 15 +- .../stdlib/merkle_tree/hash.test.cpp | 5 +- .../stdlib/merkle_tree/hash_path.hpp | 6 +- .../stdlib/merkle_tree/membership.hpp | 29 +++ .../stdlib/merkle_tree/membership.test.cpp | 70 ++++++ .../stdlib/merkle_tree/memory_tree.cpp | 4 +- .../stdlib/merkle_tree/memory_tree.hpp | 2 +- .../stdlib/merkle_tree/memory_tree.test.cpp | 6 +- .../stdlib/merkle_tree/merkle_tree.bench.cpp | 2 +- .../stdlib/merkle_tree/merkle_tree.cpp | 14 +- .../stdlib/merkle_tree/merkle_tree.hpp | 4 +- .../nullifier_tree/nullifier_leaf.hpp | 65 ++++++ .../nullifier_tree/nullifier_memory_tree.cpp | 65 ++++++ .../nullifier_tree/nullifier_memory_tree.hpp | 93 ++++++++ .../nullifier_memory_tree.test.cpp | 203 ++++++++++++++++++ .../nullifier_tree/nullifier_tree.cpp | 85 ++++++++ .../nullifier_tree/nullifier_tree.hpp | 45 ++++ .../nullifier_tree/nullifier_tree.test.cpp | 140 ++++++++++++ 18 files changed, 827 insertions(+), 26 deletions(-) create mode 100644 cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_leaf.hpp create mode 100644 cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.cpp create mode 100644 cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp create mode 100644 cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.test.cpp create mode 100644 cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.cpp create mode 100644 cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.hpp create mode 100644 cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.test.cpp diff --git a/cpp/src/barretenberg/stdlib/merkle_tree/hash.hpp b/cpp/src/barretenberg/stdlib/merkle_tree/hash.hpp index 12506ea9f8..27da44cf5a 100644 --- a/cpp/src/barretenberg/stdlib/merkle_tree/hash.hpp +++ b/cpp/src/barretenberg/stdlib/merkle_tree/hash.hpp @@ -13,12 +13,21 @@ namespace plonk { namespace stdlib { namespace merkle_tree { -inline barretenberg::fr compress_native(barretenberg::fr const& lhs, barretenberg::fr const& rhs) +inline barretenberg::fr hash_pair_native(barretenberg::fr const& lhs, barretenberg::fr const& rhs) { if (plonk::SYSTEM_COMPOSER == plonk::PLOOKUP) { - return crypto::pedersen_hash::lookup::hash_multiple({ lhs, rhs }); + return crypto::pedersen_hash::lookup::hash_multiple({ lhs, rhs }); // uses lookup tables } else { - return crypto::pedersen_hash::hash_multiple({ lhs, rhs }); + return crypto::pedersen_hash::hash_multiple({ lhs, rhs }); // uses fixed-base multiplication gate + } +} + +inline barretenberg::fr hash_multiple_native(std::vector const& inputs) +{ + if (plonk::SYSTEM_COMPOSER == plonk::PLOOKUP) { + return crypto::pedersen_hash::lookup::hash_multiple(inputs); // uses lookup tables + } else { + return crypto::pedersen_hash::hash_multiple(inputs); // uses fixed-base multiplication gate } } diff --git a/cpp/src/barretenberg/stdlib/merkle_tree/hash.test.cpp b/cpp/src/barretenberg/stdlib/merkle_tree/hash.test.cpp index c1a93c4625..4dffee6023 100644 --- a/cpp/src/barretenberg/stdlib/merkle_tree/hash.test.cpp +++ b/cpp/src/barretenberg/stdlib/merkle_tree/hash.test.cpp @@ -13,10 +13,7 @@ TEST(stdlib_merkle_tree_hash, compress_native_vs_circuit) Composer composer = Composer(); witness_ct y = witness_ct(&composer, x); field_ct z = plonk::stdlib::pedersen_hash::hash_multiple({ y, y }); - auto zz = crypto::pedersen_hash::hash_multiple({ x, x }); // uses fixed-base multiplication gate - if constexpr (Composer::type == ComposerType::PLOOKUP) { - zz = crypto::pedersen_hash::lookup::hash_multiple({ x, x }); // uses lookup tables - } + auto zz = stdlib::merkle_tree::hash_pair_native(x, x); EXPECT_EQ(z.get_value(), zz); } diff --git a/cpp/src/barretenberg/stdlib/merkle_tree/hash_path.hpp b/cpp/src/barretenberg/stdlib/merkle_tree/hash_path.hpp index 174db8b7e2..de8276bb1e 100644 --- a/cpp/src/barretenberg/stdlib/merkle_tree/hash_path.hpp +++ b/cpp/src/barretenberg/stdlib/merkle_tree/hash_path.hpp @@ -24,7 +24,7 @@ inline fr_hash_path get_new_hash_path(fr_hash_path const& old_path, uint128_t in } else { path[i].first = current; } - current = compress_native(path[i].first, path[i].second); + current = hash_pair_native(path[i].first, path[i].second); index /= 2; } return path; @@ -50,14 +50,14 @@ template inline hash_path create_witness_hash_path(Ctx& ctx, inline fr get_hash_path_root(fr_hash_path const& input) { - return compress_native(input[input.size() - 1].first, input[input.size() - 1].second); + return hash_pair_native(input[input.size() - 1].first, input[input.size() - 1].second); } inline fr zero_hash_at_height(size_t height) { auto current = fr(0); for (size_t i = 0; i < height; ++i) { - current = compress_native(current, current); + current = hash_pair_native(current, current); } return current; } diff --git a/cpp/src/barretenberg/stdlib/merkle_tree/membership.hpp b/cpp/src/barretenberg/stdlib/merkle_tree/membership.hpp index c6bb070207..44d0cc2ea8 100644 --- a/cpp/src/barretenberg/stdlib/merkle_tree/membership.hpp +++ b/cpp/src/barretenberg/stdlib/merkle_tree/membership.hpp @@ -149,6 +149,35 @@ void update_membership(field_t const& new_root, assert_check_membership(new_root, old_hashes, new_value, index, true, msg + "_new_value"); } +/** + * Asserts if the state transitions on updating multiple existing leaves with new values. + * + * @param old_root: The root of the merkle tree before it was updated, + * @param new_roots: New roots after each existing leaf was updated, + * @param new_values: The new values that are inserted in the existing leaves, + * @param old_values: The values of the existing leaves that were updated, + * @param old_paths: The hash path from the given index right before a given existing leaf is updated, + * @param old_indicies: Indices of the existing leaves that need to be updated, + * @tparam Composer: type of composer. + */ +template +field_t update_memberships(field_t old_root, + std::vector> const& new_roots, + std::vector> const& new_values, + std::vector> const& old_values, + std::vector> const& old_paths, + std::vector> const& old_indicies) +{ + for (size_t i = 0; i < old_indicies.size(); i++) { + update_membership( + new_roots[i], new_values[i], old_root, old_paths[i], old_values[i], old_indicies[i], "update_memberships"); + + old_root = new_roots[i]; + } + + return old_root; +} + /** * Asserts if old and new state of the tree is correct after a subtree-update. * diff --git a/cpp/src/barretenberg/stdlib/merkle_tree/membership.test.cpp b/cpp/src/barretenberg/stdlib/merkle_tree/membership.test.cpp index 0dee186932..4eb94c0193 100644 --- a/cpp/src/barretenberg/stdlib/merkle_tree/membership.test.cpp +++ b/cpp/src/barretenberg/stdlib/merkle_tree/membership.test.cpp @@ -9,6 +9,10 @@ using namespace barretenberg; using namespace plonk::stdlib::types; using namespace plonk::stdlib::merkle_tree; +namespace { +auto& engine = numeric::random::get_debug_engine(); +} + TEST(stdlib_merkle_tree, test_check_membership) { MemoryStore store; @@ -203,3 +207,69 @@ TEST(stdlib_merkle_tree, test_tree) bool result = verifier.verify_proof(proof); EXPECT_EQ(result, true); } + +TEST(stdlib_merkle_tree, test_update_memberships) +{ + constexpr size_t depth = 4; + MemoryStore store; + MerkleTree tree(store, depth); + + Composer composer = Composer(); + + constexpr size_t filled = (1UL << depth) / 2; + std::vector filled_values; + for (size_t i = 0; i < filled; i++) { + uint256_t val = fr::random_element(); + tree.update_element(i, val); + filled_values.push_back(val); + } + + // old state + fr old_root = tree.root(); + std::vector old_indices = { 0, 2, 5, 7 }; + + std::vector old_values; + std::vector old_hash_paths; + for (size_t i = 0; i < old_indices.size(); i++) { + old_values.push_back(filled_values[old_indices[i]]); + } + + // new state + std::vector new_values; + std::vector new_roots; + for (size_t i = 0; i < old_indices.size(); i++) { + uint256_t val = fr::random_element(); + new_values.push_back(val); + old_hash_paths.push_back(tree.get_hash_path(old_indices[i])); + new_roots.push_back(tree.update_element(old_indices[i], new_values[i])); + } + + // old state circuit types + field_ct old_root_ct = witness_ct(&composer, old_root); + std::vector> old_indices_ct; + std::vector old_values_ct; + std::vector> old_hash_paths_ct; + + // new state circuit types + std::vector new_values_ct; + std::vector new_roots_ct; + + for (size_t i = 0; i < old_indices.size(); i++) { + auto idx_vec = field_ct(witness_ct(&composer, uint256_t(old_indices[i]))).decompose_into_bits(depth); + old_indices_ct.push_back(idx_vec); + old_values_ct.push_back(witness_ct(&composer, old_values[i])); + old_hash_paths_ct.push_back(create_witness_hash_path(composer, old_hash_paths[i])); + + new_values_ct.push_back(witness_ct(&composer, new_values[i])); + new_roots_ct.push_back(witness_ct(&composer, new_roots[i])); + } + + update_memberships(old_root_ct, new_roots_ct, new_values_ct, old_values_ct, old_hash_paths_ct, old_indices_ct); + + auto prover = composer.create_prover(); + printf("composer gates = %zu\n", composer.get_num_gates()); + auto verifier = composer.create_verifier(); + plonk::proof proof = prover.construct_proof(); + bool result = verifier.verify_proof(proof); + EXPECT_EQ(result, true); +} diff --git a/cpp/src/barretenberg/stdlib/merkle_tree/memory_tree.cpp b/cpp/src/barretenberg/stdlib/merkle_tree/memory_tree.cpp index 331d5a2075..685ee73aa2 100644 --- a/cpp/src/barretenberg/stdlib/merkle_tree/memory_tree.cpp +++ b/cpp/src/barretenberg/stdlib/merkle_tree/memory_tree.cpp @@ -19,7 +19,7 @@ MemoryTree::MemoryTree(size_t depth) for (size_t i = 0; i < layer_size; ++i) { hashes_[offset + i] = current; } - current = compress_native(current, current); + current = hash_pair_native(current, current); } root_ = current; @@ -48,7 +48,7 @@ fr MemoryTree::update_element(size_t index, fr const& value) for (size_t i = 0; i < depth_; ++i) { hashes_[offset + index] = current; index &= (~0ULL) - 1; - current = compress_native(hashes_[offset + index], hashes_[offset + index + 1]); + current = hash_pair_native(hashes_[offset + index], hashes_[offset + index + 1]); offset += layer_size; layer_size >>= 1; index >>= 1; diff --git a/cpp/src/barretenberg/stdlib/merkle_tree/memory_tree.hpp b/cpp/src/barretenberg/stdlib/merkle_tree/memory_tree.hpp index 792ac7dbb0..ccec978120 100644 --- a/cpp/src/barretenberg/stdlib/merkle_tree/memory_tree.hpp +++ b/cpp/src/barretenberg/stdlib/merkle_tree/memory_tree.hpp @@ -34,7 +34,7 @@ class MemoryTree { fr root() const { return root_; } - private: + public: size_t depth_; size_t total_size_; barretenberg::fr root_; diff --git a/cpp/src/barretenberg/stdlib/merkle_tree/memory_tree.test.cpp b/cpp/src/barretenberg/stdlib/merkle_tree/memory_tree.test.cpp index cdceef6ea4..7466eeb15b 100644 --- a/cpp/src/barretenberg/stdlib/merkle_tree/memory_tree.test.cpp +++ b/cpp/src/barretenberg/stdlib/merkle_tree/memory_tree.test.cpp @@ -19,9 +19,9 @@ TEST(stdlib_merkle_tree, test_memory_store) fr e01 = VALUES[1]; fr e02 = VALUES[2]; fr e03 = VALUES[3]; - fr e10 = compress_native(e00, e01); - fr e11 = compress_native(e02, e03); - fr root = compress_native(e10, e11); + fr e10 = hash_pair_native(e00, e01); + fr e11 = hash_pair_native(e02, e03); + fr root = hash_pair_native(e10, e11); MemoryTree db(2); for (size_t i = 0; i < 4; ++i) { diff --git a/cpp/src/barretenberg/stdlib/merkle_tree/merkle_tree.bench.cpp b/cpp/src/barretenberg/stdlib/merkle_tree/merkle_tree.bench.cpp index 46a8ad9363..be36393b95 100644 --- a/cpp/src/barretenberg/stdlib/merkle_tree/merkle_tree.bench.cpp +++ b/cpp/src/barretenberg/stdlib/merkle_tree/merkle_tree.bench.cpp @@ -25,7 +25,7 @@ static std::vector VALUES = []() { void hash(State& state) noexcept { for (auto _ : state) { - compress_native({ 0, 0, 0, 0 }, { 1, 1, 1, 1 }); + hash_pair_native({ 0, 0, 0, 0 }, { 1, 1, 1, 1 }); } } BENCHMARK(hash)->MinTime(5); diff --git a/cpp/src/barretenberg/stdlib/merkle_tree/merkle_tree.cpp b/cpp/src/barretenberg/stdlib/merkle_tree/merkle_tree.cpp index f37ed0c901..cacf082136 100644 --- a/cpp/src/barretenberg/stdlib/merkle_tree/merkle_tree.cpp +++ b/cpp/src/barretenberg/stdlib/merkle_tree/merkle_tree.cpp @@ -32,7 +32,7 @@ MerkleTree::MerkleTree(Store& store, size_t depth, uint8_t tree_id) auto current = fr(0); for (size_t i = 0; i < depth; ++i) { zero_hashes_[i] = current; - current = compress_native(current, current); + current = hash_pair_native(current, current); } } @@ -51,7 +51,7 @@ template fr MerkleTree::root() const std::vector root; std::vector key = { tree_id_ }; bool status = store_.get(key, root); - return status ? from_buffer(root) : compress_native(zero_hashes_.back(), zero_hashes_.back()); + return status ? from_buffer(root) : hash_pair_native(zero_hashes_.back(), zero_hashes_.back()); } template typename MerkleTree::index_t MerkleTree::size() const @@ -103,7 +103,7 @@ template fr_hash_path MerkleTree::get_hash_path(index_t } else { path[j] = std::make_pair(current, zero_hashes_[j]); } - current = compress_native(path[j].first, path[j].second); + current = hash_pair_native(path[j].first, path[j].second); } } else { // Requesting path to a different, indepenent element. @@ -123,7 +123,7 @@ template fr_hash_path MerkleTree::get_hash_path(index_t } else { path[j] = std::make_pair(current, zero_hashes_[j]); } - current = compress_native(path[j].first, path[j].second); + current = hash_pair_native(path[j].first, path[j].second); } } break; @@ -158,7 +158,7 @@ template fr MerkleTree::binary_put(index_t a_index, fr c bool a_is_right = bit_set(a_index, height - 1); auto left = a_is_right ? b : a; auto right = a_is_right ? a : b; - auto key = compress_native(left, right); + auto key = hash_pair_native(left, right); put(key, left, right); return key; } @@ -236,7 +236,7 @@ fr MerkleTree::update_element(fr const& root, fr const& value, index_t in } else { left = subtree_root; } - auto new_root = compress_native(left, right); + auto new_root = hash_pair_native(left, right); put(new_root, left, right); // Remove the old node only while rolling back in recursion. @@ -260,7 +260,7 @@ template fr MerkleTree::compute_zero_path_hash(size_t he right = zero_hashes_[i]; left = current; } - current = compress_native(is_right ? zero_hashes_[i] : current, is_right ? current : zero_hashes_[i]); + current = hash_pair_native(left, right); } return current; } diff --git a/cpp/src/barretenberg/stdlib/merkle_tree/merkle_tree.hpp b/cpp/src/barretenberg/stdlib/merkle_tree/merkle_tree.hpp index 4d34c41947..b1efc7f49d 100644 --- a/cpp/src/barretenberg/stdlib/merkle_tree/merkle_tree.hpp +++ b/cpp/src/barretenberg/stdlib/merkle_tree/merkle_tree.hpp @@ -29,7 +29,7 @@ template class MerkleTree { index_t size() const; - private: + protected: void load_metadata(); /** @@ -88,7 +88,7 @@ template class MerkleTree { void remove(fr const& key); - private: + protected: Store& store_; std::vector zero_hashes_; size_t depth_; diff --git a/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_leaf.hpp b/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_leaf.hpp new file mode 100644 index 0000000000..d94c6638a0 --- /dev/null +++ b/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_leaf.hpp @@ -0,0 +1,65 @@ +#pragma once +#include "barretenberg/crypto/pedersen_commitment/pedersen.hpp" + +namespace plonk { +namespace stdlib { +namespace merkle_tree { + +using namespace barretenberg; +typedef uint256_t index_t; + +struct nullifier_leaf { + fr value; + index_t nextIndex; + fr nextValue; + + bool operator==(nullifier_leaf const&) const = default; + + std::ostream& operator<<(std::ostream& os) + { + os << "value = " << value << "\nnextIdx = " << nextIndex << "\nnextVal = " << nextValue; + return os; + } + + void read(uint8_t const*& it) + { + using serialize::read; + read(it, value); + read(it, nextIndex); + read(it, nextValue); + } + + inline void write(std::vector& buf) + { + using serialize::write; + write(buf, value); + write(buf, nextIndex); + write(buf, nextValue); + } + + barretenberg::fr hash() const { return stdlib::merkle_tree::hash_multiple_native({ value, nextIndex, nextValue }); } +}; + +inline std::pair find_closest_leaf(std::vector const& leaves_, fr const& new_value) +{ + std::vector diff; + bool repeated = false; + for (size_t i = 0; i < leaves_.size(); i++) { + auto leaf_value_ = uint256_t(leaves_[i].value); + auto new_value_ = uint256_t(new_value); + if (leaf_value_ > new_value_) { + diff.push_back(leaf_value_); + } else if (leaf_value_ == new_value_) { + repeated = true; + return std::make_pair(i, repeated); + } else { + diff.push_back(new_value_ - leaf_value_); + } + } + auto it = std::min_element(diff.begin(), diff.end()); + return std::make_pair(static_cast(it - diff.begin()), repeated); +} + +} // namespace merkle_tree +} // namespace stdlib +} // namespace plonk \ No newline at end of file diff --git a/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.cpp b/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.cpp new file mode 100644 index 0000000000..bbc713ed9e --- /dev/null +++ b/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.cpp @@ -0,0 +1,65 @@ +#include "nullifier_memory_tree.hpp" +#include "../hash.hpp" + +namespace plonk { +namespace stdlib { +namespace merkle_tree { + +NullifierMemoryTree::NullifierMemoryTree(size_t depth) + : MemoryTree(depth) +{ + ASSERT(depth_ >= 1 && depth <= 32); + total_size_ = 1UL << depth_; + hashes_.resize(total_size_ * 2 - 2); + + // Build the entire tree. + nullifier_leaf zero_leaf = { 0, 0, 0 }; + leaves_.push_back(zero_leaf); + auto current = zero_leaf.hash(); + update_element(0, current); + size_t layer_size = total_size_; + for (size_t offset = 0; offset < hashes_.size(); offset += layer_size, layer_size /= 2) { + for (size_t i = 0; i < layer_size; ++i) { + hashes_[offset + i] = current; + } + current = hash_pair_native(current, current); + } + + root_ = current; +} + +fr NullifierMemoryTree::update_element(fr const& value) +{ + // Find the leaf with the value closest and less than `value` + size_t current; + bool is_already_present; + std::tie(current, is_already_present) = find_closest_leaf(leaves_, value); + + nullifier_leaf new_leaf = { .value = value, + .nextIndex = leaves_[current].nextIndex, + .nextValue = leaves_[current].nextValue }; + if (!is_already_present) { + // Update the current leaf to point it to the new leaf + leaves_[current].nextIndex = leaves_.size(); + leaves_[current].nextValue = value; + + // Insert the new leaf with (nextIndex, nextValue) of the current leaf + leaves_.push_back(new_leaf); + } + + // Update the old leaf in the tree + auto old_leaf_hash = leaves_[current].hash(); + size_t old_leaf_index = current; + auto root = update_element(old_leaf_index, old_leaf_hash); + + // Insert the new leaf in the tree + auto new_leaf_hash = new_leaf.hash(); + size_t new_leaf_index = is_already_present ? old_leaf_index : leaves_.size() - 1; + root = update_element(new_leaf_index, new_leaf_hash); + + return root; +} + +} // namespace merkle_tree +} // namespace stdlib +} // namespace plonk \ No newline at end of file diff --git a/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp b/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp new file mode 100644 index 0000000000..3b49fc19bd --- /dev/null +++ b/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp @@ -0,0 +1,93 @@ +#pragma once +#include "../hash.hpp" +#include "../memory_tree.hpp" +#include "nullifier_leaf.hpp" + +namespace plonk { +namespace stdlib { +namespace merkle_tree { + +using namespace barretenberg; + +/** + * An NullifierMemoryTree is structured just like a usual merkle tree: + * + * hashes_ + * +------------------------------------------------------------------------------+ + * | 0 -> h_{0,0} h_{0,1} h_{0,2} h_{0,3} h_{0,4} h_{0,5} h_{0,6} h_{0,7} | + * i | | + * n | 8 -> h_{1,0} h_{1,1} h_{1,2} h_{1,3} | + * d | | + * e | 12 -> h_{2,0} h_{2,1} | + * x | | + * | 14 -> h_{3,0} | + * +------------------------------------------------------------------------------+ + * + * Here, depth_ = 3 and {h_{0,j}}_{i=0..7} are leaf values. + * Also, root_ = h_{3,0} and total_size_ = (2 * 8 - 2) = 14. + * Lastly, h_{i,j} = hash( h_{i-1,2j}, h_{i-1,2j+1} ) where i > 1. + * + * 1. Initial state: + * + * # + * + * # # + * + * # # # # + * + * # # # # # # # # + * + * index 0 1 2 3 4 5 6 7 + * + * val 0 0 0 0 0 0 0 0 + * nextIdx 0 0 0 0 0 0 0 0 + * nextVal 0 0 0 0 0 0 0 0 + * + * 2. Add new leaf with value 30 + * + * val 0 30 0 0 0 0 0 0 + * nextIdx 1 0 0 0 0 0 0 0 + * nextVal 30 0 0 0 0 0 0 0 + * + * 3. Add new leaf with value 10 + * + * val 0 30 10 0 0 0 0 0 + * nextIdx 2 0 1 0 0 0 0 0 + * nextVal 10 0 30 0 0 0 0 0 + * + * 4. Add new leaf with value 20 + * + * val 0 30 10 20 0 0 0 0 + * nextIdx 2 0 3 1 0 0 0 0 + * nextVal 10 0 20 30 0 0 0 0 + * + * 5. Add new leaf with value 50 + * + * val 0 30 10 20 50 0 0 0 + * nextIdx 2 4 3 1 0 0 0 0 + * nextVal 10 50 20 30 0 0 0 0 + */ +class NullifierMemoryTree : public MemoryTree { + public: + NullifierMemoryTree(size_t depth); + + using MemoryTree::get_hash_path; + using MemoryTree::root; + using MemoryTree::update_element; + + fr update_element(fr const& value); + + const std::vector& get_hashes() { return hashes_; } + const std::vector& get_leaves() { return leaves_; } + + private: + using MemoryTree::depth_; + using MemoryTree::hashes_; + using MemoryTree::root_; + using MemoryTree::total_size_; + std::vector leaves_; +}; + +} // namespace merkle_tree +} // namespace stdlib +} // namespace plonk \ No newline at end of file diff --git a/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.test.cpp b/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.test.cpp new file mode 100644 index 0000000000..9088647364 --- /dev/null +++ b/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.test.cpp @@ -0,0 +1,203 @@ +#include "nullifier_memory_tree.hpp" +#include + +using namespace barretenberg; +using namespace plonk::stdlib::merkle_tree; + +void print_tree(const size_t depth, std::vector hashes, std::string const& msg) +{ + info("\n", msg); + size_t offset = 0; + for (size_t i = 0; i < depth; i++) { + info("i = ", i); + size_t layer_size = (1UL << (depth - i)); + for (size_t j = 0; j < layer_size; j++) { + info("j = ", j, ": ", hashes[offset + j]); + } + offset += layer_size; + } +} + +bool check_hash_path(const fr& root, const fr_hash_path& path, const nullifier_leaf& leaf_value, const size_t idx) +{ + auto current = leaf_value.hash(); + size_t depth_ = path.size(); + size_t index = idx; + for (size_t i = 0; i < depth_; ++i) { + fr left = (index & 1) ? path[i].first : current; + fr right = (index & 1) ? current : path[i].second; + current = hash_pair_native(left, right); + index >>= 1; + } + return current == root; +} + +TEST(crypto_nullifier_tree, test_nullifier_memory) +{ + // Create a depth-3 indexed merkle tree + constexpr size_t depth = 3; + NullifierMemoryTree tree(depth); + + /** + * Intial state: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * val 0 0 0 0 0 0 0 0 + * nextIdx 0 0 0 0 0 0 0 0 + * nextVal 0 0 0 0 0 0 0 0 + */ + nullifier_leaf zero_leaf = { 0, 0, 0 }; + EXPECT_EQ(tree.get_leaves().size(), 1); + EXPECT_EQ(tree.get_leaves()[0], zero_leaf); + + /** + * Add new value 30: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * val 0 30 0 0 0 0 0 0 + * nextIdx 1 0 0 0 0 0 0 0 + * nextVal 30 0 0 0 0 0 0 0 + */ + tree.update_element(30); + EXPECT_EQ(tree.get_leaves().size(), 2); + EXPECT_EQ(tree.get_leaves()[0].hash(), nullifier_leaf({ 0, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), nullifier_leaf({ 30, 0, 0 }).hash()); + + /** + * Add new value 10: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * val 0 30 10 0 0 0 0 0 + * nextIdx 2 0 1 0 0 0 0 0 + * nextVal 10 0 30 0 0 0 0 0 + */ + tree.update_element(10); + EXPECT_EQ(tree.get_leaves().size(), 3); + EXPECT_EQ(tree.get_leaves()[0].hash(), nullifier_leaf({ 0, 2, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), nullifier_leaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), nullifier_leaf({ 10, 1, 30 }).hash()); + + /** + * Add new value 20: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * val 0 30 10 20 0 0 0 0 + * nextIdx 2 0 3 1 0 0 0 0 + * nextVal 10 0 20 30 0 0 0 0 + */ + tree.update_element(20); + EXPECT_EQ(tree.get_leaves().size(), 4); + EXPECT_EQ(tree.get_leaves()[0].hash(), nullifier_leaf({ 0, 2, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), nullifier_leaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), nullifier_leaf({ 10, 3, 20 }).hash()); + EXPECT_EQ(tree.get_leaves()[3].hash(), nullifier_leaf({ 20, 1, 30 }).hash()); + + // Adding the same value must not affect anything + tree.update_element(20); + EXPECT_EQ(tree.get_leaves().size(), 4); + EXPECT_EQ(tree.get_leaves()[0].hash(), nullifier_leaf({ 0, 2, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), nullifier_leaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), nullifier_leaf({ 10, 3, 20 }).hash()); + EXPECT_EQ(tree.get_leaves()[3].hash(), nullifier_leaf({ 20, 1, 30 }).hash()); + + /** + * Add new value 50: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * val 0 30 10 20 50 0 0 0 + * nextIdx 2 4 3 1 0 0 0 0 + * nextVal 10 50 20 30 0 0 0 0 + */ + tree.update_element(50); + EXPECT_EQ(tree.get_leaves().size(), 5); + EXPECT_EQ(tree.get_leaves()[0].hash(), nullifier_leaf({ 0, 2, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), nullifier_leaf({ 30, 4, 50 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), nullifier_leaf({ 10, 3, 20 }).hash()); + EXPECT_EQ(tree.get_leaves()[3].hash(), nullifier_leaf({ 20, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves()[4].hash(), nullifier_leaf({ 50, 0, 0 }).hash()); + + // Manually compute the node values + auto e000 = tree.get_leaves()[0].hash(); + auto e001 = tree.get_leaves()[1].hash(); + auto e010 = tree.get_leaves()[2].hash(); + auto e011 = tree.get_leaves()[3].hash(); + auto e100 = tree.get_leaves()[4].hash(); + auto e101 = nullifier_leaf({ 0, 0, 0 }).hash(); + auto e110 = nullifier_leaf({ 0, 0, 0 }).hash(); + auto e111 = nullifier_leaf({ 0, 0, 0 }).hash(); + + auto e00 = hash_pair_native(e000, e001); + auto e01 = hash_pair_native(e010, e011); + auto e10 = hash_pair_native(e100, e101); + auto e11 = hash_pair_native(e110, e111); + + auto e0 = hash_pair_native(e00, e01); + auto e1 = hash_pair_native(e10, e11); + auto root = hash_pair_native(e0, e1); + + // Check the hash path at index 2 and 3 + // Note: This merkle proof would also serve as a non-membership proof of values in (10, 20) and (20, 30) + fr_hash_path expected = { + std::make_pair(e010, e011), + std::make_pair(e00, e01), + std::make_pair(e0, e1), + }; + EXPECT_EQ(tree.get_hash_path(2), expected); + EXPECT_EQ(tree.get_hash_path(3), expected); + EXPECT_EQ(tree.root(), root); + + // Check the hash path at index 6 and 7 + expected = { + std::make_pair(e110, e111), + std::make_pair(e10, e11), + std::make_pair(e0, e1), + }; + EXPECT_EQ(tree.get_hash_path(6), expected); + EXPECT_EQ(tree.get_hash_path(7), expected); +} + +TEST(crypto_nullifier_tree, test_nullifier_tree) +{ + // Create a depth-8 indexed merkle tree + constexpr size_t depth = 8; + NullifierMemoryTree tree(depth); + + nullifier_leaf zero_leaf = { 0, 0, 0 }; + EXPECT_EQ(tree.get_leaves().size(), 1); + EXPECT_EQ(tree.get_leaves()[0].hash(), zero_leaf.hash()); + + // Add 20 random values to the tree + for (size_t i = 0; i < 20; i++) { + auto value = fr::random_element(); + tree.update_element(value); + } + + auto abs_diff = [](uint256_t a, uint256_t b) { + if (a > b) { + return (a - b); + } else { + return (b - a); + } + }; + + // Check if a new random value is not a member of this tree. + fr new_member = fr::random_element(); + const auto& leaves = tree.get_leaves(); + std::vector differences; + for (size_t i = 0; i < leaves.size(); i++) { + uint256_t diff_hi = abs_diff(uint256_t(new_member), uint256_t(leaves[i].value)); + uint256_t diff_lo = abs_diff(uint256_t(new_member), uint256_t(leaves[i].nextValue)); + differences.push_back(diff_hi + diff_lo); + } + auto it = std::min_element(differences.begin(), differences.end()); + auto index = static_cast(it - differences.begin()); + + // Merkle proof at `index` proves non-membership of `new_member` + auto hash_path = tree.get_hash_path(index); + EXPECT_TRUE(check_hash_path(tree.root(), hash_path, leaves[index], index)); +} \ No newline at end of file diff --git a/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.cpp b/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.cpp new file mode 100644 index 0000000000..49386f3f65 --- /dev/null +++ b/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.cpp @@ -0,0 +1,85 @@ +#include "nullifier_tree.hpp" +#include "../merkle_tree.hpp" +#include "../hash.hpp" +#include "../memory_store.hpp" +#include +#include +#include "barretenberg/common/net.hpp" +#include "barretenberg/numeric/bitop/count_leading_zeros.hpp" +#include "barretenberg/numeric/bitop/keep_n_lsb.hpp" +#include "barretenberg/numeric/uint128/uint128.hpp" + +namespace plonk { +namespace stdlib { +namespace merkle_tree { + +using namespace barretenberg; + +template inline bool bit_set(T const& index, size_t i) +{ + return bool((index >> i) & 0x1); +} + +template +NullifierTree::NullifierTree(Store& store, size_t depth, uint8_t tree_id) + : MerkleTree(store, depth, tree_id) +{ + ASSERT(depth_ >= 1 && depth <= 256); + zero_hashes_.resize(depth); + + // Compute the zero values at each layer. + // Insert the zero leaf to the `leaves` and also to the tree at index 0. + auto zero_leaf = nullifier_leaf{ .value = 0, .nextIndex = 0, .nextValue = 0 }; + leaves.push_back(zero_leaf); + auto current = zero_leaf.hash(); + update_element(0, current); + for (size_t i = 0; i < depth; ++i) { + zero_hashes_[i] = current; + current = hash_pair_native(current, current); + } +} + +template +NullifierTree::NullifierTree(NullifierTree&& other) + : MerkleTree(std::move(other)) +{} + +template NullifierTree::~NullifierTree() {} + +template fr NullifierTree::update_element(fr const& value) +{ + // Find the leaf with the value closest and less than `value` + size_t current; + bool is_already_present; + std::tie(current, is_already_present) = find_closest_leaf(leaves, value); + + nullifier_leaf new_leaf = { .value = value, + .nextIndex = leaves[current].nextIndex, + .nextValue = leaves[current].nextValue }; + if (!is_already_present) { + // Update the current leaf to point it to the new leaf + leaves[current].nextIndex = leaves.size(); + leaves[current].nextValue = value; + + // Insert the new leaf with (nextIndex, nextValue) of the current leaf + leaves.push_back(new_leaf); + } + + // Update the old leaf in the tree + auto old_leaf_hash = leaves[current].hash(); + index_t old_leaf_index = current; + auto r = update_element(old_leaf_index, old_leaf_hash); + + // Insert the new leaf in the tree + auto new_leaf_hash = new_leaf.hash(); + index_t new_leaf_index = is_already_present ? old_leaf_index : leaves.size() - 1; + r = update_element(new_leaf_index, new_leaf_hash); + + return r; +} + +template class NullifierTree; + +} // namespace merkle_tree +} // namespace stdlib +} // namespace plonk \ No newline at end of file diff --git a/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.hpp b/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.hpp new file mode 100644 index 0000000000..07ab4ed1b3 --- /dev/null +++ b/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.hpp @@ -0,0 +1,45 @@ +#pragma once +#include "../hash.hpp" +#include "../merkle_tree.hpp" +#include "nullifier_leaf.hpp" + +namespace plonk { +namespace stdlib { +namespace merkle_tree { + +using namespace barretenberg; + +template class NullifierTree : public MerkleTree { + public: + typedef uint256_t index_t; + + NullifierTree(Store& store, size_t depth, uint8_t tree_id = 0); + NullifierTree(NullifierTree const& other) = delete; + NullifierTree(NullifierTree&& other); + ~NullifierTree(); + + using MerkleTree::get_hash_path; + using MerkleTree::root; + using MerkleTree::size; + using MerkleTree::depth; + + fr update_element(fr const& value); + + private: + using MerkleTree::update_element; + using MerkleTree::get_element; + using MerkleTree::compute_zero_path_hash; + + private: + using MerkleTree::store_; + using MerkleTree::zero_hashes_; + using MerkleTree::depth_; + using MerkleTree::tree_id_; + std::vector leaves; +}; + +extern template class NullifierTree; + +} // namespace merkle_tree +} // namespace stdlib +} // namespace plonk \ No newline at end of file diff --git a/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.test.cpp b/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.test.cpp new file mode 100644 index 0000000000..be616c3256 --- /dev/null +++ b/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.test.cpp @@ -0,0 +1,140 @@ +#include "../memory_store.hpp" +#include "nullifier_memory_tree.hpp" +#include "nullifier_tree.hpp" +#include "barretenberg/common/streams.hpp" +#include "barretenberg/common/test.hpp" +#include "barretenberg/numeric/random/engine.hpp" + +using namespace barretenberg; +using namespace plonk::stdlib::merkle_tree; + +namespace { +auto& engine = numeric::random::get_debug_engine(); +auto& random_engine = numeric::random::get_engine(); +} // namespace + +static std::vector VALUES = []() { + std::vector values(1024); + for (size_t i = 0; i < 1024; ++i) { + values[i] = fr(random_engine.get_random_uint64()); + } + return values; +}(); + +inline void print_tree(const size_t depth, std::vector hashes, std::string const& msg) +{ + info("\n", msg); + size_t offset = 0; + for (size_t i = 0; i < depth; i++) { + info("i = ", i); + size_t layer_size = (1UL << (depth - i)); + for (size_t j = 0; j < layer_size; j++) { + info("j = ", j, ": ", hashes[offset + j]); + } + offset += layer_size; + } +} + +TEST(stdlib_nullifier_tree, test_kv_memory_vs_memory_consistency) +{ + constexpr size_t depth = 2; + NullifierMemoryTree memdb(depth); + + MemoryStore store; + NullifierTree db(store, depth); + + std::vector indicies(1 << depth); + std::iota(indicies.begin(), indicies.end(), 0); + std::random_device rd; + std::mt19937 g(rd()); + std::shuffle(indicies.begin(), indicies.end(), g); + + for (size_t i = 0; i < indicies.size() - 1; ++i) { + size_t idx = indicies[i]; + memdb.update_element(VALUES[idx]); + db.update_element(VALUES[idx]); + } + + for (size_t i = 0; i < indicies.size() - 1; ++i) { + size_t idx = indicies[i]; + EXPECT_EQ(db.get_hash_path(idx), memdb.get_hash_path(idx)); + } + + EXPECT_EQ(db.root(), memdb.root()); +} + +TEST(stdlib_nullifier_tree, test_size) +{ + MemoryStore store; + auto db = NullifierTree(store, 256); + + // We assume that the first leaf is already filled with (0, 0, 0). + EXPECT_EQ(db.size(), 1ULL); + + // Add a new non-zero leaf at index 1. + db.update_element(30); + EXPECT_EQ(db.size(), 2ULL); + + // Add third. + db.update_element(10); + EXPECT_EQ(db.size(), 3ULL); + + // Add forth. + db.update_element(20); + EXPECT_EQ(db.size(), 4ULL); + + // Add forth but with same value. + db.update_element(20); + EXPECT_EQ(db.size(), 4ULL); +} + +TEST(stdlib_nullifier_tree, test_get_hash_path) +{ + NullifierMemoryTree memdb(10); + + MemoryStore store; + auto db = NullifierTree(store, 10); + + EXPECT_EQ(memdb.get_hash_path(512), db.get_hash_path(512)); + + memdb.update_element(VALUES[512]); + db.update_element(VALUES[512]); + + EXPECT_EQ(db.get_hash_path(512), memdb.get_hash_path(512)); + + for (size_t i = 0; i < 512; ++i) { + memdb.update_element(VALUES[i]); + db.update_element(VALUES[i]); + } + + EXPECT_EQ(db.get_hash_path(512), memdb.get_hash_path(512)); +} + +TEST(stdlib_nullifier_tree, test_get_hash_path_layers) +{ + { + MemoryStore store; + auto db = NullifierTree(store, 3); + + auto before = db.get_hash_path(1); + db.update_element(VALUES[1]); + auto after = db.get_hash_path(1); + + EXPECT_NE(before[0], after[0]); + EXPECT_NE(before[1], after[1]); + EXPECT_NE(before[2], after[2]); + } + + { + MemoryStore store; + auto db = NullifierTree(store, 3); + + auto before = db.get_hash_path(7); + db.update_element(VALUES[1]); + auto after = db.get_hash_path(7); + + EXPECT_EQ(before[0], after[0]); + EXPECT_EQ(before[1], after[1]); + EXPECT_NE(before[2], after[2]); + } +} \ No newline at end of file