From ece1d455548bccd80a3c9660cc32149bcb129562 Mon Sep 17 00:00:00 2001 From: PhilWindle <60546371+PhilWindle@users.noreply.github.com> Date: Mon, 25 Nov 2024 11:47:43 +0000 Subject: [PATCH] chore: Remove handling of duplicates from the note hash tree (#10016) This PR removes the abilitity to handle duplicates in the note hash tree. Since #9492, all trees now contain unique leaves only. Removing the handling of duplicates reduces the complexity and storage requirements of the trees. --------- Co-authored-by: ludamad --- ...ontent_addressed_append_only_tree.test.cpp | 144 +------ .../lmdb_store/lmdb_transaction.cpp | 5 + .../lmdb_store/lmdb_transaction.hpp | 29 +- .../lmdb_store/lmdb_tree_store.cpp | 60 +-- .../lmdb_store/lmdb_tree_store.hpp | 70 +--- .../lmdb_store/lmdb_tree_store.test.cpp | 386 +----------------- .../lmdb_tree_write_transaction.cpp | 5 + .../lmdb_tree_write_transaction.hpp | 12 +- .../crypto/merkle_tree/lmdb_store/queries.cpp | 38 ++ .../crypto/merkle_tree/lmdb_store/queries.hpp | 28 +- .../cached_content_addressed_tree_store.hpp | 186 +++------ .../crypto/merkle_tree/test_fixtures.hpp | 31 +- .../barretenberg/crypto/merkle_tree/types.hpp | 14 +- .../world-state/src/native/message.ts | 3 - .../world-state/src/synchronizer/config.ts | 2 +- 15 files changed, 196 insertions(+), 817 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.test.cpp index cb718ff3253..52fc12daa9f 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.test.cpp @@ -475,6 +475,7 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, errors_are_caught_and_handle std::string name = random_string(); std::string directory = random_temp_directory(); std::filesystem::create_directories(directory); + auto& random_engine = numeric::get_randomness(); { LMDBTreeStore::SharedPtr db = std::make_shared(_directory, name, 50, _maxReaders); @@ -492,9 +493,10 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, errors_are_caught_and_handle // Add lots of values to the tree uint32_t num_values_to_add = 16 * 1024; - std::vector values(num_values_to_add, VALUES[0]); + std::vector values; for (uint32_t i = 0; i < num_values_to_add; i++) { - memdb.update_element(i, VALUES[0]); + values.emplace_back(random_engine.get_random_uint256()); + memdb.update_element(i, values[i]); } add_values(tree, values); @@ -714,46 +716,31 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, test_find_leaf_index) commit_tree(tree); - values = { 16, 4, 18, 22 }; + values = { 16, 4, 19, 22 }; add_values(tree, values); - // we now have duplicate leaf 18, one committed the other not - check_find_leaf_index(tree, 18, 5, true, true); - check_find_leaf_index(tree, 18, 5, true, false); - // verify the find index from api check_find_leaf_index_from(tree, 18, 0, 5, true, true); - check_find_leaf_index_from(tree, 18, 6, 10, true, true); - check_find_leaf_index_from(tree, 18, 6, 0, false, false); + check_find_leaf_index_from(tree, 19, 6, 10, true, true); + check_find_leaf_index_from(tree, 19, 0, 0, false, false); commit_tree(tree); - // add another leaf 18 - add_value(tree, 18); - - // should return the first index - check_find_leaf_index_from(tree, 18, 0, 5, true, false); - check_find_leaf_index_from(tree, 18, 0, 5, true, true); - add_value(tree, 88); - // and another uncommitted 18 - add_value(tree, 18); add_value(tree, 32); - // should return the first uncommitted - check_find_leaf_index_from(tree, 18, 12, 12, true, true); - check_find_leaf_index_from(tree, 18, 14, 14, true, true); - check_find_leaf_index_from(tree, 18, 15, 0, false, true); + check_size(tree, 14); + check_size(tree, 12, false); // look past the last instance of this leaf - check_find_leaf_index_from(tree, 18, 17, 0, false, true); + check_find_leaf_index_from(tree, 18, 6, 0, false, true); // look beyond the end of uncommitted - check_find_leaf_index_from(tree, 18, 18, 0, false, true); + check_find_leaf_index_from(tree, 18, 15, 0, false, true); // look beyond the end of committed and don't include uncomitted - check_find_leaf_index_from(tree, 18, 14, 0, false, false); + check_find_leaf_index_from(tree, 88, 13, 0, false, false); } TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_add_multiple_values) @@ -975,7 +962,7 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, test_find_historic_leaf_inde commit_tree(tree); - values = { 16, 4, 18, 22 }; + values = { 16, 4, 19, 22 }; add_values(tree, values); // should not be present at block 1 @@ -987,15 +974,15 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, test_find_historic_leaf_inde check_find_historic_leaf_index_from(tree, 1, 18, 2, 0, false, false); // at block 2 it should be check_find_historic_leaf_index_from(tree, 2, 18, 2, 5, true); - // at block 2, from index 6 it should not be found if looking only at committed - check_find_historic_leaf_index_from(tree, 2, 18, 6, 5, false, false); - // at block 2, from index 6 it should be found if looking at uncommitted too - check_find_historic_leaf_index_from(tree, 2, 18, 6, 10, true); + // at block 2, from index 6, 19 should not be found if looking only at committed + check_find_historic_leaf_index_from(tree, 2, 19, 6, 5, false, false); + // at block 2, from index 6, 19 should be found if looking at uncommitted too + check_find_historic_leaf_index_from(tree, 2, 19, 6, 10, true); commit_tree(tree); - // at block 3, from index 6 it should now be found in committed only - check_find_historic_leaf_index_from(tree, 3, 18, 6, 10, true, false); + // at block 3, from index 6, should now be found in committed only + check_find_historic_leaf_index_from(tree, 3, 19, 6, 10, true, false); } TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_be_filled) @@ -1418,79 +1405,6 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_unwind_all_blocks) test_unwind(_directory, "DB", _mapSize, _maxReaders, 10, 16, 16, 16, second); } -TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_unwind_blocks_with_duplicate_leaves) -{ - constexpr size_t depth = 4; - std::string name = random_string(); - LMDBTreeStore::SharedPtr db = std::make_shared(_directory, name, _mapSize, _maxReaders); - std::unique_ptr store = std::make_unique(name, depth, db); - ThreadPoolPtr pool = make_thread_pool(1); - TreeType tree(std::move(store), pool); - MemoryTree memdb(depth); - - constexpr size_t blockSize = 2; - constexpr size_t numBlocks = 2; - constexpr size_t numBlocksToUnwind = 1; - - std::vector values = create_values(blockSize); - - // Add the same batch of values many times - for (size_t i = 0; i < numBlocks; i++) { - for (size_t j = 0; j < values.size(); j++) { - size_t ind = i * blockSize + j; - memdb.update_element(ind, values[j]); - } - add_values(tree, values); - commit_tree(tree); - check_block_and_root_data(db, i + 1, memdb.root(), true); - - for (size_t j = 0; j < values.size(); j++) { - size_t ind = i * blockSize + j; - // query the indices db directly - check_indices_data(db, values[j], ind, true, true); - } - } - - for (size_t i = 0; i < numBlocks; i++) { - index_t startIndex = i * blockSize; - index_t expectedIndex = startIndex + 1; - - // search for the leaf from start of each batch - check_find_leaf_index_from(tree, values[1], startIndex, expectedIndex, true); - // search for the leaf from start of the next batch - check_find_leaf_index_from(tree, values[1], startIndex + 2, expectedIndex + blockSize, i < (numBlocks - 1)); - } - - const uint32_t blocksToRemove = numBlocksToUnwind; - for (uint32_t i = 0; i < blocksToRemove; i++) { - const index_t blockNumber = numBlocks - i; - unwind_block(tree, blockNumber); - - const index_t previousValidBlock = blockNumber - 1; - index_t deletedBlockStartIndex = previousValidBlock * blockSize; - - check_block_height(tree, previousValidBlock); - check_size(tree, deletedBlockStartIndex); - - for (size_t j = 0; j < numBlocks; j++) { - index_t startIndex = j * blockSize; - index_t expectedIndex = startIndex + 1; - - // search for the leaf from start of each batch - check_find_leaf_index_from(tree, values[1], startIndex, expectedIndex, j < previousValidBlock); - // search for the leaf from start of the next batch - check_find_leaf_index_from( - tree, values[1], startIndex + 2, expectedIndex + blockSize, j < (previousValidBlock - 1)); - - for (size_t k = 0; k < values.size(); k++) { - size_t ind = j * blockSize + k; - // query the indices db directly. If block number == 1 that means the entry should not be present - check_indices_data(db, values[k], ind, blockNumber > 1, ind < deletedBlockStartIndex); - } - } - } -} - TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_sync_and_unwind_large_blocks) { @@ -1534,23 +1448,15 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_advance_finalised_blocks index_t expectedFinalisedBlock = i < finalisedBlockDelay ? 0 : i - finalisedBlockDelay; check_finalised_block_height(tree, expectedFinalisedBlock); - index_t expectedPresentStart = i < finalisedBlockDelay ? 0 : (expectedFinalisedBlock * blockSize); - index_t expectedPresentEnd = ((i + 1) * blockSize) - 1; - std::vector toTest(values.begin() + static_cast(expectedPresentStart), - values.begin() + static_cast(expectedPresentEnd + 1)); - check_leaf_keys_are_present(db, expectedPresentStart, expectedPresentEnd, toTest); if (i >= finalisedBlockDelay) { index_t blockToFinalise = expectedFinalisedBlock + 1; - // attemnpting to finalise a block that doesn't exist should fail + // attempting to finalise a block that doesn't exist should fail finalise_block(tree, blockToFinalise + numBlocks, false); finalise_block(tree, blockToFinalise, true); - - index_t expectedNotPresentEnd = (blockToFinalise * blockSize) - 1; - check_leaf_keys_are_not_present(db, 0, expectedNotPresentEnd); } } } @@ -1585,12 +1491,7 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_finalise_multiple_blocks index_t blockToFinalise = 8; - check_leaf_keys_are_present(db, 0, (numBlocks * blockSize) - 1, values); - finalise_block(tree, blockToFinalise); - - index_t expectedNotPresentEnd = (blockToFinalise * blockSize) - 1; - check_leaf_keys_are_not_present(db, 0, expectedNotPresentEnd); } TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_not_finalise_block_beyond_pending_chain) @@ -1630,12 +1531,7 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_not_finalise_block_beyon // finalise the entire chain index_t blockToFinalise = numBlocks; - check_leaf_keys_are_present(db, 0, (numBlocks * blockSize) - 1, values); - finalise_block(tree, blockToFinalise); - - index_t expectedNotPresentEnd = (blockToFinalise * blockSize) - 1; - check_leaf_keys_are_not_present(db, 0, expectedNotPresentEnd); } TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_not_fork_from_unwound_blocks) diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.cpp index 3e1445ab706..b41787138eb 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.cpp @@ -34,4 +34,9 @@ bool LMDBTransaction::get_value(std::vector& key, std::vector& { return lmdb_queries::get_value(key, data, db, *this); } + +bool LMDBTransaction::get_value(std::vector& key, index_t& data, const LMDBDatabase& db) const +{ + return lmdb_queries::get_value(key, data, db, *this); +} } // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.hpp index 6ae56bd8f9f..cf2a55c1285 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.hpp @@ -2,6 +2,7 @@ #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_database.hpp" #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.hpp" #include "barretenberg/crypto/merkle_tree/lmdb_store/queries.hpp" +#include "lmdb.h" #include #include @@ -37,16 +38,18 @@ class LMDBTransaction { */ virtual void abort(); - template + template bool get_value_or_previous(T& key, - std::vector& data, + K& data, const LMDBDatabase& db, - const std::function&)>& is_valid) const; + const std::function& is_valid) const; - template bool get_value_or_previous(T& key, std::vector& data, const LMDBDatabase& db) const; + template bool get_value_or_previous(T& key, K& data, const LMDBDatabase& db) const; template bool get_value(T& key, std::vector& data, const LMDBDatabase& db) const; + template bool get_value(T& key, index_t& data, const LMDBDatabase& db) const; + template void get_all_values_greater_or_equal_key(const T& key, std::vector>& data, @@ -59,6 +62,8 @@ class LMDBTransaction { bool get_value(std::vector& key, std::vector& data, const LMDBDatabase& db) const; + bool get_value(std::vector& key, index_t& data, const LMDBDatabase& db) const; + protected: std::shared_ptr _environment; MDB_txn* _transaction; @@ -71,17 +76,23 @@ template bool LMDBTransaction::get_value(T& key, std::vector -bool LMDBTransaction::get_value_or_previous(T& key, std::vector& data, const LMDBDatabase& db) const +template bool LMDBTransaction::get_value(T& key, index_t& data, const LMDBDatabase& db) const +{ + std::vector keyBuffer = serialise_key(key); + return get_value(keyBuffer, data, db); +} + +template +bool LMDBTransaction::get_value_or_previous(T& key, K& data, const LMDBDatabase& db) const { return lmdb_queries::get_value_or_previous(key, data, db, *this); } -template +template bool LMDBTransaction::get_value_or_previous(T& key, - std::vector& data, + K& data, const LMDBDatabase& db, - const std::function&)>& is_valid) const + const std::function& is_valid) const { return lmdb_queries::get_value_or_previous(key, data, db, is_valid, *this); } diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.cpp index 4599df682c8..3f4f07aa829 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.cpp @@ -69,7 +69,7 @@ LMDBTreeStore::LMDBTreeStore(std::string directory, std::string name, uint64_t m { LMDBDatabaseCreationTransaction tx(_environment); - _leafValueToIndexDatabase = std::make_unique( + _leafKeyToIndexDatabase = std::make_unique( _environment, tx, _name + std::string("leaf indices"), false, false, fr_key_cmp); tx.commit(); } @@ -80,13 +80,6 @@ LMDBTreeStore::LMDBTreeStore(std::string directory, std::string name, uint64_t m _environment, tx, _name + std::string("leaf pre-images"), false, false, fr_key_cmp); tx.commit(); } - - { - LMDBDatabaseCreationTransaction tx(_environment); - _leafIndexToKeyDatabase = std::make_unique( - _environment, tx, _name + std::string("leaf keys"), false, false, index_key_cmp); - tx.commit(); - } } LMDBTreeStore::WriteTransaction::Ptr LMDBTreeStore::create_write_transaction() const @@ -110,12 +103,10 @@ void LMDBTreeStore::get_stats(TreeDBStats& stats, ReadTransaction& tx) stats.blocksDBStats = DBStats(BLOCKS_DB, stat); call_lmdb_func(mdb_stat, tx.underlying(), _leafHashToPreImageDatabase->underlying(), &stat); stats.leafPreimagesDBStats = DBStats(LEAF_PREIMAGES_DB, stat); - call_lmdb_func(mdb_stat, tx.underlying(), _leafValueToIndexDatabase->underlying(), &stat); + call_lmdb_func(mdb_stat, tx.underlying(), _leafKeyToIndexDatabase->underlying(), &stat); stats.leafIndicesDBStats = DBStats(LEAF_INDICES_DB, stat); call_lmdb_func(mdb_stat, tx.underlying(), _nodeDatabase->underlying(), &stat); stats.nodesDBStats = DBStats(NODES_DB, stat); - call_lmdb_func(mdb_stat, tx.underlying(), _leafIndexToKeyDatabase->underlying(), &stat); - stats.leafKeysDBStats = DBStats(LEAF_KEYS_DB, stat); } void LMDBTreeStore::write_block_data(uint64_t blockNumber, @@ -166,21 +157,18 @@ bool LMDBTreeStore::read_meta_data(TreeMeta& metaData, LMDBTreeStore::ReadTransa return success; } -void LMDBTreeStore::write_leaf_indices(const fr& leafValue, const Indices& indices, LMDBTreeStore::WriteTransaction& tx) +void LMDBTreeStore::write_leaf_index(const fr& leafValue, const index_t& index, LMDBTreeStore::WriteTransaction& tx) { - msgpack::sbuffer buffer; - msgpack::pack(buffer, indices); - std::vector encoded(buffer.data(), buffer.data() + buffer.size()); FrKeyType key(leafValue); // std::cout << "Writing leaf indices by key " << key << std::endl; - tx.put_value(key, encoded, *_leafValueToIndexDatabase); + tx.put_value(key, index, *_leafKeyToIndexDatabase); } -void LMDBTreeStore::delete_leaf_indices(const fr& leafValue, LMDBTreeStore::WriteTransaction& tx) +void LMDBTreeStore::delete_leaf_index(const fr& leafValue, LMDBTreeStore::WriteTransaction& tx) { FrKeyType key(leafValue); // std::cout << "Deleting leaf indices by key " << key << std::endl; - tx.delete_value(key, *_leafValueToIndexDatabase); + tx.delete_value(key, *_leafKeyToIndexDatabase); } void LMDBTreeStore::increment_node_reference_count(const fr& nodeHash, WriteTransaction& tx) @@ -230,46 +218,24 @@ void LMDBTreeStore::delete_leaf_by_hash(const fr& leafHash, WriteTransaction& tx } fr LMDBTreeStore::find_low_leaf(const fr& leafValue, - Indices& indices, + index_t& index, const std::optional& sizeLimit, ReadTransaction& tx) { - std::vector data; FrKeyType key(leafValue); - auto is_valid = [&](const std::vector& data) { - Indices tmp; - msgpack::unpack((const char*)data.data(), data.size()).get().convert(tmp); - return tmp.indices[0] < sizeLimit.value(); + auto is_valid = [&](const MDB_val& data) { + index_t tmp = 0; + deserialise_key(data.mv_data, tmp); + return tmp < sizeLimit.value(); }; if (!sizeLimit.has_value()) { - tx.get_value_or_previous(key, data, *_leafValueToIndexDatabase); - msgpack::unpack((const char*)data.data(), data.size()).get().convert(indices); + tx.get_value_or_previous(key, index, *_leafKeyToIndexDatabase); } else { - tx.get_value_or_previous(key, data, *_leafValueToIndexDatabase, is_valid); - msgpack::unpack((const char*)data.data(), data.size()).get().convert(indices); + tx.get_value_or_previous(key, index, *_leafKeyToIndexDatabase, is_valid); } return key; } -void LMDBTreeStore::write_leaf_key_by_index(const fr& leafKey, const index_t& index, WriteTransaction& tx) -{ - std::vector data = to_buffer(leafKey); - LeafIndexKeyType key(index); - tx.put_value(key, data, *_leafIndexToKeyDatabase); -} - -void LMDBTreeStore::delete_all_leaf_keys_after_or_equal_index(const index_t& index, WriteTransaction& tx) -{ - LeafIndexKeyType key(index); - tx.delete_all_values_greater_or_equal_key(key, *_leafIndexToKeyDatabase); -} - -void LMDBTreeStore::delete_all_leaf_keys_before_or_equal_index(const index_t& index, WriteTransaction& tx) -{ - LeafIndexKeyType key(index); - tx.delete_all_values_lesser_or_equal_key(key, *_leafIndexToKeyDatabase); -} - bool LMDBTreeStore::read_node(const fr& nodeHash, NodePayload& nodeData, ReadTransaction& tx) { FrKeyType key(nodeHash); diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp index 760a948dd6f..ab4dfb7316c 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp @@ -43,14 +43,6 @@ inline std::ostream& operator<<(std::ostream& os, const BlockPayload& block) return os; } -struct Indices { - std::vector indices; - - MSGPACK_FIELDS(indices); - - bool operator==(const Indices& other) const { return indices == other.indices; } -}; - struct NodePayload { std::optional left; std::optional right; @@ -96,16 +88,13 @@ class LMDBTreeStore { bool read_meta_data(TreeMeta& metaData, ReadTransaction& tx); - template bool read_leaf_indices(const fr& leafValue, Indices& indices, TxType& tx); + template bool read_leaf_index(const fr& leafValue, index_t& leafIndex, TxType& tx); - fr find_low_leaf(const fr& leafValue, - Indices& indices, - const std::optional& sizeLimit, - ReadTransaction& tx); + fr find_low_leaf(const fr& leafValue, index_t& index, const std::optional& sizeLimit, ReadTransaction& tx); - void write_leaf_indices(const fr& leafValue, const Indices& indices, WriteTransaction& tx); + void write_leaf_index(const fr& leafValue, const index_t& leafIndex, WriteTransaction& tx); - void delete_leaf_indices(const fr& leafValue, WriteTransaction& tx); + void delete_leaf_index(const fr& leafValue, WriteTransaction& tx); bool read_node(const fr& nodeHash, NodePayload& nodeData, ReadTransaction& tx); @@ -145,22 +134,16 @@ class LMDBTreeStore { LMDBEnvironment::SharedPtr _environment; LMDBDatabase::Ptr _blockDatabase; LMDBDatabase::Ptr _nodeDatabase; - LMDBDatabase::Ptr _leafValueToIndexDatabase; + LMDBDatabase::Ptr _leafKeyToIndexDatabase; LMDBDatabase::Ptr _leafHashToPreImageDatabase; - LMDBDatabase::Ptr _leafIndexToKeyDatabase; template bool get_node_data(const fr& nodeHash, NodePayload& nodeData, TxType& tx); }; -template bool LMDBTreeStore::read_leaf_indices(const fr& leafValue, Indices& indices, TxType& tx) +template bool LMDBTreeStore::read_leaf_index(const fr& leafValue, index_t& leafIndex, TxType& tx) { FrKeyType key(leafValue); - std::vector data; - bool success = tx.template get_value(key, data, *_leafValueToIndexDatabase); - if (success) { - msgpack::unpack((const char*)data.data(), data.size()).get().convert(indices); - } - return success; + return tx.template get_value(key, leafIndex, *_leafKeyToIndexDatabase); } template @@ -195,43 +178,4 @@ template bool LMDBTreeStore::get_node_data(const fr& nodeHash, } return success; } - -template bool LMDBTreeStore::read_leaf_key_by_index(const index_t& index, fr& leafKey, TxType& tx) -{ - LeafIndexKeyType key(index); - std::vector data; - bool success = tx.template get_value(key, data, *_leafIndexToKeyDatabase); - if (success) { - leafKey = from_buffer(data); - } - return success; -} - -template -void LMDBTreeStore::read_all_leaf_keys_after_or_equal_index(const index_t& index, - std::vector& leafKeys, - TxType& tx) -{ - LeafIndexKeyType key(index); - std::vector> values; - tx.get_all_values_greater_or_equal_key(key, values, *_leafIndexToKeyDatabase); - for (const auto& value : values) { - fr leafKey = from_buffer(value); - leafKeys.push_back(leafKey); - } -} - -template -void LMDBTreeStore::read_all_leaf_keys_before_or_equal_index(const index_t& index, - std::vector& leafKeys, - TxType& tx) -{ - LeafIndexKeyType key(index); - std::vector> values; - tx.get_all_values_lesser_or_equal_key(key, values, *_leafIndexToKeyDatabase); - for (const auto& value : values) { - fr leafKey = from_buffer(value); - leafKeys.push_back(leafKey); - } -} } // namespace bb::crypto::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.test.cpp index 68d7f66faf4..b905aafe1a3 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.test.cpp @@ -184,25 +184,23 @@ TEST_F(LMDBTreeStoreTest, can_serde_64bit_values) TEST_F(LMDBTreeStoreTest, can_write_and_read_leaf_indices) { - Indices indices; - indices.indices.push_back(47); - indices.indices.push_back(86); + index_t index = 47; bb::fr key = VALUES[5]; LMDBTreeStore store(_directory, "DB1", _mapSize, _maxReaders); { LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); - store.write_leaf_indices(key, indices, *transaction); + store.write_leaf_index(key, index, *transaction); transaction->commit(); } { LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); - Indices readBack; - bool success = store.read_leaf_indices(key, readBack, *transaction); + index_t readBack = 0; + bool success = store.read_leaf_index(key, readBack, *transaction); EXPECT_TRUE(success); - EXPECT_EQ(readBack, indices); + EXPECT_EQ(readBack, index); - success = store.read_leaf_indices(VALUES[6], readBack, *transaction); + success = store.read_leaf_index(VALUES[6], readBack, *transaction); EXPECT_FALSE(success); } } @@ -256,374 +254,4 @@ TEST_F(LMDBTreeStoreTest, can_write_and_read_leaves_by_hash) success = store.read_leaf_by_hash(VALUES[9], readBack, *transaction); EXPECT_FALSE(success); } -} - -TEST_F(LMDBTreeStoreTest, can_read_write_key_by_index) -{ - bb::fr leafKey = VALUES[0]; - index_t leafIndex = 45; - LMDBTreeStore store(_directory, "DB1", _mapSize, _maxReaders); - { - LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); - store.write_leaf_key_by_index(leafKey, leafIndex, *transaction); - transaction->commit(); - } - - { - LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); - bb::fr readBack; - bool success = store.read_leaf_key_by_index(leafIndex, readBack, *transaction); - EXPECT_TRUE(success); - EXPECT_EQ(readBack, leafKey); - - success = store.read_leaf_key_by_index(leafIndex + 1, readBack, *transaction); - EXPECT_FALSE(success); - } -} - -TEST_F(LMDBTreeStoreTest, can_retrieve_all_keys_greater_than_index) -{ - std::vector values = create_values(1024); - index_t leafIndexStart = 45; - LMDBTreeStore store(_directory, "DB1", _mapSize, _maxReaders); - { - LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); - for (uint32_t i = 0; i < values.size(); i++) { - store.write_leaf_key_by_index(values[i], i + leafIndexStart, *transaction); - } - transaction->commit(); - } - - { - LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); - // Retrieve all but the first 150 keys - uint32_t offset = 150; - std::vector leafKeys; - store.read_all_leaf_keys_after_or_equal_index(leafIndexStart + offset, leafKeys, *transaction); - EXPECT_EQ(leafKeys.size(), values.size() - offset); - for (uint32_t i = offset; i < leafKeys.size(); i++) { - EXPECT_EQ(leafKeys[i], values[i + offset]); - } - } - - { - LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); - // Retrieve all keys - uint32_t offset = 0; - std::vector leafKeys; - store.read_all_leaf_keys_after_or_equal_index(leafIndexStart + offset, leafKeys, *transaction); - EXPECT_EQ(leafKeys.size(), values.size() - offset); - for (uint32_t i = offset; i < leafKeys.size(); i++) { - EXPECT_EQ(leafKeys[i], values[i + offset]); - } - } - - { - LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); - // Retrieve no keys - uint32_t offset = 10000; - std::vector leafKeys; - store.read_all_leaf_keys_after_or_equal_index(leafIndexStart + offset, leafKeys, *transaction); - EXPECT_EQ(leafKeys.size(), 0); - } -} - -TEST_F(LMDBTreeStoreTest, can_delete_all_keys_greater_than_index) -{ - std::vector values = create_values(1024); - index_t leafIndexStart = 45; - uint32_t deleteFromIndex = 150; - LMDBTreeStore store(_directory, "DB1", _mapSize, _maxReaders); - { - LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); - for (uint32_t i = 0; i < values.size(); i++) { - store.write_leaf_key_by_index(values[i], i + leafIndexStart, *transaction); - } - transaction->commit(); - } - - { - LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); - store.delete_all_leaf_keys_after_or_equal_index(deleteFromIndex, *transaction); - transaction->commit(); - } - - { - LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); - std::vector leafKeys; - store.read_all_leaf_keys_after_or_equal_index(leafIndexStart, leafKeys, *transaction); - EXPECT_EQ(leafKeys.size(), deleteFromIndex - leafIndexStart); - for (uint32_t i = 0; i < leafKeys.size(); i++) { - EXPECT_EQ(leafKeys[i], values[i]); - } - } - - { - LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); - for (uint32_t i = 0; i < 1024 + leafIndexStart; i++) { - bb::fr leafKey; - bool success = store.read_leaf_key_by_index(i, leafKey, *transaction); - EXPECT_EQ(success, (i >= leafIndexStart && (i < deleteFromIndex))); - if (success) { - EXPECT_EQ(leafKey, values[i - leafIndexStart]); - } - } - } -} - -TEST_F(LMDBTreeStoreTest, can_delete_all_keys_less_than_index) -{ - std::vector values = create_values(1024); - index_t leafIndexStart = 45; - uint32_t deleteFromIndex = 150; - LMDBTreeStore store(_directory, "DB1", _mapSize, _maxReaders); - { - LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); - for (uint32_t i = 0; i < values.size(); i++) { - store.write_leaf_key_by_index(values[i], i + leafIndexStart, *transaction); - } - transaction->commit(); - } - - { - LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); - store.delete_all_leaf_keys_before_or_equal_index(deleteFromIndex, *transaction); - transaction->commit(); - } - - { - LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); - std::vector leafKeys; - store.read_all_leaf_keys_before_or_equal_index(leafIndexStart + 1023, leafKeys, *transaction); - EXPECT_EQ(leafKeys.size(), 1024 - (deleteFromIndex - leafIndexStart + 1)); - for (uint32_t i = 0; i < leafKeys.size(); i++) { - EXPECT_EQ(leafKeys[i], values[1023 - i]); - } - } - - { - LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); - for (uint32_t i = 0; i < 1024 + leafIndexStart; i++) { - bb::fr leafKey; - bool success = store.read_leaf_key_by_index(i, leafKey, *transaction); - EXPECT_EQ(success, (i > deleteFromIndex && (i <= leafIndexStart + 1023))); - if (success) { - EXPECT_EQ(leafKey, values[i - leafIndexStart]); - } - } - } -} - -TEST_F(LMDBTreeStoreTest, can_delete_all_keys_greater_than) -{ - std::vector values = create_values(1024); - index_t leafIndexStart = 45; - uint32_t deleteFromIndex = 0; - LMDBTreeStore store(_directory, "DB1", _mapSize, _maxReaders); - { - LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); - for (uint32_t i = 0; i < values.size(); i++) { - store.write_leaf_key_by_index(values[i], i + leafIndexStart, *transaction); - } - transaction->commit(); - } - - { - LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); - store.delete_all_leaf_keys_after_or_equal_index(deleteFromIndex, *transaction); - transaction->commit(); - } - - { - LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); - std::vector leafKeys; - store.read_all_leaf_keys_after_or_equal_index(leafIndexStart, leafKeys, *transaction); - EXPECT_EQ(leafKeys.size(), 0); - } - - { - LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); - std::vector leafKeys; - store.read_all_leaf_keys_after_or_equal_index(0, leafKeys, *transaction); - EXPECT_EQ(leafKeys.size(), 0); - } - { - LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); - std::vector leafKeys; - store.read_all_leaf_keys_after_or_equal_index(10000, leafKeys, *transaction); - EXPECT_EQ(leafKeys.size(), 0); - } -} - -TEST_F(LMDBTreeStoreTest, can_delete_all_keys_less_than) -{ - std::vector values = create_values(1024); - index_t leafIndexStart = 45; - uint32_t deleteFromIndex = 2000; - LMDBTreeStore store(_directory, "DB1", _mapSize, _maxReaders); - { - LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); - for (uint32_t i = 0; i < values.size(); i++) { - store.write_leaf_key_by_index(values[i], i + leafIndexStart, *transaction); - } - transaction->commit(); - } - - { - LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); - store.delete_all_leaf_keys_before_or_equal_index(deleteFromIndex, *transaction); - transaction->commit(); - } - - { - LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); - std::vector leafKeys; - store.read_all_leaf_keys_before_or_equal_index(leafIndexStart + 1023, leafKeys, *transaction); - EXPECT_EQ(leafKeys.size(), 0); - } - - { - LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); - std::vector leafKeys; - store.read_all_leaf_keys_before_or_equal_index(2000, leafKeys, *transaction); - EXPECT_EQ(leafKeys.size(), 0); - } - { - LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); - std::vector leafKeys; - store.read_all_leaf_keys_before_or_equal_index(10, leafKeys, *transaction); - EXPECT_EQ(leafKeys.size(), 0); - } -} - -TEST_F(LMDBTreeStoreTest, can_delete_no_keys_greater_than) -{ - std::vector values = create_values(1024); - index_t leafIndexStart = 45; - uint32_t deleteFromIndex = 2000; - LMDBTreeStore store(_directory, "DB1", _mapSize, _maxReaders); - { - LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); - for (uint32_t i = 0; i < values.size(); i++) { - store.write_leaf_key_by_index(values[i], i + leafIndexStart, *transaction); - } - transaction->commit(); - } - - { - LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); - store.delete_all_leaf_keys_after_or_equal_index(deleteFromIndex, *transaction); - transaction->commit(); - } - - { - LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); - std::vector leafKeys; - store.read_all_leaf_keys_after_or_equal_index(leafIndexStart, leafKeys, *transaction); - EXPECT_EQ(leafKeys.size(), 1024); - for (uint32_t i = 0; i < leafKeys.size(); i++) { - EXPECT_EQ(leafKeys[i], values[i]); - } - } - - { - LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); - std::vector leafKeys; - store.read_all_leaf_keys_after_or_equal_index(0, leafKeys, *transaction); - EXPECT_EQ(leafKeys.size(), 1024); - for (uint32_t i = 0; i < leafKeys.size(); i++) { - EXPECT_EQ(leafKeys[i], values[i]); - } - } - { - LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); - std::vector leafKeys; - store.read_all_leaf_keys_after_or_equal_index(10000, leafKeys, *transaction); - EXPECT_EQ(leafKeys.size(), 0); - } -} - -TEST_F(LMDBTreeStoreTest, can_delete_no_keys_less_than) -{ - std::vector values = create_values(1024); - index_t leafIndexStart = 45; - uint32_t deleteFromIndex = 20; - LMDBTreeStore store(_directory, "DB1", _mapSize, _maxReaders); - { - LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); - for (uint32_t i = 0; i < values.size(); i++) { - store.write_leaf_key_by_index(values[i], i + leafIndexStart, *transaction); - } - transaction->commit(); - } - - { - LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); - store.delete_all_leaf_keys_before_or_equal_index(deleteFromIndex, *transaction); - transaction->commit(); - } - - { - LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); - std::vector leafKeys; - store.read_all_leaf_keys_before_or_equal_index(leafIndexStart + 1023, leafKeys, *transaction); - EXPECT_EQ(leafKeys.size(), 1024); - for (uint32_t i = 0; i < leafKeys.size(); i++) { - EXPECT_EQ(leafKeys[i], values[1023 - i]); - } - } - - { - LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); - std::vector leafKeys; - store.read_all_leaf_keys_before_or_equal_index(2000, leafKeys, *transaction); - EXPECT_EQ(leafKeys.size(), 1024); - for (uint32_t i = 0; i < leafKeys.size(); i++) { - EXPECT_EQ(leafKeys[i], values[1023 - i]); - } - } - { - LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); - std::vector leafKeys; - store.read_all_leaf_keys_before_or_equal_index(10, leafKeys, *transaction); - EXPECT_EQ(leafKeys.size(), 0); - } -} - -TEST_F(LMDBTreeStoreTest, can_retrieve_all_keys_when_none_are_present) -{ - std::vector values = create_values(1024); - LMDBTreeStore store(_directory, "DB1", _mapSize, _maxReaders); - - { - LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); - std::vector leafKeys; - store.read_all_leaf_keys_after_or_equal_index(0, leafKeys, *transaction); - EXPECT_EQ(leafKeys.size(), 0); - } - - { - LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); - std::vector leafKeys; - store.read_all_leaf_keys_before_or_equal_index(0, leafKeys, *transaction); - EXPECT_EQ(leafKeys.size(), 0); - } -} - -TEST_F(LMDBTreeStoreTest, can_delete_all_keys_when_none_are_present) -{ - std::vector values = create_values(1024); - LMDBTreeStore store(_directory, "DB1", _mapSize, _maxReaders); - - { - LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); - store.delete_all_leaf_keys_after_or_equal_index(0, *transaction); - transaction->commit(); - } - - { - LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); - store.delete_all_leaf_keys_before_or_equal_index(0, *transaction); - transaction->commit(); - } -} +} \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_write_transaction.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_write_transaction.cpp index 4b4cd846a2f..5e524ca2fff 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_write_transaction.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_write_transaction.cpp @@ -42,6 +42,11 @@ void LMDBTreeWriteTransaction::put_value(std::vector& key, std::vector< lmdb_queries::put_value(key, data, db, *this); } +void LMDBTreeWriteTransaction::put_value(std::vector& key, const index_t& data, const LMDBDatabase& db) +{ + lmdb_queries::put_value(key, data, db, *this); +} + void LMDBTreeWriteTransaction::delete_value(std::vector& key, const LMDBDatabase& db) { lmdb_queries::delete_value(key, db, *this); diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_write_transaction.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_write_transaction.hpp index d12d5fdc3ad..927e14fb4fa 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_write_transaction.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_write_transaction.hpp @@ -32,8 +32,12 @@ class LMDBTreeWriteTransaction : public LMDBTransaction { template void put_value(T& key, std::vector& data, const LMDBDatabase& db); + template void put_value(T& key, const index_t& data, const LMDBDatabase& db); + void put_value(std::vector& key, std::vector& data, const LMDBDatabase& db); + void put_value(std::vector& key, const index_t& data, const LMDBDatabase& db); + template void delete_value(T& key, const LMDBDatabase& db); void delete_value(std::vector& key, const LMDBDatabase& db); @@ -51,7 +55,13 @@ template void LMDBTreeWriteTransaction::put_value(T& key, std::vector& data, const LMDBDatabase& db) { std::vector keyBuffer = serialise_key(key); - lmdb_queries::put_value(keyBuffer, data, db, *this); + put_value(keyBuffer, data, db); +} + +template void LMDBTreeWriteTransaction::put_value(T& key, const index_t& data, const LMDBDatabase& db) +{ + std::vector keyBuffer = serialise_key(key); + put_value(keyBuffer, data, db); } template void LMDBTreeWriteTransaction::delete_value(T& key, const LMDBDatabase& db) diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/queries.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/queries.cpp index 311b7484d45..939cd58dde1 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/queries.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/queries.cpp @@ -1,5 +1,7 @@ #include "barretenberg/crypto/merkle_tree/lmdb_store/queries.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/callbacks.hpp" #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_write_transaction.hpp" +#include namespace bb::crypto::merkle_tree::lmdb_queries { @@ -18,6 +20,24 @@ void put_value(std::vector& key, call_lmdb_func("mdb_put", mdb_put, tx.underlying(), db.underlying(), &dbKey, &dbVal, 0U); } +void put_value(std::vector& key, + const index_t& data, + const LMDBDatabase& db, + bb::crypto::merkle_tree::LMDBTreeWriteTransaction& tx) +{ + MDB_val dbKey; + dbKey.mv_size = key.size(); + dbKey.mv_data = (void*)key.data(); + + // use the serialise key method for serialising the index + std::vector serialised = serialise_key(data); + + MDB_val dbVal; + dbVal.mv_size = serialised.size(); + dbVal.mv_data = (void*)serialised.data(); + call_lmdb_func("mdb_put", mdb_put, tx.underlying(), db.underlying(), &dbKey, &dbVal, 0U); +} + void delete_value(std::vector& key, const LMDBDatabase& db, bb::crypto::merkle_tree::LMDBTreeWriteTransaction& tx) @@ -49,4 +69,22 @@ bool get_value(std::vector& key, copy_to_vector(dbVal, data); return true; } + +bool get_value(std::vector& key, + index_t& data, + const LMDBDatabase& db, + const bb::crypto::merkle_tree::LMDBTransaction& tx) +{ + MDB_val dbKey; + dbKey.mv_size = key.size(); + dbKey.mv_data = (void*)key.data(); + + MDB_val dbVal; + if (!call_lmdb_func(mdb_get, tx.underlying(), db.underlying(), &dbKey, &dbVal)) { + return false; + } + // use the deserialise key method for deserialising the index + deserialise_key(dbVal.mv_data, data); + return true; +} } // namespace bb::crypto::merkle_tree::lmdb_queries \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/queries.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/queries.hpp index 3269dc13952..aa97a2d2518 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/queries.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/queries.hpp @@ -14,8 +14,8 @@ class LMDBTreeWriteTransaction; namespace lmdb_queries { -template -bool get_value_or_previous(TKey& key, std::vector& data, const LMDBDatabase& db, const TxType& tx) +template +bool get_value_or_previous(TKey& key, TValue& data, const LMDBDatabase& db, const TxType& tx) { std::vector keyBuffer = serialise_key(key); uint32_t keySize = static_cast(keyBuffer.size()); @@ -36,7 +36,7 @@ bool get_value_or_previous(TKey& key, std::vector& data, const LMDBData std::vector temp = mdb_val_to_vector(dbKey); if (keyBuffer == temp) { // we have the exact key - copy_to_vector(dbVal, data); + deserialise_key(dbVal.mv_data, data); success = true; } else { // We have a key of the same size but larger value OR a larger size @@ -48,7 +48,7 @@ bool get_value_or_previous(TKey& key, std::vector& data, const LMDBData if (dbKey.mv_size != keySize) { // There is no previous key, do nothing } else { - copy_to_vector(dbVal, data); + deserialise_key(dbVal.mv_data, data); deserialise_key(dbKey.mv_data, key); success = true; } @@ -66,7 +66,7 @@ bool get_value_or_previous(TKey& key, std::vector& data, const LMDBData if (dbKey.mv_size != keySize) { // The key is not the same size, same as not found, do nothing } else { - copy_to_vector(dbVal, data); + deserialise_key(dbVal.mv_data, data); deserialise_key(dbKey.mv_data, key); success = true; } @@ -86,11 +86,11 @@ bool get_value_or_previous(TKey& key, std::vector& data, const LMDBData return success; } -template +template bool get_value_or_previous(TKey& key, - std::vector& data, + TValue& data, const LMDBDatabase& db, - const std::function&)>& is_valid, + const std::function& is_valid, const TxType& tx) { std::vector keyBuffer = serialise_key(key); @@ -114,8 +114,8 @@ bool get_value_or_previous(TKey& key, std::vector temp = mdb_val_to_vector(dbKey); if (keyBuffer == temp || lower) { // We have the exact key, we need to determine if it is valid - copy_to_vector(dbVal, data); - if (is_valid(data)) { + if (is_valid(dbVal)) { + deserialise_key(dbVal.mv_data, data); deserialise_key(dbKey.mv_data, key); success = true; // It's valid @@ -151,8 +151,8 @@ bool get_value_or_previous(TKey& key, // The key is not the same size, same as not found, exit break; } - copy_to_vector(dbVal, data); - if (is_valid(data)) { + if (is_valid(dbVal)) { + deserialise_key(dbVal.mv_data, data); deserialise_key(dbKey.mv_data, key); success = true; // It's valid @@ -406,11 +406,15 @@ void put_value(std::vector& key, const LMDBDatabase& db, LMDBTreeWriteTransaction& tx); +void put_value(std::vector& key, const index_t& data, const LMDBDatabase& db, LMDBTreeWriteTransaction& tx); + void delete_value(std::vector& key, const LMDBDatabase& db, LMDBTreeWriteTransaction& tx); bool get_value(std::vector& key, std::vector& data, const LMDBDatabase& db, const LMDBTransaction& tx); + +bool get_value(std::vector& key, index_t& data, const LMDBDatabase& db, const LMDBTransaction& tx); } // namespace lmdb_queries } // namespace bb::crypto::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp index 72c4ec34cf3..85e5cef2b40 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp @@ -217,9 +217,8 @@ template class ContentAddressedCachedTreeStore { std::unordered_map nodes_; // This is a store mapping the leaf key (e.g. slot for public data or nullifier value for nullifier tree) to the - // indices in the tree For indexed tress there is only ever one index against the key, for append-only trees there - // can be multiple - std::map indices_; + // index in the tree + std::map indices_; // This is a mapping from leaf hash to leaf pre-image. This will contain entries that need to be omitted when // commiting updates @@ -242,12 +241,8 @@ template class ContentAddressedCachedTreeStore { void persist_meta(TreeMeta& m, WriteTransaction& tx); - void hydrate_indices_from_persisted_store(ReadTransaction& tx); - void persist_leaf_indices(WriteTransaction& tx); - void persist_leaf_keys(const index_t& startIndex, WriteTransaction& tx); - void persist_leaf_pre_image(const fr& hash, WriteTransaction& tx); void persist_node(const std::optional& optional_hash, uint32_t level, WriteTransaction& tx); @@ -259,9 +254,7 @@ template class ContentAddressedCachedTreeStore { void remove_leaf(const fr& hash, const std::optional& maxIndex, WriteTransaction& tx); - void remove_leaf_indices(const fr& key, const index_t& maxIndex, WriteTransaction& tx); - - void remove_leaf_indices_after_or_equal_index(const index_t& maxIndex, WriteTransaction& tx); + void remove_leaf_index(const fr& key, const index_t& maxIndex, WriteTransaction& tx); void extract_db_stats(TreeDBStats& stats); @@ -316,14 +309,14 @@ std::pair ContentAddressedCachedTreeStore::find_lo const fr& new_leaf_key, const RequestContext& requestContext, ReadTransaction& tx) const { auto new_value_as_number = uint256_t(new_leaf_key); - Indices committed; + index_t committed = 0; std::optional sizeLimit = std::nullopt; if (initialised_from_block_.has_value() || requestContext.blockNumber.has_value()) { sizeLimit = constrain_tree_size(requestContext, tx); } fr found_key = dataStore_->find_low_leaf(new_leaf_key, committed, sizeLimit, tx); - auto db_index = committed.indices[0]; + index_t db_index = committed; uint256_t retrieved_value = found_key; // Accessing indices_ from here under a lock @@ -340,12 +333,12 @@ std::pair ContentAddressedCachedTreeStore::find_lo --it; // we need to return the larger of the db value or the cached value - return std::make_pair(false, it->first > retrieved_value ? it->second.indices[0] : db_index); + return std::make_pair(false, it->first > retrieved_value ? it->second : db_index); } if (it->first == uint256_t(new_value_as_number)) { // the value is already present and the iterator points to it - return std::make_pair(true, it->second.indices[0]); + return std::make_pair(true, it->second); } // the iterator points to the element immediately larger than the requested value // We need to return the highest value from @@ -357,7 +350,7 @@ std::pair ContentAddressedCachedTreeStore::find_lo } --it; // it now points to the value less than that requested - return std::make_pair(false, it->first > retrieved_value ? it->second.indices[0] : db_index); + return std::make_pair(false, it->first > retrieved_value ? it->second : db_index); } template @@ -429,14 +422,7 @@ void ContentAddressedCachedTreeStore::update_index(const index_t& // std::cout << "update_index at index " << index << " leaf " << leaf << std::endl; // Accessing indices_ under a lock std::unique_lock lock(mtx_); - auto it = indices_.find(uint256_t(leaf)); - if (it == indices_.end()) { - Indices ind; - ind.indices.push_back(index); - indices_[uint256_t(leaf)] = ind; - return; - } - it->second.indices.push_back(index); + indices_.insert({ uint256_t(leaf), index }); } template @@ -454,47 +440,35 @@ std::optional ContentAddressedCachedTreeStore::find_leaf ReadTransaction& tx, bool includeUncommitted) const { - Indices committed; - std::optional result = std::nullopt; - FrKeyType key = leaf; - std::vector value; - bool success = dataStore_->read_leaf_indices(key, committed, tx); - if (success) { - index_t sizeLimit = constrain_tree_size(requestContext, tx); - if (!committed.indices.empty()) { - for (index_t ind : committed.indices) { - if (ind < start_index) { - continue; - } - if (ind >= sizeLimit) { - continue; - } - if (!result.has_value()) { - result = ind; - continue; - } - result = std::min(ind, result.value()); - } - } - } if (includeUncommitted) { // Accessing indices_ under a lock std::unique_lock lock(mtx_); auto it = indices_.find(uint256_t(leaf)); - if (it != indices_.end() && !it->second.indices.empty()) { - for (index_t ind : it->second.indices) { - if (ind < start_index) { - continue; - } - if (!result.has_value()) { - result = ind; - continue; - } - result = std::min(ind, result.value()); + if (it != indices_.end()) { + // we have an uncommitted value, we will return from here + if (it->second >= start_index) { + // we have a qualifying value + return std::make_optional(it->second); } + return std::nullopt; + } + } + + // we have been asked to not include uncommitted data, or there is none available + index_t committed = 0; + FrKeyType key = leaf; + bool success = dataStore_->read_leaf_index(key, committed, tx); + if (success) { + index_t sizeLimit = constrain_tree_size(requestContext, tx); + if (committed < start_index) { + return std::nullopt; } + if (committed >= sizeLimit) { + return std::nullopt; + } + return std::make_optional(committed); } - return result; + return std::nullopt; } template @@ -650,10 +624,6 @@ void ContentAddressedCachedTreeStore::commit(TreeMeta& finalMeta, auto currentRootIter = nodes_.find(uncommittedMeta.root); dataPresent = currentRootIter != nodes_.end(); - if (dataPresent) { - // data is present, hydrate persisted indices - hydrate_indices_from_persisted_store(*tx); - } } { WriteTransactionPtr tx = create_write_transaction(); @@ -661,7 +631,6 @@ void ContentAddressedCachedTreeStore::commit(TreeMeta& finalMeta, if (dataPresent) { // std::cout << "Persisting data for block " << uncommittedMeta.unfinalisedBlockHeight + 1 << std::endl; persist_leaf_indices(*tx); - persist_leaf_keys(uncommittedMeta.committedSize, *tx); } // If we are commiting a block, we need to persist the root, since the new block "references" this root // However, if the root is the empty root we can't persist it, since it's not a real node @@ -714,23 +683,7 @@ void ContentAddressedCachedTreeStore::persist_leaf_indices(WriteT { for (auto& idx : indices_) { FrKeyType key = idx.first; - dataStore_->write_leaf_indices(key, idx.second, tx); - } -} - -template -void ContentAddressedCachedTreeStore::persist_leaf_keys(const index_t& startIndex, WriteTransaction& tx) -{ - for (auto& idx : indices_) { - FrKeyType key = idx.first; - - // write the leaf key against the indices, this is for the pending chain store of indices - for (index_t indexForKey : idx.second.indices) { - if (indexForKey < startIndex) { - continue; - } - dataStore_->write_leaf_key_by_index(key, indexForKey, tx); - } + dataStore_->write_leaf_index(key, idx.second, tx); } } @@ -795,21 +748,6 @@ void ContentAddressedCachedTreeStore::persist_node(const std::opt } } -template -void ContentAddressedCachedTreeStore::hydrate_indices_from_persisted_store(ReadTransaction& tx) -{ - for (auto& idx : indices_) { - std::vector value; - FrKeyType key = idx.first; - Indices persistedIndices; - bool success = dataStore_->read_leaf_indices(key, persistedIndices, tx); - if (success) { - idx.second.indices.insert( - idx.second.indices.begin(), persistedIndices.indices.begin(), persistedIndices.indices.end()); - } - } -} - template void ContentAddressedCachedTreeStore::rollback() { // Extract the committed meta data and destroy the cache @@ -818,7 +756,7 @@ template void ContentAddressedCachedTreeStore(); - indices_ = std::map(); + indices_ = std::map(); leaves_ = std::unordered_map(); nodes_by_index_ = std::vector>(depth_ + 1, std::unordered_map()); leaf_pre_image_by_index_ = std::unordered_map(); @@ -871,11 +809,7 @@ void ContentAddressedCachedTreeStore::advance_finalised_block(con // commit the new finalised block WriteTransactionPtr writeTx = create_write_transaction(); try { - // determine where we need to prune the leaf keys store up to - index_t highestIndexToRemove = blockPayload.size - 1; committedMeta.finalisedBlockHeight = blockNumber; - // clean up the leaf keys index table - dataStore_->delete_all_leaf_keys_before_or_equal_index(highestIndexToRemove, *writeTx); // persist the new meta data persist_meta(committedMeta, *writeTx); writeTx->commit(); @@ -904,7 +838,7 @@ void ContentAddressedCachedTreeStore::unwind_block(const index_t& BlockPayload blockData; BlockPayload previousBlockData; if (blockNumber < 1) { - throw std::runtime_error(format("Unable to remove historical block: ", blockNumber, ". Tree name: ", name_)); + throw std::runtime_error(format("Unable to unwind block: ", blockNumber, ". Tree name: ", name_)); } if (initialised_from_block_.has_value()) { throw std::runtime_error("Removing a block on a fork is forbidden"); @@ -962,7 +896,6 @@ void ContentAddressedCachedTreeStore::unwind_block(const index_t& remove_node(std::optional(blockData.root), 0, maxIndex, *writeTx); // remove the block from the block data table dataStore_->delete_block_data(blockNumber, *writeTx); - remove_leaf_indices_after_or_equal_index(previousBlockData.size, *writeTx); uncommittedMeta.unfinalisedBlockHeight = previousBlockData.blockNumber; uncommittedMeta.size = previousBlockData.size; uncommittedMeta.committedSize = previousBlockData.size; @@ -1056,44 +989,17 @@ void ContentAddressedCachedTreeStore::remove_historical_block(con } template -void ContentAddressedCachedTreeStore::remove_leaf_indices_after_or_equal_index(const index_t& index, - WriteTransaction& tx) -{ - std::vector leafKeys; - dataStore_->read_all_leaf_keys_after_or_equal_index(index, leafKeys, tx); - for (const fr& key : leafKeys) { - remove_leaf_indices(key, index, tx); - } - dataStore_->delete_all_leaf_keys_after_or_equal_index(index, tx); -} - -template -void ContentAddressedCachedTreeStore::remove_leaf_indices(const fr& key, - const index_t& maxIndex, - WriteTransaction& tx) +void ContentAddressedCachedTreeStore::remove_leaf_index(const fr& key, + const index_t& maxIndex, + WriteTransaction& tx) { - // We now have the key, extract the indices - Indices indices; - // std::cout << "Reading indices for key " << key << std::endl; - dataStore_->read_leaf_indices(key, indices, tx); - // std::cout << "Indices length before removal " << indices.indices.size() << std::endl; - - size_t lengthBefore = indices.indices.size(); - - indices.indices.erase( - std::remove_if(indices.indices.begin(), indices.indices.end(), [&](index_t& ind) { return ind >= maxIndex; }), - indices.indices.end()); - - size_t lengthAfter = indices.indices.size(); - // std::cout << "Indices length after removal " << indices.indices.size() << std::endl; - - if (lengthBefore != lengthAfter) { - if (indices.indices.empty()) { - // std::cout << "Deleting indices" << std::endl; - dataStore_->delete_leaf_indices(key, tx); - } else { - // std::cout << "Writing indices" << std::endl; - dataStore_->write_leaf_indices(key, indices, tx); + // We now have the key, extract the index + index_t index = 0; + // std::cout << "Reading index for key " << key << std::endl; + if (dataStore_->read_leaf_index(key, index, tx)) { + if (index >= maxIndex) { + // std::cout << "Deleting index" << std::endl; + dataStore_->delete_leaf_index(key, tx); } } } @@ -1106,7 +1012,7 @@ void ContentAddressedCachedTreeStore::remove_leaf(const fr& hash, // std::cout << "Removing leaf " << hash << std::endl; if (maxIndex.has_value()) { // std::cout << "Max Index" << std::endl; - // We need to clear the entry from the leaf key to indices database as this leaf never existed + // We need to clear the entry from the leaf key to index database as this leaf never existed IndexedLeafValueType leaf; fr key; if (requires_preimage_for_key()) { @@ -1119,7 +1025,7 @@ void ContentAddressedCachedTreeStore::remove_leaf(const fr& hash, } else { key = hash; } - remove_leaf_indices(key, maxIndex.value(), tx); + remove_leaf_index(key, maxIndex.value(), tx); } // std::cout << "Deleting leaf by hash " << std::endl; dataStore_->delete_leaf_by_hash(hash, tx); diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/test_fixtures.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/test_fixtures.hpp index dbf7eaa44d6..1aa333d061b 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/test_fixtures.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/test_fixtures.hpp @@ -56,13 +56,12 @@ void inline check_block_and_size_data(LMDBTreeStore::SharedPtr db, void inline check_indices_data( LMDBTreeStore::SharedPtr db, fr leaf, index_t index, bool entryShouldBePresent, bool indexShouldBePresent) { - Indices indices; + index_t retrieved = 0; LMDBTreeStore::ReadTransaction::Ptr tx = db->create_read_transaction(); - bool success = db->read_leaf_indices(leaf, indices, *tx); + bool success = db->read_leaf_index(leaf, retrieved, *tx); EXPECT_EQ(success, entryShouldBePresent); if (entryShouldBePresent) { - bool found = std::find(indices.indices.begin(), indices.indices.end(), index) != std::end(indices.indices); - EXPECT_EQ(found, indexShouldBePresent); + EXPECT_EQ(index == retrieved, indexShouldBePresent); } } @@ -78,28 +77,4 @@ void check_leaf_by_hash(LMDBTreeStore::SharedPtr db, IndexedLeaf leaf, } } -void inline check_leaf_keys_are_present(LMDBTreeStore::SharedPtr db, - uint64_t startIndex, - uint64_t endIndex, - const std::vector& keys) -{ - LMDBTreeStore::ReadTransaction::Ptr tx = db->create_read_transaction(); - for (uint64_t i = startIndex; i <= endIndex; i++) { - fr leafKey; - bool success = db->read_leaf_key_by_index(i, leafKey, *tx); - EXPECT_TRUE(success); - EXPECT_EQ(leafKey, keys[i - startIndex]); - } -} - -void inline check_leaf_keys_are_not_present(LMDBTreeStore::SharedPtr db, uint64_t startIndex, uint64_t endIndex) -{ - LMDBTreeStore::ReadTransaction::Ptr tx = db->create_read_transaction(); - for (uint64_t i = startIndex; i < endIndex; i++) { - fr leafKey; - bool success = db->read_leaf_key_by_index(i, leafKey, *tx); - EXPECT_FALSE(success); - } -} - } // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/types.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/types.hpp index 6f2ce79c474..8d9c5de4933 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/types.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/types.hpp @@ -16,7 +16,6 @@ struct RequestContext { const std::string BLOCKS_DB = "blocks"; const std::string NODES_DB = "nodes"; const std::string LEAF_PREIMAGES_DB = "leaf preimages"; -const std::string LEAF_KEYS_DB = "leaf keys"; const std::string LEAF_INDICES_DB = "leaf indices"; struct DBStats { @@ -71,7 +70,6 @@ struct TreeDBStats { DBStats blocksDBStats; DBStats nodesDBStats; DBStats leafPreimagesDBStats; - DBStats leafKeysDBStats; DBStats leafIndicesDBStats; TreeDBStats() = default; @@ -82,13 +80,11 @@ struct TreeDBStats { const DBStats& blockStats, const DBStats& nodesStats, const DBStats& leafPreimagesDBStats, - const DBStats& leafKeysDBStats, const DBStats& leafIndicesStats) : mapSize(mapSize) , blocksDBStats(blockStats) , nodesDBStats(nodesStats) , leafPreimagesDBStats(leafPreimagesDBStats) - , leafKeysDBStats(leafKeysDBStats) , leafIndicesDBStats(leafIndicesStats) {} TreeDBStats(const TreeDBStats& other) = default; @@ -96,13 +92,12 @@ struct TreeDBStats { ~TreeDBStats() = default; - MSGPACK_FIELDS(mapSize, blocksDBStats, nodesDBStats, leafPreimagesDBStats, leafKeysDBStats, leafIndicesDBStats) + MSGPACK_FIELDS(mapSize, blocksDBStats, nodesDBStats, leafPreimagesDBStats, leafIndicesDBStats) bool operator==(const TreeDBStats& other) const { return mapSize == other.mapSize && blocksDBStats == other.blocksDBStats && nodesDBStats == other.nodesDBStats && - leafPreimagesDBStats == other.leafPreimagesDBStats && leafKeysDBStats == other.leafPreimagesDBStats && - leafIndicesDBStats == other.leafIndicesDBStats; + leafPreimagesDBStats == other.leafPreimagesDBStats && leafIndicesDBStats == other.leafIndicesDBStats; } TreeDBStats& operator=(TreeDBStats&& other) noexcept @@ -112,7 +107,6 @@ struct TreeDBStats { blocksDBStats = std::move(other.blocksDBStats); nodesDBStats = std::move(other.nodesDBStats); leafPreimagesDBStats = std::move(other.leafPreimagesDBStats); - leafKeysDBStats = std::move(other.leafKeysDBStats); leafIndicesDBStats = std::move(other.leafIndicesDBStats); } return *this; @@ -123,8 +117,8 @@ struct TreeDBStats { friend std::ostream& operator<<(std::ostream& os, const TreeDBStats& stats) { os << "Map Size: " << stats.mapSize << " Blocks DB " << stats.blocksDBStats << ", Nodes DB " - << stats.nodesDBStats << ", Leaf Pre-images DB " << stats.leafPreimagesDBStats << ", Leaf Keys DB " - << stats.leafKeysDBStats << ", Leaf Indices DB " << stats.leafIndicesDBStats; + << stats.nodesDBStats << ", Leaf Pre-images DB " << stats.leafPreimagesDBStats << ", Leaf Indices DB " + << stats.leafIndicesDBStats; return os; } }; diff --git a/yarn-project/world-state/src/native/message.ts b/yarn-project/world-state/src/native/message.ts index 3ee22e1aed3..8013e81e772 100644 --- a/yarn-project/world-state/src/native/message.ts +++ b/yarn-project/world-state/src/native/message.ts @@ -136,8 +136,6 @@ export interface TreeDBStats { nodesDBStats: DBStats; /** Stats for the 'leaf pre-images' DB */ leafPreimagesDBStats: DBStats; - /** Stats for the 'leaf keys' DB */ - leafKeysDBStats: DBStats; /** Stats for the 'leaf indices' DB */ leafIndicesDBStats: DBStats; } @@ -271,7 +269,6 @@ export function sanitiseMeta(meta: TreeMeta) { export function sanitiseTreeDBStats(stats: TreeDBStats) { stats.blocksDBStats = sanitiseDBStats(stats.blocksDBStats); stats.leafIndicesDBStats = sanitiseDBStats(stats.leafIndicesDBStats); - stats.leafKeysDBStats = sanitiseDBStats(stats.leafKeysDBStats); stats.leafPreimagesDBStats = sanitiseDBStats(stats.leafPreimagesDBStats); stats.nodesDBStats = sanitiseDBStats(stats.nodesDBStats); stats.mapSize = BigInt(stats.mapSize); diff --git a/yarn-project/world-state/src/synchronizer/config.ts b/yarn-project/world-state/src/synchronizer/config.ts index 4b90127e952..bdd96365da9 100644 --- a/yarn-project/world-state/src/synchronizer/config.ts +++ b/yarn-project/world-state/src/synchronizer/config.ts @@ -11,7 +11,7 @@ export interface WorldStateConfig { /** Size of the batch for each get-blocks request from the synchronizer to the archiver. */ worldStateBlockRequestBatchSize?: number; - /** The maximum size of the combined world state db in KB, optional, will inherit from the general dataStoreMapSizeKB if not specified*/ + /** The map size to be provided to LMDB for each world state tree DB, optional, will inherit from the general dataStoreMapSizeKB if not specified*/ worldStateDbMapSizeKb?: number; /** Optional directory for the world state DB, if unspecified will default to the general data directory */