diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp index d645b460708..b27be4d3aae 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp @@ -159,32 +159,34 @@ template class ContentAddressedAppendOn /** * @brief Returns the index of the provided leaf in the tree */ - void find_leaf_index(const fr& leaf, bool includeUncommitted, const FindLeafCallback& on_completion) const; + void find_leaf_indices(const std::vector& leaves, + bool includeUncommitted, + const FindLeafCallback& on_completion) const; /** * @brief Returns the index of the provided leaf in the tree */ - void find_leaf_index(const fr& leaf, - const block_number_t& blockNumber, - bool includeUncommitted, - const FindLeafCallback& on_completion) const; + void find_leaf_indices(const std::vector& leaves, + const block_number_t& blockNumber, + bool includeUncommitted, + const FindLeafCallback& on_completion) const; /** * @brief Returns the index of the provided leaf in the tree only if it exists after the index value provided */ - void find_leaf_index_from(const fr& leaf, - const index_t& start_index, - bool includeUncommitted, - const FindLeafCallback& on_completion) const; + void find_leaf_indices_from(const std::vector& leaves, + const index_t& start_index, + bool includeUncommitted, + const FindLeafCallback& on_completion) const; /** * @brief Returns the index of the provided leaf in the tree only if it exists after the index value provided */ - void find_leaf_index_from(const fr& leaf, - const index_t& start_index, - const block_number_t& blockNumber, - bool includeUncommitted, - const FindLeafCallback& on_completion) const; + void find_leaf_indices_from(const std::vector& leaves, + const index_t& start_index, + const block_number_t& blockNumber, + bool includeUncommitted, + const FindLeafCallback& on_completion) const; /** * @brief Returns the block numbers that correspond to the given indices values @@ -415,14 +417,9 @@ void ContentAddressedAppendOnlyTree::find_block_numbers( execute_and_report( [=, this](TypedResponse& response) { response.inner.blockNumbers.reserve(indices.size()); - TreeMeta meta; ReadTransactionPtr tx = store_->create_read_transaction(); - store_->get_meta(meta, *tx, true); - index_t maxIndex = meta.committedSize; for (index_t index : indices) { - bool outOfRange = index >= maxIndex; - std::optional block = - outOfRange ? std::nullopt : store_->find_block_for_index(index, *tx); + std::optional block = store_->find_block_for_index(index, *tx); response.inner.blockNumbers.emplace_back(block); } }, @@ -441,16 +438,14 @@ void ContentAddressedAppendOnlyTree::find_block_numbers( execute_and_report( [=, this](TypedResponse& response) { response.inner.blockNumbers.reserve(indices.size()); - TreeMeta meta; BlockPayload blockPayload; ReadTransactionPtr tx = store_->create_read_transaction(); - store_->get_meta(meta, *tx, true); if (!store_->get_block_data(blockNumber, blockPayload, *tx)) { throw std::runtime_error(format("Unable to find block numbers for indices for block ", blockNumber, ", failed to get block data.")); } - index_t maxIndex = std::min(meta.committedSize, blockPayload.size); + index_t maxIndex = blockPayload.size; for (index_t index : indices) { bool outOfRange = index >= maxIndex; std::optional block = @@ -718,43 +713,45 @@ void ContentAddressedAppendOnlyTree::get_leaf(const index_ } template -void ContentAddressedAppendOnlyTree::find_leaf_index(const fr& leaf, - bool includeUncommitted, - const FindLeafCallback& on_completion) const +void ContentAddressedAppendOnlyTree::find_leaf_indices( + const std::vector& leaves, + bool includeUncommitted, + const FindLeafCallback& on_completion) const { - find_leaf_index_from(leaf, 0, includeUncommitted, on_completion); + find_leaf_indices_from(leaves, 0, includeUncommitted, on_completion); } template -void ContentAddressedAppendOnlyTree::find_leaf_index(const fr& leaf, - const block_number_t& blockNumber, - bool includeUncommitted, - const FindLeafCallback& on_completion) const +void ContentAddressedAppendOnlyTree::find_leaf_indices( + const std::vector& leaves, + const block_number_t& blockNumber, + bool includeUncommitted, + const FindLeafCallback& on_completion) const { - find_leaf_index_from(leaf, 0, blockNumber, includeUncommitted, on_completion); + find_leaf_indices_from(leaves, 0, blockNumber, includeUncommitted, on_completion); } template -void ContentAddressedAppendOnlyTree::find_leaf_index_from( - const fr& leaf, const index_t& start_index, bool includeUncommitted, const FindLeafCallback& on_completion) const +void ContentAddressedAppendOnlyTree::find_leaf_indices_from( + const std::vector& leaves, + const index_t& start_index, + bool includeUncommitted, + const FindLeafCallback& on_completion) const { auto job = [=, this]() -> void { execute_and_report( [=, this](TypedResponse& response) { - if (leaf == fr::zero()) { - throw std::runtime_error("Requesting indices for zero leaves is prohibited"); - } + response.inner.leaf_indices.reserve(leaves.size()); ReadTransactionPtr tx = store_->create_read_transaction(); + RequestContext requestContext; requestContext.includeUncommitted = includeUncommitted; requestContext.root = store_->get_current_root(*tx, includeUncommitted); - std::optional leaf_index = - store_->find_leaf_index_from(leaf, start_index, requestContext, *tx, includeUncommitted); - response.success = leaf_index.has_value(); - if (response.success) { - response.inner.leaf_index = leaf_index.value(); - } else { - response.message = format("Failed to find index from ", start_index, " for leaf ", leaf); + + for (const auto& leaf : leaves) { + std::optional leaf_index = + store_->find_leaf_index_from(leaf, start_index, requestContext, *tx); + response.inner.leaf_indices.emplace_back(leaf_index); } }, on_completion); @@ -763,8 +760,8 @@ void ContentAddressedAppendOnlyTree::find_leaf_index_from( } template -void ContentAddressedAppendOnlyTree::find_leaf_index_from( - const fr& leaf, +void ContentAddressedAppendOnlyTree::find_leaf_indices_from( + const std::vector& leaves, const index_t& start_index, const block_number_t& blockNumber, bool includeUncommitted, @@ -773,12 +770,10 @@ void ContentAddressedAppendOnlyTree::find_leaf_index_from( auto job = [=, this]() -> void { execute_and_report( [=, this](TypedResponse& response) { + response.inner.leaf_indices.reserve(leaves.size()); if (blockNumber == 0) { throw std::runtime_error("Unable to find leaf index for block number 0"); } - if (leaf == fr::zero()) { - throw std::runtime_error("Requesting indices for zero leaves is prohibited"); - } ReadTransactionPtr tx = store_->create_read_transaction(); BlockPayload blockData; if (!store_->get_block_data(blockNumber, blockData, *tx)) { @@ -788,18 +783,17 @@ void ContentAddressedAppendOnlyTree::find_leaf_index_from( blockNumber, ", failed to get block data.")); } + RequestContext requestContext; requestContext.blockNumber = blockNumber; requestContext.includeUncommitted = includeUncommitted; requestContext.root = blockData.root; - std::optional leaf_index = - store_->find_leaf_index_from(leaf, start_index, requestContext, *tx, includeUncommitted); - response.success = leaf_index.has_value(); - if (response.success) { - response.inner.leaf_index = leaf_index.value(); - } else { - response.message = format( - "Failed to find index from ", start_index, " for leaf ", leaf, " at block ", blockNumber); + requestContext.maxIndex = blockData.size; + + for (const auto& leaf : leaves) { + std::optional leaf_index = + store_->find_leaf_index_from(leaf, start_index, requestContext, *tx); + response.inner.leaf_indices.emplace_back(leaf_index); } }, on_completion); diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.test.cpp index 83f72c9ca1f..fb16c9d9053 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.test.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -220,83 +221,6 @@ void finalise_block(TreeType& tree, const block_number_t& blockNumber, bool expe signal.wait_for_level(); } -void check_find_leaf_index( - TreeType& tree, const fr& leaf, index_t expected_index, bool expected_success, bool includeUncommitted = true) -{ - Signal signal; - auto completion = [&](const TypedResponse& response) -> void { - EXPECT_EQ(response.success, expected_success); - if (response.success) { - EXPECT_EQ(response.inner.leaf_index, expected_index); - } - signal.signal_level(); - }; - - tree.find_leaf_index(leaf, includeUncommitted, completion); - signal.wait_for_level(); -} - -void check_find_historic_leaf_index(TreeType& tree, - const index_t& block_number, - const fr& leaf, - index_t expected_index, - bool expected_success, - bool includeUncommitted = true) -{ - Signal signal; - auto completion = [&](const TypedResponse& response) -> void { - EXPECT_EQ(response.success, expected_success); - if (response.success) { - EXPECT_EQ(response.inner.leaf_index, expected_index); - } - signal.signal_level(); - }; - - tree.find_leaf_index(leaf, block_number, includeUncommitted, completion); - signal.wait_for_level(); -} - -void check_find_historic_leaf_index_from(TreeType& tree, - const index_t& block_number, - const fr& leaf, - index_t start_index, - index_t expected_index, - bool expected_success, - bool includeUncommitted = true) -{ - Signal signal; - auto completion = [&](const TypedResponse& response) -> void { - EXPECT_EQ(response.success, expected_success); - if (response.success) { - EXPECT_EQ(response.inner.leaf_index, expected_index); - } - signal.signal_level(); - }; - - tree.find_leaf_index_from(leaf, start_index, block_number, includeUncommitted, completion); - signal.wait_for_level(); -} - -void check_find_leaf_index_from(TreeType& tree, - const fr& leaf, - index_t start_index, - index_t expected_index, - bool expected_success, - bool includeUncommitted = true) -{ - Signal signal; - auto completion = [&](const TypedResponse& response) -> void { - EXPECT_EQ(response.success, expected_success); - if (response.success) { - EXPECT_EQ(response.inner.leaf_index, expected_index); - } - signal.signal_level(); - }; - - tree.find_leaf_index_from(leaf, start_index, includeUncommitted, completion); - signal.wait_for_level(); -} - void check_leaf( TreeType& tree, const fr& leaf, index_t leaf_index, bool expected_success, bool includeUncommitted = true) { @@ -711,19 +635,19 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, test_find_leaf_index) add_value(tree, 40); // check the committed state and that the uncommitted state is empty - check_find_leaf_index(tree, 10, 1, true, true); - check_find_leaf_index(tree, 10, 0, false, false); + check_find_leaf_index(tree, fr(10), 1, true, true); + check_find_leaf_index(tree, { fr(10) }, { std::nullopt }, true, false); - check_find_leaf_index(tree, 15, 0, false, true); - check_find_leaf_index(tree, 15, 0, false, false); + check_find_leaf_index(tree, { fr(15) }, { std::nullopt }, true, true); + check_find_leaf_index(tree, { fr(15) }, { std::nullopt }, true, false); - check_find_leaf_index(tree, 40, 3, true, true); - check_find_leaf_index(tree, 30, 0, true, true); - check_find_leaf_index(tree, 20, 2, true, true); + check_find_leaf_index(tree, fr(40), 3, true, true); + check_find_leaf_index(tree, fr(30), 0, true, true); + check_find_leaf_index(tree, fr(20), 2, true, true); - check_find_leaf_index(tree, 40, 0, false, false); - check_find_leaf_index(tree, 30, 0, false, false); - check_find_leaf_index(tree, 20, 0, false, false); + check_find_leaf_index(tree, { fr(40) }, { std::nullopt }, true, false); + check_find_leaf_index(tree, { fr(30) }, { std::nullopt }, true, false); + check_find_leaf_index(tree, { fr(20) }, { std::nullopt }, true, false); commit_tree(tree); @@ -731,13 +655,13 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, test_find_leaf_index) add_values(tree, values); // check the now committed state - check_find_leaf_index(tree, 40, 3, true, false); - check_find_leaf_index(tree, 30, 0, true, false); - check_find_leaf_index(tree, 20, 2, true, false); + check_find_leaf_index(tree, fr(40), 3, true, false); + check_find_leaf_index(tree, fr(30), 0, true, false); + check_find_leaf_index(tree, fr(20), 2, true, false); // check the new uncommitted state - check_find_leaf_index(tree, 18, 5, true, true); - check_find_leaf_index(tree, 18, 0, false, false); + check_find_leaf_index(tree, fr(18), 5, true, true); + check_find_leaf_index(tree, { fr(18) }, { std::nullopt }, true, false); commit_tree(tree); @@ -745,9 +669,9 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, test_find_leaf_index) add_values(tree, values); // verify the find index from api - check_find_leaf_index_from(tree, 18, 0, 5, true, true); - check_find_leaf_index_from(tree, 19, 6, 10, true, true); - check_find_leaf_index_from(tree, 19, 0, 0, false, false); + check_find_leaf_index_from(tree, fr(18), 0, 5, true, true); + check_find_leaf_index_from(tree, fr(19), 6, 10, true, true); + check_find_leaf_index_from(tree, { fr(19) }, 0, { std::nullopt }, true, false); commit_tree(tree); @@ -759,13 +683,13 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, test_find_leaf_index) check_size(tree, 12, false); // look past the last instance of this leaf - check_find_leaf_index_from(tree, 18, 6, 0, false, true); + check_find_leaf_index_from(tree, { fr(18) }, 6, { std::nullopt }, true, true); // look beyond the end of uncommitted - check_find_leaf_index_from(tree, 18, 15, 0, false, true); + check_find_leaf_index_from(tree, { fr(18) }, 15, { std::nullopt }, true, true); // look beyond the end of committed and don't include uncomitted - check_find_leaf_index_from(tree, 88, 13, 0, false, false); + check_find_leaf_index_from(tree, { fr(88) }, 13, { std::nullopt }, true, false); } TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_add_multiple_values) @@ -856,10 +780,10 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_not_retrieve_zero_leaf_i add_values(tree, to_add); commit_tree(tree); fr leaf = fr::zero(); - check_find_leaf_index(tree, leaf, 0, false); - check_find_historic_leaf_index(tree, 1, leaf, 0, false); - check_find_leaf_index_from(tree, leaf, 0, 0, false); - check_find_historic_leaf_index_from(tree, 1, leaf, 0, 0, false); + check_find_leaf_index(tree, { leaf }, { std::nullopt }, true); + check_historic_find_leaf_index(tree, { leaf }, 1, { std::nullopt }, true); + check_find_leaf_index_from(tree, { leaf }, 0, { std::nullopt }, true); + check_historic_find_leaf_index_from(tree, { leaf }, 1, 0, { std::nullopt }, true); } TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_commit_multiple_blocks) @@ -1042,23 +966,23 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, test_find_historic_leaf_inde add_values(tree, values); // should not be present at block 1 - check_find_historic_leaf_index(tree, 1, 26, 0, false); + check_historic_find_leaf_index(tree, { fr(26) }, 1, { std::nullopt }, true); // should be present at block 2 - check_find_historic_leaf_index(tree, 2, 26, 6, true); + check_historic_find_leaf_index(tree, fr(26), 2, 6, true); // at block 1 leaf 18 should not be found if only considering committed - check_find_historic_leaf_index_from(tree, 1, 18, 2, 0, false, false); + check_historic_find_leaf_index_from(tree, { fr(18) }, 1, 2, { std::nullopt }, true, false); // at block 2 it should be - check_find_historic_leaf_index_from(tree, 2, 18, 2, 5, true); + check_historic_find_leaf_index_from(tree, fr(18), 2, 2, 5, true); // at block 2, from index 6, 19 should not be found if looking only at committed - check_find_historic_leaf_index_from(tree, 2, 19, 6, 5, false, false); + check_historic_find_leaf_index_from(tree, { fr(19) }, 2, 6, { std::nullopt }, true, false); // at block 2, from index 6, 19 should be found if looking at uncommitted too - check_find_historic_leaf_index_from(tree, 2, 19, 6, 10, true); + check_historic_find_leaf_index_from(tree, fr(19), 2, 6, 10, true); commit_tree(tree); // at block 3, from index 6, should now be found in committed only - check_find_historic_leaf_index_from(tree, 3, 19, 6, 10, true, false); + check_historic_find_leaf_index_from(tree, fr(19), 3, 6, 10, true, false); } TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_be_filled) @@ -1249,12 +1173,12 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_create_images_at_histori check_root(treeAtBlock2, block2Root); check_sibling_path(treeAtBlock2, 3, block2SiblingPathIndex3, false, true); check_leaf(treeAtBlock2, 20, 2, true); - check_find_leaf_index(treeAtBlock2, 10, 1, true); - check_find_leaf_index_from(treeAtBlock2, 15, 1, 4, true); + check_find_leaf_index(treeAtBlock2, fr(10), 1, true); + check_find_leaf_index_from(treeAtBlock2, fr(15), 1, 4, true); // should not exist in our image check_leaf(treeAtBlock2, 4, 9, false); - check_find_leaf_index(treeAtBlock2, 4, 0, false); + check_find_leaf_index(treeAtBlock2, { fr(4) }, { std::nullopt }, true); // now add the same values to our image add_values(treeAtBlock2, values); @@ -1269,12 +1193,12 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_create_images_at_histori // now check historic data check_historic_sibling_path(treeAtBlock2, 3, block1SiblingPathIndex3, 1); - check_find_historic_leaf_index(treeAtBlock2, 1, 10, 1, true); - check_find_historic_leaf_index(treeAtBlock2, 2, 16, 8, true, true); - check_find_historic_leaf_index(treeAtBlock2, 2, 16, 8, false, false); + check_historic_find_leaf_index(treeAtBlock2, fr(10), 2, 1, true); + check_historic_find_leaf_index(treeAtBlock2, fr(16), 2, 8, true, true); + check_historic_find_leaf_index(treeAtBlock2, { fr(16) }, 2, { std::nullopt }, true, false); - check_find_historic_leaf_index_from(treeAtBlock2, 1, 18, 3, 0, false, false); - check_find_historic_leaf_index_from(treeAtBlock2, 1, 20, 0, 2, true, false); + check_historic_find_leaf_index_from(treeAtBlock2, { fr(18) }, 1, 3, { std::nullopt }, true, false); + check_historic_find_leaf_index_from(treeAtBlock2, fr(20), 1, 0, 2, true, false); check_block_height(treeAtBlock2, 2); @@ -1318,8 +1242,8 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_remove_historic_block_da const index_t leafIndex = 6; check_historic_leaf(tree, blockNumber, VALUES[leafIndex], leafIndex, expectedSuccess); - check_find_historic_leaf_index(tree, blockNumber, VALUES[leafIndex], leafIndex, expectedSuccess); - check_find_historic_leaf_index_from(tree, blockNumber, VALUES[leafIndex], 0, leafIndex, expectedSuccess); + check_historic_find_leaf_index(tree, VALUES[leafIndex], blockNumber, leafIndex, expectedSuccess); + check_historic_find_leaf_index_from(tree, VALUES[leafIndex], blockNumber, 0, leafIndex, expectedSuccess); } }; @@ -1453,7 +1377,7 @@ void test_unwind(std::string directory, // Trying to find leaves appended in the block that was removed should fail check_leaf(tree, values[1 + deletedBlockStartIndex], 1 + deletedBlockStartIndex, false); - check_find_leaf_index(tree, values[1 + deletedBlockStartIndex], 1 + deletedBlockStartIndex, false); + check_find_leaf_index(tree, { values[1 + deletedBlockStartIndex] }, { std::nullopt }, true); for (index_t j = 0; j < numBlocks; j++) { index_t historicBlockNumber = j + 1; @@ -1466,18 +1390,20 @@ void test_unwind(std::string directory, const index_t leafIndex = 1; check_historic_leaf(tree, historicBlockNumber, values[leafIndex], leafIndex, expectedSuccess); + std::vector> expected_results; + if (expectedSuccess) { + if (values[leafIndex] != fr::zero()) { + expected_results.emplace_back(std::make_optional(leafIndex)); + } else { + expected_results.emplace_back(std::nullopt); + } + } + // find historic leaves, provided they are not zero leaves - check_find_historic_leaf_index(tree, - historicBlockNumber, - values[leafIndex], - leafIndex, - expectedSuccess && values[leafIndex] != fr::zero()); - check_find_historic_leaf_index_from(tree, - historicBlockNumber, - values[leafIndex], - 0, - leafIndex, - expectedSuccess && values[leafIndex] != fr::zero()); + check_historic_find_leaf_index( + tree, { values[leafIndex] }, historicBlockNumber, expected_results, expectedSuccess); + check_historic_find_leaf_index_from( + tree, { values[leafIndex] }, historicBlockNumber, 0, expected_results, expectedSuccess); } } } diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp index e075e36315d..94b8d2723bf 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp @@ -127,23 +127,6 @@ class ContentAddressedIndexedTree : public ContentAddressedAppendOnlyTree::FindLeafCallback& on_completion) const; - - /** - * @brief Find the index of the provided leaf value if it exists, only considers indexed beyond the value provided - */ - void find_leaf_index_from( - const LeafValueType& leaf, - const index_t& start_index, - bool includeUncommitted, - const ContentAddressedAppendOnlyTree::FindLeafCallback& on_completion) const; - /** * @brief Find the leaf with the value immediately lower then the value provided */ @@ -154,25 +137,6 @@ class ContentAddressedIndexedTree : public ContentAddressedAppendOnlyTree::FindLeafCallback& on_completion) const; - - /** - * @brief Find the index of the provided leaf value if it exists, only considers indexed beyond the value provided - */ - void find_leaf_index_from( - const LeafValueType& leaf, - const block_number_t& blockNumber, - const index_t& start_index, - bool includeUncommitted, - const ContentAddressedAppendOnlyTree::FindLeafCallback& on_completion) const; - /** * @brief Find the leaf with the value immediately lower then the value provided */ @@ -447,95 +411,6 @@ void ContentAddressedIndexedTree::get_leaf(const index_t& workers_->enqueue(job); } -template -void ContentAddressedIndexedTree::find_leaf_index( - const LeafValueType& leaf, - bool includeUncommitted, - const ContentAddressedAppendOnlyTree::FindLeafCallback& on_completion) const -{ - find_leaf_index_from(leaf, 0, includeUncommitted, on_completion); -} - -template -void ContentAddressedIndexedTree::find_leaf_index( - const LeafValueType& leaf, - const block_number_t& blockNumber, - bool includeUncommitted, - const ContentAddressedAppendOnlyTree::FindLeafCallback& on_completion) const -{ - find_leaf_index_from(leaf, blockNumber, 0, includeUncommitted, on_completion); -} - -template -void ContentAddressedIndexedTree::find_leaf_index_from( - const LeafValueType& leaf, - const index_t& start_index, - bool includeUncommitted, - const ContentAddressedAppendOnlyTree::FindLeafCallback& on_completion) const -{ - auto job = [=, this]() -> void { - execute_and_report( - [=, this](TypedResponse& response) { - typename Store::ReadTransactionPtr tx = store_->create_read_transaction(); - RequestContext requestContext; - requestContext.includeUncommitted = includeUncommitted; - requestContext.root = store_->get_current_root(*tx, includeUncommitted); - std::optional leaf_index = - store_->find_leaf_index_from(leaf, start_index, requestContext, *tx, includeUncommitted); - response.success = leaf_index.has_value(); - if (response.success) { - response.inner.leaf_index = leaf_index.value(); - } else { - response.message = format("Index not found for leaf ", leaf); - } - }, - on_completion); - }; - workers_->enqueue(job); -} - -template -void ContentAddressedIndexedTree::find_leaf_index_from( - const LeafValueType& leaf, - const block_number_t& blockNumber, - const index_t& start_index, - bool includeUncommitted, - const ContentAddressedAppendOnlyTree::FindLeafCallback& on_completion) const -{ - auto job = [=, this]() -> void { - execute_and_report( - [=, this](TypedResponse& response) { - if (blockNumber == 0) { - throw std::runtime_error("Unable to find leaf index from for block 0"); - } - typename Store::ReadTransactionPtr tx = store_->create_read_transaction(); - BlockPayload blockData; - if (!store_->get_block_data(blockNumber, blockData, *tx)) { - throw std::runtime_error(format("Unable to find leaf from index ", - start_index, - " for block ", - blockNumber, - ", failed to get block data.")); - } - RequestContext requestContext; - requestContext.blockNumber = blockNumber; - requestContext.includeUncommitted = includeUncommitted; - requestContext.root = blockData.root; - std::optional leaf_index = - store_->find_leaf_index_from(leaf, start_index, requestContext, *tx, includeUncommitted); - response.success = leaf_index.has_value(); - if (response.success) { - response.inner.leaf_index = leaf_index.value(); - } else { - response.message = - format("Unable to find leaf from index ", start_index, " for block ", blockNumber); - } - }, - on_completion); - }; - workers_->enqueue(job); -} - template void ContentAddressedIndexedTree::find_low_leaf(const fr& leaf_key, bool includeUncommitted, @@ -580,6 +455,7 @@ void ContentAddressedIndexedTree::find_low_leaf(const fr& requestContext.blockNumber = blockNumber; requestContext.includeUncommitted = includeUncommitted; requestContext.root = blockData.root; + requestContext.maxIndex = blockData.size; std::pair result = store_->find_low_value(leaf_key, requestContext, *tx); response.inner.index = result.second; response.inner.is_already_present = result.first; diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.test.cpp index 24e8565cc4f..b42d3175574 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.test.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -195,90 +196,6 @@ GetLowIndexedLeafResponse get_historic_low_leaf(TypeOfTree& tree, return low_leaf_info; } -template -void check_find_leaf_index(TypeOfTree& tree, - const LeafValueType& leaf, - index_t expected_index, - bool expected_success, - bool includeUncommitted = true) -{ - Signal signal; - auto completion = [&](const TypedResponse& response) -> void { - EXPECT_EQ(response.success, expected_success); - if (response.success) { - EXPECT_EQ(response.inner.leaf_index, expected_index); - } - signal.signal_level(); - }; - - tree.find_leaf_index(leaf, includeUncommitted, completion); - signal.wait_for_level(); -} - -template -void check_find_leaf_index_from(TypeOfTree& tree, - const LeafValueType& leaf, - index_t start_index, - index_t expected_index, - bool expected_success, - bool includeUncommitted = true) -{ - Signal signal; - auto completion = [&](const TypedResponse& response) -> void { - EXPECT_EQ(response.success, expected_success); - if (response.success) { - EXPECT_EQ(response.inner.leaf_index, expected_index); - } - signal.signal_level(); - }; - - tree.find_leaf_index_from(leaf, start_index, includeUncommitted, completion); - signal.wait_for_level(); -} - -template -void check_historic_find_leaf_index(TypeOfTree& tree, - const LeafValueType& leaf, - block_number_t blockNumber, - index_t expected_index, - bool expected_success, - bool includeUncommitted = true) -{ - Signal signal; - auto completion = [&](const TypedResponse& response) -> void { - EXPECT_EQ(response.success, expected_success); - if (response.success) { - EXPECT_EQ(response.inner.leaf_index, expected_index); - } - signal.signal_level(); - }; - - tree.find_leaf_index(leaf, blockNumber, includeUncommitted, completion); - signal.wait_for_level(); -} - -template -void check_historic_find_leaf_index_from(TypeOfTree& tree, - const LeafValueType& leaf, - block_number_t blockNumber, - index_t start_index, - index_t expected_index, - bool expected_success, - bool includeUncommitted = true) -{ - Signal signal; - auto completion = [&](const TypedResponse& response) -> void { - EXPECT_EQ(response.success, expected_success); - if (response.success) { - EXPECT_EQ(response.inner.leaf_index, expected_index); - } - signal.signal_level(); - }; - - tree.find_leaf_index_from(leaf, blockNumber, start_index, includeUncommitted, completion); - signal.wait_for_level(); -} - template void check_historic_leaf(TypeOfTree& tree, const LeafValueType& leaf, @@ -625,18 +542,23 @@ TEST_F(PersistedContentAddressedIndexedTreeTest, test_find_leaf_index) // check the committed state and that the uncommitted state is empty check_find_leaf_index(tree, NullifierLeafValue(10), 1 + initial_size, true, true); - check_find_leaf_index(tree, NullifierLeafValue(10), 0, false, false); + check_find_leaf_index( + tree, { NullifierLeafValue(10) }, { std::nullopt }, true, false); - check_find_leaf_index(tree, NullifierLeafValue(15), 0, false, true); - check_find_leaf_index(tree, NullifierLeafValue(15), 0, false, false); + check_find_leaf_index(tree, { NullifierLeafValue(15) }, { std::nullopt }, true, true); + check_find_leaf_index( + tree, { NullifierLeafValue(15) }, { std::nullopt }, true, false); check_find_leaf_index(tree, NullifierLeafValue(40), 3 + initial_size, true, true); check_find_leaf_index(tree, NullifierLeafValue(30), 0 + initial_size, true, true); check_find_leaf_index(tree, NullifierLeafValue(20), 2 + initial_size, true, true); - check_find_leaf_index(tree, NullifierLeafValue(40), 0, false, false); - check_find_leaf_index(tree, NullifierLeafValue(30), 0, false, false); - check_find_leaf_index(tree, NullifierLeafValue(20), 0, false, false); + check_find_leaf_index( + tree, { NullifierLeafValue(40) }, { std::nullopt }, true, false); + check_find_leaf_index( + tree, { NullifierLeafValue(30) }, { std::nullopt }, true, false); + check_find_leaf_index( + tree, { NullifierLeafValue(20) }, { std::nullopt }, true, false); commit_tree(tree); @@ -654,7 +576,8 @@ TEST_F(PersistedContentAddressedIndexedTreeTest, test_find_leaf_index) // check the new uncommitted state check_find_leaf_index(tree, NullifierLeafValue(18), 5 + initial_size, true, true); - check_find_leaf_index(tree, NullifierLeafValue(18), 0, false, false); + check_find_leaf_index( + tree, { NullifierLeafValue(18) }, { std::nullopt }, true, false); commit_tree(tree); @@ -1854,8 +1777,9 @@ TEST_F(PersistedContentAddressedIndexedTreeTest, test_historical_leaves) LMDBTreeStore::SharedPtr db = std::make_shared(_directory, name, _mapSize, _maxReaders); std::unique_ptr> store = std::make_unique>(name, depth, db); - auto tree = ContentAddressedIndexedTree, Poseidon2HashPolicy>( - std::move(store), workers, current_size); + using LocalTreeType = + ContentAddressedIndexedTree, Poseidon2HashPolicy>; + auto tree = LocalTreeType(std::move(store), workers, current_size); /** * Intial state: @@ -1913,7 +1837,7 @@ TEST_F(PersistedContentAddressedIndexedTreeTest, test_historical_leaves) auto leaf2AtBlock2 = PublicDataLeafValue(30, 5); check_historic_leaf(tree, leaf1AtBlock1, 1, 1, true); - // shoudl find this leaf at both blocks 1 and 2 as it looks for the slot which doesn't change + // should find this leaf at both blocks 1 and 2 as it looks for the slot which doesn't change check_historic_find_leaf_index(tree, leaf1AtBlock1, 1, 1, true); check_historic_find_leaf_index(tree, leaf1AtBlock1, 2, 1, true); @@ -1965,7 +1889,8 @@ TEST_F(PersistedContentAddressedIndexedTreeTest, test_historical_leaves) check_historic_leaf(tree, leaf2AtBlock3, 2, 3, true); // should not be found at block 1 - check_historic_find_leaf_index_from(tree, PublicDataLeafValue(10, 20), 1, 0, 0, false); + check_historic_find_leaf_index_from( + tree, { PublicDataLeafValue(10, 20) }, 1, 0, { std::nullopt }, true); // should be found at block check_historic_find_leaf_index_from(tree, PublicDataLeafValue(10, 20), 2, 0, 3, true); @@ -2109,7 +2034,7 @@ TEST_F(PersistedContentAddressedIndexedTreeTest, test_can_create_forks_at_histor // should not exist in our image get_leaf(treeAtBlock2, 35 + batch_size, false, false); - check_find_leaf_index(treeAtBlock2, batch3[4], 0, false); + check_find_leaf_index(treeAtBlock2, { batch3[4] }, { std::nullopt }, true); // now add the same values to our image add_values(treeAtBlock2, batch3); @@ -2129,10 +2054,12 @@ TEST_F(PersistedContentAddressedIndexedTreeTest, test_can_create_forks_at_histor EXPECT_EQ(historicSiblingPath, block1SiblingPathIndex3); check_historic_find_leaf_index(treeAtBlock2, batch1[3], 1, 3 + batch_size, true); check_historic_find_leaf_index(treeAtBlock2, batch3[3], 2, 35 + batch_size, true, true); - check_historic_find_leaf_index(treeAtBlock2, batch3[3], 2, 35 + batch_size, false, false); + check_historic_find_leaf_index( + treeAtBlock2, { batch3[3] }, 2, { std::nullopt }, true, false); check_historic_find_leaf_index_from(treeAtBlock2, batch1[3], 2, 0, 3 + batch_size, true, false); - check_historic_find_leaf_index_from(treeAtBlock2, batch3[3], 2, 20 + batch_size, 35 + batch_size, false, false); + check_historic_find_leaf_index_from( + treeAtBlock2, { batch3[3] }, 2, 20 + batch_size, { std::nullopt }, true, false); check_historic_find_leaf_index_from(treeAtBlock2, batch3[3], 2, 20 + batch_size, 35 + batch_size, true, true); check_unfinalised_block_height(treeAtBlock2, 2); @@ -2151,8 +2078,9 @@ TEST_F(PersistedContentAddressedIndexedTreeTest, test_remove_historical_blocks) LMDBTreeStore::SharedPtr db = std::make_shared(_directory, name, _mapSize, _maxReaders); std::unique_ptr> store = std::make_unique>(name, depth, db); - auto tree = ContentAddressedIndexedTree, Poseidon2HashPolicy>( - std::move(store), workers, current_size); + using LocalTreeType = + ContentAddressedIndexedTree, Poseidon2HashPolicy>; + auto tree = LocalTreeType(std::move(store), workers, current_size); /** * Intial state: @@ -2268,7 +2196,8 @@ TEST_F(PersistedContentAddressedIndexedTreeTest, test_remove_historical_blocks) check_historic_leaf(tree, leaf2AtBlock3, 2, 3, true); // should not be found at block 1 - check_historic_find_leaf_index_from(tree, PublicDataLeafValue(10, 20), 1, 0, 0, false); + check_historic_find_leaf_index_from( + tree, { PublicDataLeafValue(10, 20) }, 1, 0, { std::nullopt }, true); // should be found at block check_historic_find_leaf_index_from(tree, PublicDataLeafValue(10, 20), 2, 0, 3, true); @@ -2288,7 +2217,8 @@ TEST_F(PersistedContentAddressedIndexedTreeTest, test_remove_historical_blocks) // Historic queries against block 1 should no longer work check_historic_leaf(tree, leaf1AtBlock1, 1, 1, false); - check_historic_find_leaf_index(tree, leaf1AtBlock1, 1, 1, false); + check_historic_find_leaf_index( + tree, { leaf1AtBlock1 }, 1, { std::nullopt }, false); // Queries against block 2 should work check_historic_leaf(tree, leaf2AtBlock2, 2, 2, true); @@ -2312,8 +2242,9 @@ TEST_F(PersistedContentAddressedIndexedTreeTest, test_unwind_blocks) LMDBTreeStore::SharedPtr db = std::make_shared(_directory, name, _mapSize, _maxReaders); std::unique_ptr> store = std::make_unique>(name, depth, db); - auto tree = ContentAddressedIndexedTree, Poseidon2HashPolicy>( - std::move(store), workers, current_size); + using LocalTreeType = + ContentAddressedIndexedTree, Poseidon2HashPolicy>; + auto tree = LocalTreeType(std::move(store), workers, current_size); /** * Intial state: @@ -2477,7 +2408,8 @@ TEST_F(PersistedContentAddressedIndexedTreeTest, test_unwind_blocks) check_historic_leaf(tree, leaf2AtBlock3, 2, 3, true); // should not be found at block 1 - check_historic_find_leaf_index_from(tree, PublicDataLeafValue(10, 20), 1, 0, 0, false); + check_historic_find_leaf_index_from( + tree, { PublicDataLeafValue(10, 20) }, 1, 0, { std::nullopt }, true); // should be found at block check_historic_find_leaf_index_from(tree, PublicDataLeafValue(10, 20), 2, 0, 3, true); @@ -2516,8 +2448,10 @@ TEST_F(PersistedContentAddressedIndexedTreeTest, test_unwind_blocks) check_block_and_size_data(db, 3, current_size, true); // should fail to find the leaf at index 4 - check_find_leaf_index(tree, PublicDataLeafValue(50, 8), 4, false); - check_find_leaf_index_from(tree, PublicDataLeafValue(50, 8), 0, 5, false); + check_find_leaf_index( + tree, { PublicDataLeafValue(50, 8) }, { std::nullopt }, true); + check_find_leaf_index_from( + tree, { PublicDataLeafValue(50, 8) }, 0, { std::nullopt }, true); // the leaf at index 2 should no longer be as it was after block 5 EXPECT_NE(get_leaf(tree, 2), create_indexed_public_data_leaf(30, 6, 4, 50)); @@ -2797,8 +2731,8 @@ void test_nullifier_tree_unwind(std::string directory, // Trying to find leaves appended in the block that was removed should fail get_leaf(tree, 1 + deletedBlockStartIndex, false, false); - check_find_leaf_index( - tree, leafValues[1 + deletedBlockStartIndexIntoLocalValues], 1 + deletedBlockStartIndex, false); + check_find_leaf_index( + tree, { leafValues[1 + deletedBlockStartIndexIntoLocalValues] }, { std::nullopt }, true); } for (index_t j = 0; j < numBlocks; j++) { @@ -2817,10 +2751,15 @@ void test_nullifier_tree_unwind(std::string directory, const index_t expectedIndexInTree = leafIndex + batchSize; check_historic_leaf( tree, leafValues[leafIndex], expectedIndexInTree, historicBlockNumber, expectedSuccess, false); - check_historic_find_leaf_index( - tree, leafValues[leafIndex], historicBlockNumber, expectedIndexInTree, expectedSuccess, false); - check_historic_find_leaf_index_from( - tree, leafValues[leafIndex], historicBlockNumber, 0, expectedIndexInTree, expectedSuccess, false); + + std::vector> expectedResults; + if (expectedSuccess) { + expectedResults.emplace_back(std::make_optional(expectedIndexInTree)); + } + check_historic_find_leaf_index( + tree, { leafValues[leafIndex] }, historicBlockNumber, expectedResults, expectedSuccess, true); + check_historic_find_leaf_index_from( + tree, { leafValues[leafIndex] }, historicBlockNumber, 0, expectedResults, expectedSuccess, true); } } } diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp index abaec64a3c7..cdd5e102754 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp @@ -35,26 +35,6 @@ template <> struct std::hash { namespace bb::crypto::merkle_tree { -template fr preimage_to_key(const LeafType& leaf) -{ - return leaf.get_key(); -} - -inline fr preimage_to_key(const fr& leaf) -{ - return leaf; -} - -template bool requires_preimage_for_key() -{ - return true; -} - -template <> inline bool requires_preimage_for_key() -{ - return false; -} - /** * @brief Serves as a key-value node store for merkle trees. Caches all changes in memory before persisting them during * a 'commit' operation. @@ -154,8 +134,7 @@ template class ContentAddressedCachedTreeStore { */ std::optional find_leaf_index(const LeafValueType& leaf, const RequestContext& requestContext, - ReadTransaction& tx, - bool includeUncommitted) const; + ReadTransaction& tx) const; /** * @brief Finds the index of the given leaf value in the tree if available. Includes uncommitted data if requested. @@ -163,8 +142,7 @@ template class ContentAddressedCachedTreeStore { std::optional find_leaf_index_from(const LeafValueType& leaf, const index_t& start_index, const RequestContext& requestContext, - ReadTransaction& tx, - bool includeUncommitted) const; + ReadTransaction& tx) const; /** * @brief Commits the uncommitted data to the underlying store @@ -298,14 +276,13 @@ template index_t ContentAddressedCachedTreeStore::constrain_tree_size(const RequestContext& requestContext, ReadTransaction& tx) const { + // We need to identify the size of the committed tree as it exists from our perspective + // To do this we read the uncommitted meta which will contained the committed size at our initialisation point TreeMeta m; get_meta(m, tx, true); index_t sizeLimit = m.committedSize; - if (requestContext.blockNumber.has_value()) { - BlockPayload blockData; - if (dataStore_->read_block_data(requestContext.blockNumber.value(), blockData, tx)) { - sizeLimit = std::min(meta_.committedSize, blockData.size); - } + if (requestContext.maxIndex.has_value() && requestContext.maxIndex.value() < sizeLimit) { + sizeLimit = requestContext.maxIndex.value(); } return sizeLimit; } @@ -314,6 +291,12 @@ template std::optional ContentAddressedCachedTreeStore::find_block_for_index( const index_t& index, ReadTransaction& tx) const { + RequestContext context; + context.maxIndex = index + 1; + index_t constrainedSize = constrain_tree_size(context, tx); + if (index >= constrainedSize) { + return std::nullopt; + } block_number_t blockNumber = 0; bool success = dataStore_->find_block_for_index(index, blockNumber, tx); return success ? std::make_optional(blockNumber) : std::nullopt; @@ -341,10 +324,7 @@ std::pair ContentAddressedCachedTreeStore::find_lo { auto new_value_as_number = uint256_t(new_leaf_key); index_t committed = 0; - std::optional sizeLimit = std::nullopt; - if (initialised_from_block_.has_value() || requestContext.blockNumber.has_value()) { - sizeLimit = constrain_tree_size(requestContext, tx); - } + std::optional sizeLimit = constrain_tree_size(requestContext, tx); fr found_key = dataStore_->find_low_leaf(new_leaf_key, committed, sizeLimit, tx); index_t db_index = committed; @@ -458,9 +438,9 @@ void ContentAddressedCachedTreeStore::update_index(const index_t& template std::optional ContentAddressedCachedTreeStore::find_leaf_index( - const LeafValueType& leaf, const RequestContext& requestContext, ReadTransaction& tx, bool includeUncommitted) const + const LeafValueType& leaf, const RequestContext& requestContext, ReadTransaction& tx) const { - return find_leaf_index_from(leaf, 0, requestContext, tx, includeUncommitted); + return find_leaf_index_from(leaf, 0, requestContext, tx); } template @@ -468,10 +448,9 @@ std::optional ContentAddressedCachedTreeStore::find_leaf const LeafValueType& leaf, const index_t& start_index, const RequestContext& requestContext, - ReadTransaction& tx, - bool includeUncommitted) const + ReadTransaction& tx) const { - if (includeUncommitted) { + if (requestContext.includeUncommitted) { // Accessing indices_ under a lock std::unique_lock lock(mtx_); auto it = indices_.find(uint256_t(leaf)); diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp index d525acb8672..9af6f103f50 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp @@ -108,7 +108,7 @@ struct BlockForIndexResponse { }; struct FindLeafIndexResponse { - index_t leaf_index; + std::vector> leaf_indices; FindLeafIndexResponse() = default; ~FindLeafIndexResponse() = default; diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/test_fixtures.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/test_fixtures.hpp index 354ba949c20..db52d26b003 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/test_fixtures.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/test_fixtures.hpp @@ -3,10 +3,13 @@ #include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp" +#include "barretenberg/crypto/merkle_tree/response.hpp" +#include "barretenberg/crypto/merkle_tree/signal.hpp" #include "barretenberg/ecc/curves/bn254/fr.hpp" #include "barretenberg/numeric/random/engine.hpp" #include #include +#include namespace bb::crypto::merkle_tree { @@ -80,4 +83,141 @@ void check_leaf_by_hash(LMDBTreeStore::SharedPtr db, IndexedLeaf leaf, } } +template +void check_find_leaf_index(TypeOfTree& tree, + const std::vector& leaves, + const std::vector>& expected_indices, + bool expected_success, + bool includeUncommitted = true) +{ + Signal signal; + auto completion = [&](const TypedResponse& response) -> void { + EXPECT_EQ(response.success, expected_success); + if (expected_success) { + EXPECT_EQ(response.inner.leaf_indices, expected_indices); + } + signal.signal_level(); + }; + + tree.find_leaf_indices(leaves, includeUncommitted, completion); + signal.wait_for_level(); +} + +template +void check_find_leaf_index_from(TypeOfTree& tree, + const std::vector& leaves, + index_t start_index, + const std::vector>& expected_indices, + bool expected_success, + bool includeUncommitted = true) +{ + Signal signal; + auto completion = [&](const TypedResponse& response) -> void { + EXPECT_EQ(response.success, expected_success); + if (expected_success) { + EXPECT_EQ(response.inner.leaf_indices, expected_indices); + } + signal.signal_level(); + }; + + tree.find_leaf_indices_from(leaves, start_index, includeUncommitted, completion); + signal.wait_for_level(); +} + +template +void check_historic_find_leaf_index(TypeOfTree& tree, + const std::vector& leaves, + block_number_t blockNumber, + const std::vector>& expected_indices, + bool expected_success, + bool includeUncommitted = true) +{ + Signal signal; + auto completion = [&](const TypedResponse& response) -> void { + EXPECT_EQ(response.success, expected_success); + if (expected_success) { + EXPECT_EQ(response.inner.leaf_indices, expected_indices); + } + signal.signal_level(); + }; + + tree.find_leaf_indices(leaves, blockNumber, includeUncommitted, completion); + signal.wait_for_level(); +} + +template +void check_historic_find_leaf_index_from(TypeOfTree& tree, + const std::vector& leaves, + block_number_t blockNumber, + index_t start_index, + const std::vector>& expected_indices, + bool expected_success, + bool includeUncommitted = true) +{ + Signal signal; + auto completion = [&](const TypedResponse& response) -> void { + EXPECT_EQ(response.success, expected_success); + if (expected_success) { + EXPECT_EQ(response.inner.leaf_indices, expected_indices); + } + signal.signal_level(); + }; + + tree.find_leaf_indices_from(leaves, start_index, blockNumber, includeUncommitted, completion); + signal.wait_for_level(); +} + +template +void check_find_leaf_index(TypeOfTree& tree, + const LeafValueType& leaf, + index_t expected_index, + bool expected_success, + bool includeUncommitted = true) +{ + check_find_leaf_index( + tree, { leaf }, { std::make_optional(expected_index) }, expected_success, includeUncommitted); +} + +template +void check_find_leaf_index_from(TypeOfTree& tree, + const LeafValueType& leaf, + index_t start_index, + index_t expected_index, + bool expected_success, + bool includeUncommitted = true) +{ + check_find_leaf_index_from( + tree, { leaf }, start_index, { std::make_optional(expected_index) }, expected_success, includeUncommitted); +} + +template +void check_historic_find_leaf_index(TypeOfTree& tree, + const LeafValueType& leaf, + block_number_t blockNumber, + index_t expected_index, + bool expected_success, + bool includeUncommitted = true) +{ + check_historic_find_leaf_index( + tree, { leaf }, blockNumber, { std::make_optional(expected_index) }, expected_success, includeUncommitted); +} + +template +void check_historic_find_leaf_index_from(TypeOfTree& tree, + const LeafValueType& leaf, + block_number_t blockNumber, + index_t start_index, + index_t expected_index, + bool expected_success, + bool includeUncommitted = true) +{ + check_historic_find_leaf_index_from(tree, + { leaf }, + blockNumber, + start_index, + { std::make_optional(expected_index) }, + expected_success, + includeUncommitted); +} + } // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/types.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/types.hpp index c8ce520fb9c..54a1fa3e9be 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/types.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/types.hpp @@ -12,8 +12,39 @@ struct RequestContext { bool includeUncommitted; std::optional blockNumber; bb::fr root; + std::optional maxIndex; }; +template fr preimage_to_key(const LeafType& leaf) +{ + return leaf.get_key(); +} + +inline fr preimage_to_key(const fr& leaf) +{ + return leaf; +} + +template bool is_empty(const LeafType& leaf) +{ + return leaf.is_empty(); +} + +inline bool is_empty(const fr& leaf) +{ + return leaf == fr::zero(); +} + +template bool requires_preimage_for_key() +{ + return true; +} + +template <> inline bool requires_preimage_for_key() +{ + return false; +} + const std::string BLOCKS_DB = "blocks"; const std::string NODES_DB = "nodes"; const std::string LEAF_PREIMAGES_DB = "leaf preimages"; diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp b/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp index 7e74fe44896..033ad3a51c3 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp +++ b/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp @@ -913,19 +913,19 @@ bb::fr WorldState::compute_initial_archive(const StateReference& initial_state_r bool WorldState::is_archive_tip(const WorldStateRevision& revision, const bb::fr& block_header_hash) const { - std::optional leaf_index = std::nullopt; + std::vector> indices; try { - leaf_index = find_leaf_index(revision, MerkleTreeId::ARCHIVE, block_header_hash); + find_leaf_indices(revision, MerkleTreeId::ARCHIVE, { block_header_hash }, indices); } catch (std::runtime_error&) { } - if (!leaf_index.has_value()) { + if (indices.empty() || !indices[0].has_value()) { return false; } TreeMetaResponse archive_state = get_tree_info(revision, MerkleTreeId::ARCHIVE); - return archive_state.meta.size == leaf_index.value() + 1; + return archive_state.meta.size == indices[0].value() + 1; } void WorldState::get_status_summary(WorldStateStatusSummary& status) const diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp b/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp index c66412aae77..a87ff94db65 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp +++ b/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp @@ -156,15 +156,16 @@ class WorldState { * * @param revision The revision to query * @param tree_id The ID of the tree - * @param leaf The leaf to find + * @param leaves The leaves to find + * @param indices The indices to be updated * @param start_index The index to start searching from - * @return std::optional */ template - std::optional find_leaf_index(const WorldStateRevision& revision, - MerkleTreeId tree_id, - const T& leaf, - index_t start_index = 0) const; + void find_leaf_indices(const WorldStateRevision& revision, + MerkleTreeId tree_id, + const std::vector& leaves, + std::vector>& indices, + index_t start_index = 0) const; /** * @brief Appends a set of leaves to an existing Merkle Tree. @@ -475,10 +476,11 @@ std::optional WorldState::get_leaf(const WorldStateRevision& revision, } template -std::optional WorldState::find_leaf_index(const WorldStateRevision& rev, - MerkleTreeId id, - const T& leaf, - index_t start_index) const +void WorldState::find_leaf_indices(const WorldStateRevision& rev, + MerkleTreeId id, + const std::vector& leaves, + std::vector>& indices, + index_t start_index) const { using namespace crypto::merkle_tree; @@ -493,9 +495,10 @@ std::optional WorldState::find_leaf_index(const WorldStateRevision& rev if constexpr (std::is_same_v) { const auto& wrapper = std::get>(fork->_trees.at(id)); if (rev.blockNumber) { - wrapper.tree->find_leaf_index_from(leaf, start_index, rev.blockNumber, rev.includeUncommitted, callback); + wrapper.tree->find_leaf_indices_from( + leaves, start_index, rev.blockNumber, rev.includeUncommitted, callback); } else { - wrapper.tree->find_leaf_index_from(leaf, start_index, rev.includeUncommitted, callback); + wrapper.tree->find_leaf_indices_from(leaves, start_index, rev.includeUncommitted, callback); } } else { @@ -504,19 +507,20 @@ std::optional WorldState::find_leaf_index(const WorldStateRevision& rev auto& wrapper = std::get>(fork->_trees.at(id)); if (rev.blockNumber) { - wrapper.tree->find_leaf_index_from(leaf, rev.blockNumber, start_index, rev.includeUncommitted, callback); + wrapper.tree->find_leaf_indices_from( + leaves, start_index, rev.blockNumber, rev.includeUncommitted, callback); } else { - wrapper.tree->find_leaf_index_from(leaf, start_index, rev.includeUncommitted, callback); + wrapper.tree->find_leaf_indices_from(leaves, start_index, rev.includeUncommitted, callback); } } signal.wait_for_level(0); - if (!local.success) { - return std::nullopt; + if (!local.success || local.inner.leaf_indices.size() != leaves.size()) { + throw std::runtime_error(local.message); } - return local.inner.leaf_index; + indices = std::move(local.inner.leaf_indices); } template void WorldState::append_leaves(MerkleTreeId id, const std::vector& leaves, Fork::Id fork_id) diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp b/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp index a5ced2921ad..8d5e67f89d1 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp +++ b/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -72,20 +73,24 @@ template void assert_leaf_exists( const WorldState& ws, WorldStateRevision revision, MerkleTreeId tree_id, const Leaf& expected_value, bool exists) { - std::optional index = ws.find_leaf_index(revision, tree_id, expected_value); - EXPECT_EQ(index.has_value(), exists); + std::vector> indices; + ws.find_leaf_indices(revision, tree_id, { expected_value }, indices); + EXPECT_EQ(indices.size(), 1); + EXPECT_EQ(indices[0].has_value(), exists); } template void assert_leaf_index( const WorldState& ws, WorldStateRevision revision, MerkleTreeId tree_id, const Leaf& value, index_t expected_index) { - std::optional index = ws.find_leaf_index(revision, tree_id, value); - EXPECT_TRUE(index.has_value()); - if (!index.has_value()) { + std::vector> indices; + ws.find_leaf_indices(revision, tree_id, { value }, indices); + EXPECT_EQ(indices.size(), 1); + EXPECT_TRUE(indices[0].has_value()); + if (!indices[0].has_value()) { return; } - EXPECT_EQ(index.value(), expected_index); + EXPECT_EQ(indices[0].value(), expected_index); } void assert_tree_size(const WorldState& ws, WorldStateRevision revision, MerkleTreeId tree_id, size_t expected_size) @@ -689,7 +694,11 @@ TEST_F(WorldStateTest, SyncEmptyBlock) ws.sync_block(block_state_ref, fr(1), {}, {}, {}, {}); StateReference after_sync = ws.get_state_reference(WorldStateRevision::committed()); EXPECT_EQ(block_state_ref, after_sync); - EXPECT_EQ(ws.find_leaf_index(WorldStateRevision::committed(), MerkleTreeId::ARCHIVE, fr(1)), 1); + + std::vector> indices; + ws.find_leaf_indices(WorldStateRevision::committed(), MerkleTreeId::ARCHIVE, { fr(1) }, indices); + std::vector> expected{ std::make_optional(1) }; + EXPECT_EQ(indices, expected); } TEST_F(WorldStateTest, ForkingAtBlock0SameState) diff --git a/barretenberg/cpp/src/barretenberg/world_state_napi/addon.cpp b/barretenberg/cpp/src/barretenberg/world_state_napi/addon.cpp index f3290da5e9e..1f343e32e2f 100644 --- a/barretenberg/cpp/src/barretenberg/world_state_napi/addon.cpp +++ b/barretenberg/cpp/src/barretenberg/world_state_napi/addon.cpp @@ -156,8 +156,8 @@ WorldStateAddon::WorldStateAddon(const Napi::CallbackInfo& info) }); _dispatcher.registerTarget( - WorldStateMessageType::FIND_LEAF_INDEX, - [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return find_leaf_index(obj, buffer); }); + WorldStateMessageType::FIND_LEAF_INDICES, + [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return find_leaf_indices(obj, buffer); }); _dispatcher.registerTarget( WorldStateMessageType::FIND_LOW_LEAF, @@ -410,38 +410,43 @@ bool WorldStateAddon::get_block_numbers_for_leaf_indices(msgpack::object& obj, m return true; } -bool WorldStateAddon::find_leaf_index(msgpack::object& obj, msgpack::sbuffer& buffer) const +bool WorldStateAddon::find_leaf_indices(msgpack::object& obj, msgpack::sbuffer& buffer) const { TypedMessage request; obj.convert(request); - std::optional index; + FindLeafIndicesResponse response; + switch (request.value.treeId) { case MerkleTreeId::NOTE_HASH_TREE: case MerkleTreeId::L1_TO_L2_MESSAGE_TREE: case MerkleTreeId::ARCHIVE: { - TypedMessage> r1; + TypedMessage> r1; obj.convert(r1); - index = _ws->find_leaf_index(request.value.revision, request.value.treeId, r1.value.leaf); + _ws->find_leaf_indices( + request.value.revision, request.value.treeId, r1.value.leaves, response.indices, r1.value.startIndex); break; } case MerkleTreeId::PUBLIC_DATA_TREE: { - TypedMessage> r2; + TypedMessage> r2; obj.convert(r2); - index = _ws->find_leaf_index(request.value.revision, request.value.treeId, r2.value.leaf); + _ws->find_leaf_indices( + request.value.revision, request.value.treeId, r2.value.leaves, response.indices, r2.value.startIndex); break; } case MerkleTreeId::NULLIFIER_TREE: { - TypedMessage> r3; + TypedMessage> r3; obj.convert(r3); - index = _ws->find_leaf_index(request.value.revision, request.value.treeId, r3.value.leaf); + _ws->find_leaf_indices( + request.value.revision, request.value.treeId, r3.value.leaves, response.indices, r3.value.startIndex); break; } } MsgHeader header(request.header.messageId); - messaging::TypedMessage> resp_msg(WorldStateMessageType::FIND_LEAF_INDEX, header, index); + messaging::TypedMessage resp_msg( + WorldStateMessageType::FIND_LEAF_INDICES, header, response); msgpack::pack(buffer, resp_msg); return true; diff --git a/barretenberg/cpp/src/barretenberg/world_state_napi/addon.hpp b/barretenberg/cpp/src/barretenberg/world_state_napi/addon.hpp index d0b33be2532..14318c1bb20 100644 --- a/barretenberg/cpp/src/barretenberg/world_state_napi/addon.hpp +++ b/barretenberg/cpp/src/barretenberg/world_state_napi/addon.hpp @@ -40,7 +40,7 @@ class WorldStateAddon : public Napi::ObjectWrap { bool get_sibling_path(msgpack::object& obj, msgpack::sbuffer& buffer) const; bool get_block_numbers_for_leaf_indices(msgpack::object& obj, msgpack::sbuffer& buffer) const; - bool find_leaf_index(msgpack::object& obj, msgpack::sbuffer& buffer) const; + bool find_leaf_indices(msgpack::object& obj, msgpack::sbuffer& buffer) const; bool find_low_leaf(msgpack::object& obj, msgpack::sbuffer& buffer) const; bool append_leaves(msgpack::object& obj, msgpack::sbuffer& buffer); diff --git a/barretenberg/cpp/src/barretenberg/world_state_napi/message.hpp b/barretenberg/cpp/src/barretenberg/world_state_napi/message.hpp index b98a8c6a69d..d903ed7dc2f 100644 --- a/barretenberg/cpp/src/barretenberg/world_state_napi/message.hpp +++ b/barretenberg/cpp/src/barretenberg/world_state_napi/message.hpp @@ -23,7 +23,7 @@ enum WorldStateMessageType { GET_SIBLING_PATH, GET_BLOCK_NUMBERS_FOR_LEAF_INDICES, - FIND_LEAF_INDEX, + FIND_LEAF_INDICES, FIND_LOW_LEAF, APPEND_LEAVES, @@ -143,11 +143,17 @@ struct GetBlockNumbersForLeafIndicesResponse { MSGPACK_FIELDS(blockNumbers); }; -template struct FindLeafIndexRequest { +template struct FindLeafIndicesRequest { MerkleTreeId treeId; WorldStateRevision revision; - T leaf; - MSGPACK_FIELDS(treeId, revision, leaf); + std::vector leaves; + index_t startIndex; + MSGPACK_FIELDS(treeId, revision, leaves, startIndex); +}; + +struct FindLeafIndicesResponse { + std::vector> indices; + MSGPACK_FIELDS(indices); }; struct FindLowLeafRequest { diff --git a/yarn-project/aztec-node/src/aztec-node/server.test.ts b/yarn-project/aztec-node/src/aztec-node/server.test.ts index 649c9bfe8fe..942c544b26f 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.test.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.test.ts @@ -39,6 +39,10 @@ describe('aztec node', () => { globalVariablesBuilder = mock(); merkleTreeOps = mock(); + merkleTreeOps.findLeafIndices.mockImplementation((_treeId: MerkleTreeId, _value: any[]) => { + return Promise.resolve([undefined]); + }); + const worldState = mock({ getCommitted: () => merkleTreeOps, }); @@ -106,9 +110,9 @@ describe('aztec node', () => { // We make a nullifier from `doubleSpendWithExistingTx` a part of the nullifier tree, so it gets rejected as double spend const doubleSpendNullifier = doubleSpendWithExistingTx.data.forRollup!.end.nullifiers[0].toBuffer(); - merkleTreeOps.findLeafIndex.mockImplementation((treeId: MerkleTreeId, value: any) => { + merkleTreeOps.findLeafIndices.mockImplementation((treeId: MerkleTreeId, value: any[]) => { return Promise.resolve( - treeId === MerkleTreeId.NULLIFIER_TREE && value.equals(doubleSpendNullifier) ? 1n : undefined, + treeId === MerkleTreeId.NULLIFIER_TREE && value[0].equals(doubleSpendNullifier) ? [1n] : [undefined], ); }); diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index 35476df3fc8..fa7904296cc 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -457,7 +457,10 @@ export class AztecNodeService implements AztecNode { leafValues: Fr[], ): Promise<(bigint | undefined)[]> { const committedDb = await this.#getWorldState(blockNumber); - return await Promise.all(leafValues.map(leafValue => committedDb.findLeafIndex(treeId, leafValue.toBuffer()))); + return await committedDb.findLeafIndices( + treeId, + leafValues.map(x => x.toBuffer()), + ); } /** @@ -664,7 +667,7 @@ export class AztecNodeService implements AztecNode { nullifier: Fr, ): Promise { const db = await this.#getWorldState(blockNumber); - const index = await db.findLeafIndex(MerkleTreeId.NULLIFIER_TREE, nullifier.toBuffer()); + const index = (await db.findLeafIndices(MerkleTreeId.NULLIFIER_TREE, [nullifier.toBuffer()]))[0]; if (!index) { return undefined; } @@ -837,7 +840,7 @@ export class AztecNodeService implements AztecNode { new MetadataTxValidator(new Fr(this.l1ChainId), new Fr(blockNumber)), new DoubleSpendTxValidator({ getNullifierIndex(nullifier) { - return db.findLeafIndex(MerkleTreeId.NULLIFIER_TREE, nullifier.toBuffer()); + return db.findLeafIndices(MerkleTreeId.NULLIFIER_TREE, [nullifier.toBuffer()]).then(x => x[0]); }, }), ]; diff --git a/yarn-project/circuit-types/src/interfaces/merkle_tree_operations.ts b/yarn-project/circuit-types/src/interfaces/merkle_tree_operations.ts index db5728ac211..288c16ff3ae 100644 --- a/yarn-project/circuit-types/src/interfaces/merkle_tree_operations.ts +++ b/yarn-project/circuit-types/src/interfaces/merkle_tree_operations.ts @@ -176,7 +176,10 @@ export interface MerkleTreeReadOperations { * @param treeId - The tree for which the index should be returned. * @param value - The value to search for in the tree. */ - findLeafIndex(treeId: ID, value: MerkleTreeLeafType): Promise; + findLeafIndices( + treeId: ID, + values: MerkleTreeLeafType[], + ): Promise<(bigint | undefined)[]>; /** * Returns the first index containing a leaf value after `startIndex`. @@ -184,11 +187,11 @@ export interface MerkleTreeReadOperations { * @param value - The value to search for in the tree. * @param startIndex - The index to start searching from (used when skipping nullified messages) */ - findLeafIndexAfter( + findLeafIndicesAfter( treeId: ID, - value: MerkleTreeLeafType, + values: MerkleTreeLeafType[], startIndex: bigint, - ): Promise; + ): Promise<(bigint | undefined)[]>; /** * Gets the value for a leaf in the tree. diff --git a/yarn-project/p2p/src/service/libp2p_service.ts b/yarn-project/p2p/src/service/libp2p_service.ts index b0f10aec8bf..03bf7fc4ae6 100644 --- a/yarn-project/p2p/src/service/libp2p_service.ts +++ b/yarn-project/p2p/src/service/libp2p_service.ts @@ -538,7 +538,7 @@ export class LibP2PService extends WithTracer implements P2PService { const doubleSpendValidator = new DoubleSpendTxValidator({ getNullifierIndex: async (nullifier: Fr) => { const merkleTree = this.worldStateSynchronizer.getCommitted(); - const index = await merkleTree.findLeafIndex(MerkleTreeId.NULLIFIER_TREE, nullifier.toBuffer()); + const index = (await merkleTree.findLeafIndices(MerkleTreeId.NULLIFIER_TREE, [nullifier.toBuffer()]))[0]; return index; }, }); @@ -551,7 +551,7 @@ export class LibP2PService extends WithTracer implements P2PService { const merkleTree = this.worldStateSynchronizer.getSnapshot( blockNumber - this.config.severePeerPenaltyBlockLength, ); - const index = await merkleTree.findLeafIndex(MerkleTreeId.NULLIFIER_TREE, nullifier.toBuffer()); + const index = (await merkleTree.findLeafIndices(MerkleTreeId.NULLIFIER_TREE, [nullifier.toBuffer()]))[0]; return index; }, }); diff --git a/yarn-project/prover-client/src/orchestrator/block-building-helpers.ts b/yarn-project/prover-client/src/orchestrator/block-building-helpers.ts index d6c913125ca..67e23655c88 100644 --- a/yarn-project/prover-client/src/orchestrator/block-building-helpers.ts +++ b/yarn-project/prover-client/src/orchestrator/block-building-helpers.ts @@ -547,7 +547,7 @@ export async function getMembershipWitnessFor( return makeEmptyMembershipWitness(height); } - const index = await db.findLeafIndex(treeId, value.toBuffer()); + const index = (await db.findLeafIndices(treeId, [value.toBuffer()]))[0]; if (index === undefined) { throw new Error(`Leaf with value ${value} not found in tree ${MerkleTreeId[treeId]}`); } diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts index aeb73d79725..1807a42a5e1 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts @@ -127,6 +127,10 @@ describe('sequencer', () => { merkleTreeOps = mock(); blockBuilder = mock(); + merkleTreeOps.findLeafIndices.mockImplementation((_treeId: MerkleTreeId, _value: any[]) => { + return Promise.resolve([undefined]); + }); + p2p = mock({ getStatus: mockFn().mockResolvedValue({ state: P2PClientState.IDLE, @@ -315,9 +319,9 @@ describe('sequencer', () => { // We make a nullifier from tx1 a part of the nullifier tree, so it gets rejected as double spend const doubleSpendNullifier = doubleSpendTx.data.forRollup!.end.nullifiers[0].toBuffer(); - merkleTreeOps.findLeafIndex.mockImplementation((treeId: MerkleTreeId, value: any) => { + merkleTreeOps.findLeafIndices.mockImplementation((treeId: MerkleTreeId, value: any[]) => { return Promise.resolve( - treeId === MerkleTreeId.NULLIFIER_TREE && value.equals(doubleSpendNullifier) ? 1n : undefined, + treeId === MerkleTreeId.NULLIFIER_TREE && value[0].equals(doubleSpendNullifier) ? [1n] : [undefined], ); }); diff --git a/yarn-project/sequencer-client/src/tx_validator/tx_validator_factory.ts b/yarn-project/sequencer-client/src/tx_validator/tx_validator_factory.ts index d0bab6af43e..dd36e4ebc3d 100644 --- a/yarn-project/sequencer-client/src/tx_validator/tx_validator_factory.ts +++ b/yarn-project/sequencer-client/src/tx_validator/tx_validator_factory.ts @@ -29,7 +29,8 @@ export class TxValidatorFactory { private enforceFees: boolean, ) { this.nullifierSource = { - getNullifierIndex: nullifier => this.committedDb.findLeafIndex(MerkleTreeId.NULLIFIER_TREE, nullifier.toBuffer()), + getNullifierIndex: nullifier => + this.committedDb.findLeafIndices(MerkleTreeId.NULLIFIER_TREE, [nullifier.toBuffer()]).then(x => x[0]), }; this.publicStateSource = { @@ -51,7 +52,8 @@ export class TxValidatorFactory { validatorForProcessedTxs(fork: MerkleTreeReadOperations): TxValidator { return new DoubleSpendTxValidator({ - getNullifierIndex: nullifier => fork.findLeafIndex(MerkleTreeId.NULLIFIER_TREE, nullifier.toBuffer()), + getNullifierIndex: nullifier => + fork.findLeafIndices(MerkleTreeId.NULLIFIER_TREE, [nullifier.toBuffer()]).then(x => x[0]), }); } } diff --git a/yarn-project/simulator/src/public/public_db_sources.ts b/yarn-project/simulator/src/public/public_db_sources.ts index ab9add3a40b..5e0584a7a1b 100644 --- a/yarn-project/simulator/src/public/public_db_sources.ts +++ b/yarn-project/simulator/src/public/public_db_sources.ts @@ -207,7 +207,7 @@ export class WorldStateDB extends ContractsDataSourcePublicDB implements PublicS nullifier: Fr, ): Promise { const timer = new Timer(); - const index = await this.db.findLeafIndex(MerkleTreeId.NULLIFIER_TREE, nullifier.toBuffer()); + const index = (await this.db.findLeafIndices(MerkleTreeId.NULLIFIER_TREE, [nullifier.toBuffer()]))[0]; if (!index) { return undefined; } @@ -240,7 +240,7 @@ export class WorldStateDB extends ContractsDataSourcePublicDB implements PublicS ): Promise> { const timer = new Timer(); - const messageIndex = await this.db.findLeafIndex(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, messageHash); + const messageIndex = (await this.db.findLeafIndices(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, [messageHash]))[0]; if (messageIndex === undefined) { throw new Error(`No L1 to L2 message found for message hash ${messageHash.toString()}`); } @@ -279,7 +279,7 @@ export class WorldStateDB extends ContractsDataSourcePublicDB implements PublicS public async getCommitmentIndex(commitment: Fr): Promise { const timer = new Timer(); - const index = await this.db.findLeafIndex(MerkleTreeId.NOTE_HASH_TREE, commitment); + const index = (await this.db.findLeafIndices(MerkleTreeId.NOTE_HASH_TREE, [commitment]))[0]; this.logger.debug(`[DB] Fetched commitment index`, { eventName: 'public-db-access', duration: timer.ms(), @@ -301,7 +301,7 @@ export class WorldStateDB extends ContractsDataSourcePublicDB implements PublicS public async getNullifierIndex(nullifier: Fr): Promise { const timer = new Timer(); - const index = await this.db.findLeafIndex(MerkleTreeId.NULLIFIER_TREE, nullifier.toBuffer()); + const index = (await this.db.findLeafIndices(MerkleTreeId.NULLIFIER_TREE, [nullifier.toBuffer()]))[0]; this.logger.debug(`[DB] Fetched nullifier index`, { eventName: 'public-db-access', duration: timer.ms(), diff --git a/yarn-project/txe/src/oracle/txe_oracle.ts b/yarn-project/txe/src/oracle/txe_oracle.ts index 35340639972..38efd48367b 100644 --- a/yarn-project/txe/src/oracle/txe_oracle.ts +++ b/yarn-project/txe/src/oracle/txe_oracle.ts @@ -369,8 +369,7 @@ export class TXE implements TypedOracle { async getMembershipWitness(blockNumber: number, treeId: MerkleTreeId, leafValue: Fr): Promise { const db = await this.#getTreesAt(blockNumber); - - const index = await db.findLeafIndex(treeId, leafValue.toBuffer()); + const index = (await db.findLeafIndices(treeId, [leafValue.toBuffer()]))[0]; if (index === undefined) { throw new Error(`Leaf value: ${leafValue} not found in ${MerkleTreeId[treeId]} at block ${blockNumber}`); } @@ -390,7 +389,7 @@ export class TXE implements TypedOracle { nullifier: Fr, ): Promise { const db = await this.#getTreesAt(blockNumber); - const index = await db.findLeafIndex(MerkleTreeId.NULLIFIER_TREE, nullifier.toBuffer()); + const index = (await db.findLeafIndices(MerkleTreeId.NULLIFIER_TREE, [nullifier.toBuffer()]))[0]; if (!index) { return undefined; } @@ -540,7 +539,7 @@ export class TXE implements TypedOracle { async checkNullifierExists(innerNullifier: Fr): Promise { const nullifier = siloNullifier(this.contractAddress, innerNullifier!); const db = await this.trees.getLatest(); - const index = await db.findLeafIndex(MerkleTreeId.NULLIFIER_TREE, nullifier.toBuffer()); + const index = (await db.findLeafIndices(MerkleTreeId.NULLIFIER_TREE, [nullifier.toBuffer()]))[0]; return index !== undefined; } @@ -976,7 +975,7 @@ export class TXE implements TypedOracle { async avmOpcodeNullifierExists(innerNullifier: Fr, targetAddress: AztecAddress): Promise { const nullifier = siloNullifier(targetAddress, innerNullifier!); const db = await this.trees.getLatest(); - const index = await db.findLeafIndex(MerkleTreeId.NULLIFIER_TREE, nullifier.toBuffer()); + const index = (await db.findLeafIndices(MerkleTreeId.NULLIFIER_TREE, [nullifier.toBuffer()]))[0]; return index !== undefined; } diff --git a/yarn-project/world-state/src/native/merkle_trees_facade.ts b/yarn-project/world-state/src/native/merkle_trees_facade.ts index 6d3ea76d7fd..28e5886de1d 100644 --- a/yarn-project/world-state/src/native/merkle_trees_facade.ts +++ b/yarn-project/world-state/src/native/merkle_trees_facade.ts @@ -45,27 +45,29 @@ export class MerkleTreesFacade implements MerkleTreeReadOperations { return this.initialHeader; } - findLeafIndex(treeId: MerkleTreeId, value: MerkleTreeLeafType): Promise { - return this.findLeafIndexAfter(treeId, value, 0n); + findLeafIndices(treeId: MerkleTreeId, values: MerkleTreeLeafType[]): Promise<(bigint | undefined)[]> { + return this.findLeafIndicesAfter(treeId, values, 0n); } - async findLeafIndexAfter( + async findLeafIndicesAfter( treeId: MerkleTreeId, - leaf: MerkleTreeLeafType, + leaves: MerkleTreeLeafType[], startIndex: bigint, - ): Promise { - const index = await this.instance.call(WorldStateMessageType.FIND_LEAF_INDEX, { - leaf: serializeLeaf(hydrateLeaf(treeId, leaf)), + ): Promise<(bigint | undefined)[]> { + const response = await this.instance.call(WorldStateMessageType.FIND_LEAF_INDICES, { + leaves: leaves.map(leaf => serializeLeaf(hydrateLeaf(treeId, leaf))), revision: this.revision, treeId, startIndex, }); - if (typeof index === 'number' || typeof index === 'bigint') { - return BigInt(index); - } else { - return undefined; - } + return response.indices.map(index => { + if (typeof index === 'number' || typeof index === 'bigint') { + return BigInt(index); + } else { + return undefined; + } + }); } async getLeafPreimage(treeId: IndexedTreeId, leafIndex: bigint): Promise { diff --git a/yarn-project/world-state/src/native/message.ts b/yarn-project/world-state/src/native/message.ts index 6f95755ce7c..6cd4b4f751a 100644 --- a/yarn-project/world-state/src/native/message.ts +++ b/yarn-project/world-state/src/native/message.ts @@ -56,7 +56,7 @@ export enum WorldStateMessageType { GET_SIBLING_PATH, GET_BLOCK_NUMBERS_FOR_LEAF_INDICES, - FIND_LEAF_INDEX, + FIND_LEAF_INDICES, FIND_LOW_LEAF, APPEND_LEAVES, @@ -329,8 +329,8 @@ export type SerializedIndexedLeaf = { nextValue: Buffer; // Fr }; -interface WithLeafValue { - leaf: SerializedLeafValue; +interface WithLeafValues { + leaves: SerializedLeafValue[]; } interface BlockShiftRequest { @@ -371,10 +371,12 @@ type GetLeafResponse = SerializedLeafValue | undefined; interface GetLeafPreImageRequest extends WithTreeId, WithLeafIndex, WithWorldStateRevision {} type GetLeafPreImageResponse = SerializedIndexedLeaf | undefined; -interface FindLeafIndexRequest extends WithTreeId, WithLeafValue, WithWorldStateRevision { +interface FindLeafIndicesRequest extends WithTreeId, WithLeafValues, WithWorldStateRevision { startIndex: bigint; } -type FindLeafIndexResponse = bigint | null; +interface FindLeafIndicesResponse { + indices: bigint[]; +} interface FindLowLeafRequest extends WithTreeId, WithWorldStateRevision { key: Fr; @@ -461,7 +463,7 @@ export type WorldStateRequest = { [WorldStateMessageType.GET_SIBLING_PATH]: GetSiblingPathRequest; [WorldStateMessageType.GET_BLOCK_NUMBERS_FOR_LEAF_INDICES]: GetBlockNumbersForLeafIndicesRequest; - [WorldStateMessageType.FIND_LEAF_INDEX]: FindLeafIndexRequest; + [WorldStateMessageType.FIND_LEAF_INDICES]: FindLeafIndicesRequest; [WorldStateMessageType.FIND_LOW_LEAF]: FindLowLeafRequest; [WorldStateMessageType.APPEND_LEAVES]: AppendLeavesRequest; @@ -497,7 +499,7 @@ export type WorldStateResponse = { [WorldStateMessageType.GET_SIBLING_PATH]: GetSiblingPathResponse; [WorldStateMessageType.GET_BLOCK_NUMBERS_FOR_LEAF_INDICES]: GetBlockNumbersForLeafIndicesResponse; - [WorldStateMessageType.FIND_LEAF_INDEX]: FindLeafIndexResponse; + [WorldStateMessageType.FIND_LEAF_INDICES]: FindLeafIndicesResponse; [WorldStateMessageType.FIND_LOW_LEAF]: FindLowLeafResponse; [WorldStateMessageType.APPEND_LEAVES]: void; diff --git a/yarn-project/world-state/src/native/native_world_state.test.ts b/yarn-project/world-state/src/native/native_world_state.test.ts index ff2767e5971..0fc564db7a1 100644 --- a/yarn-project/world-state/src/native/native_world_state.test.ts +++ b/yarn-project/world-state/src/native/native_world_state.test.ts @@ -47,6 +47,14 @@ describe('NativeWorldState', () => { let block: L2Block; let messages: Fr[]; + const findLeafIndex = async (leaf: Fr, ws: NativeWorldStateService) => { + const indices = await ws.getCommitted().findLeafIndices(MerkleTreeId.NOTE_HASH_TREE, [leaf]); + if (indices.length === 0) { + return undefined; + } + return indices[0]; + }; + beforeAll(async () => { const ws = await NativeWorldStateService.new(rollupAddress, dataDir, defaultDBMapSize); const fork = await ws.fork(); @@ -59,9 +67,7 @@ describe('NativeWorldState', () => { it('correctly restores committed state', async () => { const ws = await NativeWorldStateService.new(rollupAddress, dataDir, defaultDBMapSize); - await expect( - ws.getCommitted().findLeafIndex(MerkleTreeId.NOTE_HASH_TREE, block.body.txEffects[0].noteHashes[0]), - ).resolves.toBeDefined(); + await expect(findLeafIndex(block.body.txEffects[0].noteHashes[0], ws)).resolves.toBeDefined(); const status = await ws.getStatusSummary(); expect(status.unfinalisedBlockNumber).toBe(1n); await ws.close(); @@ -71,18 +77,14 @@ describe('NativeWorldState', () => { // open ws against the same data dir but a different rollup let ws = await NativeWorldStateService.new(EthAddress.random(), dataDir, defaultDBMapSize); // db should be empty - await expect( - ws.getCommitted().findLeafIndex(MerkleTreeId.NOTE_HASH_TREE, block.body.txEffects[0].noteHashes[0]), - ).resolves.toBeUndefined(); + await expect(findLeafIndex(block.body.txEffects[0].noteHashes[0], ws)).resolves.toBeUndefined(); await ws.close(); // later on, open ws against the original rollup and same data dir // db should be empty because we wiped all its files earlier ws = await NativeWorldStateService.new(rollupAddress, dataDir, defaultDBMapSize); - await expect( - ws.getCommitted().findLeafIndex(MerkleTreeId.NOTE_HASH_TREE, block.body.txEffects[0].noteHashes[0]), - ).resolves.toBeUndefined(); + await expect(findLeafIndex(block.body.txEffects[0].noteHashes[0], ws)).resolves.toBeUndefined(); const status = await ws.getStatusSummary(); expect(status.unfinalisedBlockNumber).toBe(0n); await ws.close(); @@ -490,6 +492,43 @@ describe('NativeWorldState', () => { }); }); + describe('finding leaves', () => { + let block: L2Block; + let messages: Fr[]; + + it('retrieves leaf indices', async () => { + const ws = await NativeWorldStateService.new(rollupAddress, dataDir, defaultDBMapSize); + const numBlocks = 2; + const txsPerBlock = 2; + const noteHashes: Fr[] = []; + for (let i = 0; i < numBlocks; i++) { + const fork = await ws.fork(); + ({ block, messages } = await mockBlock(1, txsPerBlock, fork)); + noteHashes.push(...block.body.txEffects.flatMap(x => x.noteHashes.flatMap(x => x))); + await fork.close(); + await ws.handleL2BlockAndMessages(block, messages); + } + + const leavesToRequest: Fr[] = [ + noteHashes[0], + Fr.random(), + noteHashes[45], + noteHashes[89], + Fr.random(), + noteHashes[102], + ]; + const expectedIndices = [0n, undefined, 45n, 89n, undefined, 102n]; + const indices = await ws.getCommitted().findLeafIndices(MerkleTreeId.NOTE_HASH_TREE, leavesToRequest); + expect(indices).toEqual(expectedIndices); + + const expectedIndicesAfter = [undefined, undefined, undefined, 89n, undefined, 102n]; + const indicesAfter = await ws + .getCommitted() + .findLeafIndicesAfter(MerkleTreeId.NOTE_HASH_TREE, leavesToRequest, 89n); + expect(indicesAfter).toEqual(expectedIndicesAfter); + }); + }); + describe('block numbers for indices', () => { let block: L2Block; let messages: Fr[]; diff --git a/yarn-project/world-state/src/native/native_world_state.ts b/yarn-project/world-state/src/native/native_world_state.ts index b7a996470e3..332a22008be 100644 --- a/yarn-project/world-state/src/native/native_world_state.ts +++ b/yarn-project/world-state/src/native/native_world_state.ts @@ -136,7 +136,8 @@ export class NativeWorldStateService implements MerkleTreeDatabase { // the initial header _must_ be the first element in the archive tree // if this assertion fails, check that the hashing done in Header in yarn-project matches the initial header hash done in world_state.cpp - const initialHeaderIndex = await committed.findLeafIndex(MerkleTreeId.ARCHIVE, this.initialHeader.hash()); + const indices = await committed.findLeafIndices(MerkleTreeId.ARCHIVE, [this.initialHeader.hash()]); + const initialHeaderIndex = indices[0]; assert.strictEqual(initialHeaderIndex, 0n, 'Invalid initial archive state'); } diff --git a/yarn-project/world-state/src/native/native_world_state_instance.ts b/yarn-project/world-state/src/native/native_world_state_instance.ts index d36c39740c7..4c91b95664a 100644 --- a/yarn-project/world-state/src/native/native_world_state_instance.ts +++ b/yarn-project/world-state/src/native/native_world_state_instance.ts @@ -180,17 +180,6 @@ export class NativeWorldState implements NativeWorldStateInstance { data['blockHeaderHash'] = '0x' + body.blockHeaderHash.toString('hex'); } - if ('leaf' in body) { - if (Buffer.isBuffer(body.leaf)) { - data['leaf'] = '0x' + body.leaf.toString('hex'); - } else if ('slot' in body.leaf) { - data['slot'] = '0x' + body.leaf.slot.toString('hex'); - data['value'] = '0x' + body.leaf.value.toString('hex'); - } else { - data['nullifier'] = '0x' + body.leaf.value.toString('hex'); - } - } - if ('leaves' in body) { data['leavesCount'] = body.leaves.length; } diff --git a/yarn-project/world-state/src/world-state-db/merkle_tree_operations_facade.ts b/yarn-project/world-state/src/world-state-db/merkle_tree_operations_facade.ts index 69f01189aef..c4900241a62 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_tree_operations_facade.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_tree_operations_facade.ts @@ -110,8 +110,11 @@ export class MerkleTreeReadOperationsFacade implements MerkleTreeWriteOperations * @param value - The leaf value to look for. * @returns The index of the first leaf found with a given value (undefined if not found). */ - findLeafIndex(treeId: ID, value: MerkleTreeLeafType): Promise { - return this.trees.findLeafIndex(treeId, value, this.includeUncommitted); + findLeafIndices( + treeId: ID, + values: MerkleTreeLeafType[], + ): Promise<(bigint | undefined)[]> { + return Promise.all(values.map(leaf => this.trees.findLeafIndex(treeId, leaf, this.includeUncommitted))); } /** @@ -120,12 +123,14 @@ export class MerkleTreeReadOperationsFacade implements MerkleTreeWriteOperations * @param value - The value to search for in the tree. * @param startIndex - The index to start searching from (used when skipping nullified messages) */ - findLeafIndexAfter( + findLeafIndicesAfter( treeId: ID, - value: MerkleTreeLeafType, + values: MerkleTreeLeafType[], startIndex: bigint, - ): Promise { - return this.trees.findLeafIndexAfter(treeId, value, startIndex, this.includeUncommitted); + ): Promise<(bigint | undefined)[]> { + return Promise.all( + values.map(leaf => this.trees.findLeafIndexAfter(treeId, leaf, startIndex, this.includeUncommitted)), + ); } /** diff --git a/yarn-project/world-state/src/world-state-db/merkle_tree_snapshot_operations_facade.ts b/yarn-project/world-state/src/world-state-db/merkle_tree_snapshot_operations_facade.ts index 5e703e9c313..498375a64ea 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_tree_snapshot_operations_facade.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_tree_snapshot_operations_facade.ts @@ -40,20 +40,23 @@ export class MerkleTreeSnapshotOperationsFacade implements MerkleTreeReadOperati return this.#treeSnapshots[treeId]!; } - async findLeafIndex(treeId: ID, value: MerkleTreeLeafType): Promise { + async findLeafIndices( + treeId: ID, + values: MerkleTreeLeafType[], + ): Promise<(bigint | undefined)[]> { const tree = await this.#getTreeSnapshot(treeId); // TODO #5448 fix "as any" - return tree.findLeafIndex(value as any); + return values.map(leaf => tree.findLeafIndex(leaf as any)); } - async findLeafIndexAfter( + async findLeafIndicesAfter( treeId: MerkleTreeId, - value: MerkleTreeLeafType, + values: MerkleTreeLeafType[], startIndex: bigint, - ): Promise { + ): Promise<(bigint | undefined)[]> { const tree = await this.#getTreeSnapshot(treeId); // TODO #5448 fix "as any" - return tree.findLeafIndexAfter(value as any, startIndex); + return values.map(leaf => tree.findLeafIndexAfter(leaf as any, startIndex)); } async getLeafPreimage(