From c1aa6f73d13e860d5fcee07e82347a7633f8b334 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Wed, 11 Sep 2024 19:51:22 +0100 Subject: [PATCH] feat: native world state (#7516) This PR adds a new CPP module for managing the merkle trees that make up the world state. A new implementetion of `MerkleTreeDb` is provided to interact with the native code. msgpack is used to pass messages across the js<->cpp boundary. Tests have been added to assert that the two world state implementations work in the same way (more tests would be better). This PR builds on top of #7037. PS: I'm not as experienced with C++ --------- Co-authored-by: PhilWindle --- barretenberg/cpp/CMakeLists.txt | 1 + barretenberg/cpp/CMakePresets.json | 37 ++ barretenberg/cpp/Earthfile | 6 + barretenberg/cpp/bootstrap.sh | 12 +- barretenberg/cpp/scripts/world_state_tests.sh | 12 + barretenberg/cpp/src/CMakeLists.txt | 9 + .../src/barretenberg/messaging/dispatcher.hpp | 41 ++ .../barretenberg/world_state/CMakeLists.txt | 1 + .../world_state/tree_with_store.hpp | 34 ++ .../src/barretenberg/world_state/types.hpp | 38 ++ .../barretenberg/world_state/world_state.cpp | 290 ++++++++++ .../barretenberg/world_state/world_state.hpp | 381 +++++++++++++ .../world_state/world_state.test.cpp | 505 ++++++++++++++++++ .../world_state_napi/CMakeLists.txt | 30 ++ .../barretenberg/world_state_napi/addon.cpp | 485 +++++++++++++++++ .../barretenberg/world_state_napi/addon.hpp | 57 ++ .../world_state_napi/async_op.hpp | 64 +++ .../barretenberg/world_state_napi/message.hpp | 162 ++++++ .../world_state_napi/package.json | 15 + .../barretenberg/world_state_napi/yarn.lock | 13 + noir-projects/Earthfile | 4 + yarn-project/Earthfile | 14 +- .../src/interfaces/merkle_tree_operations.ts | 2 + .../prover-client/src/mocks/test_context.ts | 25 +- .../src/orchestrator/orchestrator.ts | 2 +- .../src/test/bb_prover_base_rollup.test.ts | 2 +- .../src/test/bb_prover_full_rollup.test.ts | 2 +- .../src/test/bb_prover_parity.test.ts | 2 +- yarn-project/watch.sh | 11 +- yarn-project/world-state/package.json | 9 +- yarn-project/world-state/package.local.json | 5 + yarn-project/world-state/scripts/build.sh | 44 ++ yarn-project/world-state/src/native/index.ts | 1 + .../world-state/src/native/message.ts | 247 +++++++++ .../src/native/native_world_state.test.ts | 102 ++++ .../src/native/native_world_state.ts | 462 ++++++++++++++++ .../native_world_state_integration.test.ts | 295 ++++++++++ .../server_world_state_synchronizer.ts | 2 +- .../src/world-state-db/merkle_tree_db.ts | 3 + yarn-project/yarn.lock | 26 +- 40 files changed, 3438 insertions(+), 15 deletions(-) create mode 100755 barretenberg/cpp/scripts/world_state_tests.sh create mode 100644 barretenberg/cpp/src/barretenberg/messaging/dispatcher.hpp create mode 100644 barretenberg/cpp/src/barretenberg/world_state/CMakeLists.txt create mode 100644 barretenberg/cpp/src/barretenberg/world_state/tree_with_store.hpp create mode 100644 barretenberg/cpp/src/barretenberg/world_state/types.hpp create mode 100644 barretenberg/cpp/src/barretenberg/world_state/world_state.cpp create mode 100644 barretenberg/cpp/src/barretenberg/world_state/world_state.hpp create mode 100644 barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp create mode 100644 barretenberg/cpp/src/barretenberg/world_state_napi/CMakeLists.txt create mode 100644 barretenberg/cpp/src/barretenberg/world_state_napi/addon.cpp create mode 100644 barretenberg/cpp/src/barretenberg/world_state_napi/addon.hpp create mode 100644 barretenberg/cpp/src/barretenberg/world_state_napi/async_op.hpp create mode 100644 barretenberg/cpp/src/barretenberg/world_state_napi/message.hpp create mode 100644 barretenberg/cpp/src/barretenberg/world_state_napi/package.json create mode 100644 barretenberg/cpp/src/barretenberg/world_state_napi/yarn.lock create mode 100644 yarn-project/world-state/package.local.json create mode 100755 yarn-project/world-state/scripts/build.sh create mode 100644 yarn-project/world-state/src/native/index.ts create mode 100644 yarn-project/world-state/src/native/message.ts create mode 100644 yarn-project/world-state/src/native/native_world_state.test.ts create mode 100644 yarn-project/world-state/src/native/native_world_state.ts create mode 100644 yarn-project/world-state/src/native/native_world_state_integration.test.ts diff --git a/barretenberg/cpp/CMakeLists.txt b/barretenberg/cpp/CMakeLists.txt index e2caea121ac..24ce4f1ad42 100644 --- a/barretenberg/cpp/CMakeLists.txt +++ b/barretenberg/cpp/CMakeLists.txt @@ -38,6 +38,7 @@ option(ENABLE_HEAVY_TESTS "Enable heavy tests when collecting coverage" OFF) # Note: Must do 'sudo apt-get install libdw-dev' or equivalent option(CHECK_CIRCUIT_STACKTRACES "Enable (slow) stack traces for check circuit" OFF) option(ENABLE_TRACY "Enable low-medium overhead profiling for memory and performance with tracy" OFF) +option(ENABLE_PIC "Builds with position independent code" OFF) if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64" OR CMAKE_SYSTEM_PROCESSOR MATCHES "arm64") message(STATUS "Compiling for ARM.") diff --git a/barretenberg/cpp/CMakePresets.json b/barretenberg/cpp/CMakePresets.json index 02ad94ce29d..140b5780ed0 100644 --- a/barretenberg/cpp/CMakePresets.json +++ b/barretenberg/cpp/CMakePresets.json @@ -21,6 +21,23 @@ "TARGET_ARCH": "skylake" } }, + { + "name": "default-pic", + "displayName": "Build with Clang with Position Independent Code", + "description": "Build with globally installed Clang with Position Independent Code", + "binaryDir": "build-pic", + "generator": "Ninja", + "environment": { + "CC": "clang", + "CXX": "clang++", + "CMAKE_EXPORT_COMPILE_COMMANDS": "ON", + "LDFLAGS": "-Wl,-undefined,dynamic_lookup" + }, + "cacheVariables": { + "TARGET_ARCH": "skylake", + "ENABLE_PIC": "ON" + } + }, { "name": "homebrew", "displayName": "Homebrew + Clang", @@ -66,6 +83,16 @@ "CXX": "clang++-16" } }, + { + "name": "clang16-pic", + "displayName": "Release build with Position Independent Code", + "description": "Build with globally installed Clang-16 using Position Independent Code", + "inherits": "clang16", + "binaryDir": "build-pic", + "cacheVariables": { + "ENABLE_PIC": "ON" + } + }, { "name": "clang16-dbg", "displayName": "Debugging build with Clang-16", @@ -446,6 +473,16 @@ "inherits": "default", "configurePreset": "tracy" }, + { + "name": "clang16-pic", + "inherits": "default", + "configurePreset": "clang16-pic" + }, + { + "name": "default-pic", + "inherits": "default", + "configurePreset": "default-pic" + }, { "name": "tracy-gates", "inherits": "default", diff --git a/barretenberg/cpp/Earthfile b/barretenberg/cpp/Earthfile index 73b8cdeb5cd..574be394941 100644 --- a/barretenberg/cpp/Earthfile +++ b/barretenberg/cpp/Earthfile @@ -29,6 +29,11 @@ preset-release: RUN cmake --preset clang16 -Bbuild && cmake --build build --target bb && rm -rf build/{deps,lib,src} SAVE ARTIFACT build/bin +preset-release-world-state: + FROM +source + RUN cmake --preset clang16-pic -Bbuild && cmake --build build --target world_state_napi && cp ./build/lib/world_state_napi.node ./build/bin && rm -rf build/{deps,lib,src} + SAVE ARTIFACT build/bin + preset-release-assert: FROM +source RUN cmake --preset clang16-assert -Bbuild && cmake --build build --target bb crypto_merkle_tree_tests && rm -rf build/{deps,lib,src} @@ -264,3 +269,4 @@ build: BUILD +preset-wasm BUILD +preset-wasm-threads BUILD +preset-release + BUILD +preset-release-world-state diff --git a/barretenberg/cpp/bootstrap.sh b/barretenberg/cpp/bootstrap.sh index bd82cc67e1c..532e9b1165e 100755 --- a/barretenberg/cpp/bootstrap.sh +++ b/barretenberg/cpp/bootstrap.sh @@ -46,17 +46,27 @@ else fi fi +PIC_PRESET="$PRESET-pic" + # Remove cmake cache files. rm -f {build,build-wasm,build-wasm-threads}/CMakeCache.txt +(cd src/barretenberg/world_state_napi && yarn --frozen-lockfile --prefer-offline) + echo "#################################" echo "# Building with preset: $PRESET" echo "# When running cmake directly, remember to use: --build --preset $PRESET" echo "#################################" function build_native { + # Build bb with standard preset and world_state_napi with Position Independent code variant cmake --preset $PRESET -DCMAKE_BUILD_TYPE=RelWithAssert + cmake --preset $PIC_PRESET -DCMAKE_BUILD_TYPE=RelWithAssert cmake --build --preset $PRESET --target bb + cmake --build --preset $PIC_PRESET --target world_state_napi + # copy the world_state_napi build artifact over to the world state in yarn-project + mkdir -p ../../yarn-project/world-state/build/ + cp ./build-pic/lib/world_state_napi.node ../../yarn-project/world-state/build/ } function build_wasm { @@ -106,7 +116,7 @@ fi if [ ! -d ./srs_db/grumpkin ]; then # The Grumpkin SRS is generated manually at the moment, only up to a large enough size for tests - # If tests require more points, the parameter can be increased here. Note: IPA requires + # If tests require more points, the parameter can be increased here. Note: IPA requires # dyadic_circuit_size + 1 points so in general this number will be a power of two plus 1 cd ./build && cmake --build . --parallel --target grumpkin_srs_gen && ./bin/grumpkin_srs_gen 8193 fi diff --git a/barretenberg/cpp/scripts/world_state_tests.sh b/barretenberg/cpp/scripts/world_state_tests.sh new file mode 100755 index 00000000000..2eba93bbf62 --- /dev/null +++ b/barretenberg/cpp/scripts/world_state_tests.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -e + +# run commands relative to parent directory +cd $(dirname $0)/.. + +TEST=${1:-*} +PRESET=${PRESET:-clang16} + +cmake --build --preset $PRESET --target world_state_tests +./build/bin/world_state_tests --gtest_filter=WorldStateTest.${TEST} diff --git a/barretenberg/cpp/src/CMakeLists.txt b/barretenberg/cpp/src/CMakeLists.txt index 1a7a853b55b..a4476f0e909 100644 --- a/barretenberg/cpp/src/CMakeLists.txt +++ b/barretenberg/cpp/src/CMakeLists.txt @@ -1,5 +1,6 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) # Enable the following warnings project wide. # If any compilation issues arise in the future, they should not be silenced here but rather in the @@ -53,6 +54,13 @@ else() message(STATUS "Using optimized assembly for field arithmetic.") endif() +if (ENABLE_PIC AND CMAKE_CXX_COMPILER_ID MATCHES "Clang") + message("Building with Position Independent Code") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC") + add_subdirectory(barretenberg/world_state_napi) +endif() + add_subdirectory(barretenberg/aztec_ivc) add_subdirectory(barretenberg/bb) add_subdirectory(barretenberg/circuit_checker) @@ -87,6 +95,7 @@ add_subdirectory(barretenberg/translator_vm) add_subdirectory(barretenberg/ultra_honk) add_subdirectory(barretenberg/vm) add_subdirectory(barretenberg/wasi) +add_subdirectory(barretenberg/world_state) if(SMT) add_subdirectory(barretenberg/smt_verification) diff --git a/barretenberg/cpp/src/barretenberg/messaging/dispatcher.hpp b/barretenberg/cpp/src/barretenberg/messaging/dispatcher.hpp new file mode 100644 index 00000000000..20327d4757b --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/messaging/dispatcher.hpp @@ -0,0 +1,41 @@ +#pragma once + +#include "barretenberg/messaging/header.hpp" +#include "barretenberg/serialize/cbind.hpp" +#include +#include +#include +#include +#include + +namespace bb::messaging { + +using message_handler = std::function; + +class MessageDispatcher { + private: + std::unordered_map messageHandlers; + + public: + MessageDispatcher() = default; + + bool onNewData(msgpack::object& obj, msgpack::sbuffer& buffer) + { + bb::messaging::HeaderOnlyMessage header; + obj.convert(header); + + auto iter = messageHandlers.find(header.msgType); + if (iter == messageHandlers.end()) { + throw std::runtime_error("No registered handler for message of type " + std::to_string(header.msgType)); + } + + return (iter->second)(obj, buffer); + } + + void registerTarget(uint32_t msgType, const message_handler& handler) + { + messageHandlers.insert({ msgType, handler }); + } +}; + +} // namespace bb::messaging diff --git a/barretenberg/cpp/src/barretenberg/world_state/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/world_state/CMakeLists.txt new file mode 100644 index 00000000000..d9825cf5383 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/world_state/CMakeLists.txt @@ -0,0 +1 @@ +barretenberg_module(world_state crypto_merkle_tree stdlib_poseidon2) diff --git a/barretenberg/cpp/src/barretenberg/world_state/tree_with_store.hpp b/barretenberg/cpp/src/barretenberg/world_state/tree_with_store.hpp new file mode 100644 index 00000000000..78bb2c5e526 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/world_state/tree_with_store.hpp @@ -0,0 +1,34 @@ +#pragma once + +#include + +namespace bb::world_state { + +template struct TreeWithStore { + using TreeType = Tree; + std::unique_ptr tree; + std::unique_ptr store; + std::unique_ptr persisted_store; + + TreeWithStore(std::unique_ptr t, + std::unique_ptr s, + std::unique_ptr p) + : tree(std::move(t)) + , store(std::move(s)) + , persisted_store(std::move(p)) + {} + + TreeWithStore(TreeWithStore&& other) noexcept + : tree(std::move(other.tree)) + , store(std::move(other.store)) + , persisted_store(std::move(other.persisted_store)) + {} + + TreeWithStore(const TreeWithStore& other) = delete; + ~TreeWithStore() = default; + + TreeWithStore& operator=(TreeWithStore&& other) = delete; + TreeWithStore& operator=(const TreeWithStore& other) = delete; +}; + +} // namespace bb::world_state diff --git a/barretenberg/cpp/src/barretenberg/world_state/types.hpp b/barretenberg/cpp/src/barretenberg/world_state/types.hpp new file mode 100644 index 00000000000..8e4dc6f2744 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/world_state/types.hpp @@ -0,0 +1,38 @@ +#pragma once + +#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/types.hpp" +#include "barretenberg/ecc/curves/bn254/fr.hpp" +#include +#include + +namespace bb::world_state { + +enum MerkleTreeId { + NULLIFIER_TREE = 0, + NOTE_HASH_TREE = 1, + PUBLIC_DATA_TREE = 2, + L1_TO_L2_MESSAGE_TREE = 3, + ARCHIVE = 4, +}; + +using TreeStateReference = std::pair; +using StateReference = std::unordered_map; + +struct WorldStateRevision { + struct FinalisedBlock { + uint32_t block; + }; + + struct CurrentState { + bool uncommitted; + }; + + using Revision = std::variant; + Revision inner; + + static WorldStateRevision committed() { return { CurrentState{ false } }; } + static WorldStateRevision uncommitted() { return { CurrentState{ true } }; } + static WorldStateRevision finalised_block(uint32_t block_number) { return { FinalisedBlock{ block_number } }; } +}; +} // namespace bb::world_state diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp b/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp new file mode 100644 index 00000000000..307c99cb980 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp @@ -0,0 +1,290 @@ +#include "barretenberg/world_state/world_state.hpp" +#include "barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp" +#include "barretenberg/crypto/merkle_tree/fixtures.hpp" +#include "barretenberg/crypto/merkle_tree/hash_path.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/callbacks.hpp" +#include "barretenberg/crypto/merkle_tree/response.hpp" +#include "barretenberg/crypto/merkle_tree/signal.hpp" +#include "barretenberg/world_state/tree_with_store.hpp" +#include "barretenberg/world_state/types.hpp" +#include +#include +#include +#include +#include +#include + +namespace bb::world_state { + +const uint WORLD_STATE_MAX_DB_COUNT = 16; + +using namespace bb::crypto::merkle_tree; + +WorldState::WorldState(uint threads, const std::string& data_dir, uint map_size_kb) + : _workers(threads) +{ + _lmdb_env = std::make_unique(data_dir, map_size_kb, WORLD_STATE_MAX_DB_COUNT, threads); + + { + const auto* name = "nullifier_tree"; + auto lmdb_store = std::make_unique(*_lmdb_env, name, false, false, integer_key_cmp); + auto store = std::make_unique(name, NULLIFIER_TREE_HEIGHT, *lmdb_store); + auto tree = std::make_unique(*store, _workers, 128); + _trees.insert( + { MerkleTreeId::NULLIFIER_TREE, TreeWithStore(std::move(tree), std::move(store), std::move(lmdb_store)) }); + } + + { + const auto* name = "note_hash_tree"; + auto lmdb_store = std::make_unique(*_lmdb_env, name, false, false, integer_key_cmp); + auto store = std::make_unique(name, NOTE_HASH_TREE_HEIGHT, *lmdb_store); + auto tree = std::make_unique(*store, this->_workers); + _trees.insert( + { MerkleTreeId::NOTE_HASH_TREE, TreeWithStore(std::move(tree), std::move(store), std::move(lmdb_store)) }); + } + + { + const auto* name = "public_data_tree"; + auto lmdb_store = std::make_unique(*_lmdb_env, name, false, false, integer_key_cmp); + auto store = std::make_unique(name, PUBLIC_DATA_TREE_HEIGHT, *lmdb_store); + auto tree = std::make_unique(*store, this->_workers, 128); + _trees.insert({ MerkleTreeId::PUBLIC_DATA_TREE, + TreeWithStore(std::move(tree), std::move(store), std::move(lmdb_store)) }); + } + + { + const auto* name = "message_tree"; + auto lmdb_store = std::make_unique(*_lmdb_env, name, false, false, integer_key_cmp); + auto store = std::make_unique(name, L1_TO_L2_MSG_TREE_HEIGHT, *lmdb_store); + auto tree = std::make_unique(*store, this->_workers); + _trees.insert({ MerkleTreeId::L1_TO_L2_MESSAGE_TREE, + TreeWithStore(std::move(tree), std::move(store), std::move(lmdb_store)) }); + } + + { + const auto* name = "archive_tree"; + auto lmdb_store = std::make_unique(*_lmdb_env, name, false, false, integer_key_cmp); + auto store = std::make_unique(name, ARCHIVE_TREE_HEIGHT, *lmdb_store); + auto tree = std::make_unique(*store, this->_workers); + _trees.insert( + { MerkleTreeId::ARCHIVE, TreeWithStore(std::move(tree), std::move(store), std::move(lmdb_store)) }); + } +} + +TreeMetaResponse WorldState::get_tree_info(WorldStateRevision revision, MerkleTreeId tree_id) const +{ + return std::visit( + [=](auto&& wrapper) { + Signal signal(1); + TreeMetaResponse response; + + auto callback = [&](const TypedResponse& meta) { + response = meta.inner; + signal.signal_level(0); + }; + + wrapper.tree->get_meta_data(include_uncommitted(revision), callback); + signal.wait_for_level(0); + + return response; + }, + _trees.at(tree_id)); +} + +StateReference WorldState::get_state_reference(WorldStateRevision revision) const +{ + Signal signal(static_cast(_trees.size())); + StateReference state_reference; + bool uncommitted = include_uncommitted(revision); + + for (const auto& [id, tree] : _trees) { + std::visit( + [&signal, &state_reference, id, uncommitted](auto&& wrapper) { + auto callback = [&signal, &state_reference, id](const TypedResponse& meta) { + state_reference.insert({ id, { meta.inner.root, meta.inner.size } }); + signal.signal_decrement(); + }; + wrapper.tree->get_meta_data(uncommitted, callback); + }, + tree); + } + + signal.wait_for_level(0); + return state_reference; +} + +fr_sibling_path WorldState::get_sibling_path(WorldStateRevision revision, + MerkleTreeId tree_id, + index_t leaf_index) const +{ + bool uncommited = include_uncommitted(revision); + return std::visit( + [leaf_index, uncommited](auto&& wrapper) { + Signal signal(1); + fr_sibling_path path; + + auto callback = [&signal, &path](const TypedResponse& response) { + path = response.inner.path; + signal.signal_level(0); + }; + + wrapper.tree->get_sibling_path(leaf_index, callback, uncommited); + signal.wait_for_level(0); + + return path; + }, + _trees.at(tree_id)); +} + +void WorldState::update_public_data(const PublicDataLeafValue& new_value) +{ + if (const auto* wrapper = std::get_if>(&_trees.at(MerkleTreeId::PUBLIC_DATA_TREE))) { + Signal signal; + wrapper->tree->add_or_update_value(new_value, [&signal](const auto&) { signal.signal_level(0); }); + signal.wait_for_level(); + } else { + throw std::runtime_error("Invalid tree type for PublicDataTree"); + } +} + +void WorldState::commit() +{ + // TODO (alexg) should this lock _all_ the trees until they are all committed? + // otherwise another request could come in to modify one of the trees + // or reads would give inconsistent results + Signal signal(static_cast(_trees.size())); + for (auto& [id, tree] : _trees) { + std::visit( + [&signal](auto&& wrapper) { wrapper.tree->commit([&](const Response&) { signal.signal_decrement(); }); }, + tree); + } + + signal.wait_for_level(0); +} + +void WorldState::rollback() +{ + // TODO (alexg) should this lock _all_ the trees until they are all committed? + // otherwise another request could come in to modify one of the trees + // or reads would give inconsistent results + Signal signal(static_cast(_trees.size())); + for (auto& [id, tree] : _trees) { + std::visit( + [&signal](auto&& wrapper) { + wrapper.tree->rollback([&signal](const Response&) { signal.signal_decrement(); }); + }, + tree); + } + signal.wait_for_level(); +} + +bool WorldState::sync_block(StateReference& block_state_ref, + fr block_hash, + const std::vector& notes, + const std::vector& l1_to_l2_messages, + const std::vector& nullifiers, + const std::vector>& public_writes) +{ + auto current_state = get_state_reference(WorldStateRevision::uncommitted()); + if (block_state_matches_world_state(block_state_ref, current_state)) { + append_leaves(MerkleTreeId::ARCHIVE, { block_hash }); + commit(); + return true; + } + + rollback(); + + // the public data tree gets updated once per batch and every other gets one update + Signal signal(static_cast(_trees.size())); + auto decr = [&signal](const auto&) { signal.signal_decrement(); }; + + { + auto& wrapper = std::get>(_trees.at(MerkleTreeId::NULLIFIER_TREE)); + wrapper.tree->add_or_update_values(nullifiers, 0, decr); + } + + { + auto& wrapper = std::get>(_trees.at(MerkleTreeId::NOTE_HASH_TREE)); + wrapper.tree->add_values(notes, decr); + } + + { + auto& wrapper = std::get>(_trees.at(MerkleTreeId::L1_TO_L2_MESSAGE_TREE)); + wrapper.tree->add_values(l1_to_l2_messages, decr); + } + + { + auto& wrapper = std::get>(_trees.at(MerkleTreeId::ARCHIVE)); + wrapper.tree->add_value(block_hash, decr); + } + + { + auto& wrapper = std::get>(_trees.at(MerkleTreeId::PUBLIC_DATA_TREE)); + for (const auto& batch : public_writes) { + Signal batch_signal(1); + // TODO (alexg) should trees serialize writes internally or should we do it here? + wrapper.tree->add_or_update_values( + batch, 0, [&batch_signal](const auto&) { batch_signal.signal_level(0); }); + batch_signal.wait_for_level(0); + } + + signal.signal_decrement(); + } + + signal.wait_for_level(); + + current_state = get_state_reference(WorldStateRevision::uncommitted()); + if (block_state_matches_world_state(block_state_ref, current_state)) { + commit(); + return false; + } + + // TODO (alexg) should we rollback here? + // Potentiall not since all the changes exist only in-memory and this error will cause the process to die + throw std::runtime_error("Block state does not match world state"); +} + +std::pair WorldState::find_low_leaf_index(const WorldStateRevision revision, + MerkleTreeId tree_id, + fr leaf_key) const +{ + Signal signal; + std::pair low_leaf_info; + auto callback = [&signal, &low_leaf_info](const TypedResponse>& response) { + low_leaf_info = response.inner; + signal.signal_level(); + }; + + if (const auto* wrapper = std::get_if>(&_trees.at(tree_id))) { + wrapper->tree->find_low_leaf(leaf_key, include_uncommitted(revision), callback); + } else if (const auto* wrapper = std::get_if>(&_trees.at(tree_id))) { + wrapper->tree->find_low_leaf(leaf_key, include_uncommitted(revision), callback); + } else { + throw std::runtime_error("Invalid tree type for find_low_leaf"); + } + + signal.wait_for_level(); + return low_leaf_info; +} + +bool WorldState::include_uncommitted(WorldStateRevision rev) +{ + return std::get(rev.inner).uncommitted; +} + +bool WorldState::block_state_matches_world_state(const StateReference& block_state_ref, + const StateReference& tree_state_ref) +{ + std::vector tree_ids{ + MerkleTreeId::NULLIFIER_TREE, + MerkleTreeId::NOTE_HASH_TREE, + MerkleTreeId::PUBLIC_DATA_TREE, + MerkleTreeId::L1_TO_L2_MESSAGE_TREE, + }; + + return std::all_of( + tree_ids.begin(), tree_ids.end(), [&](auto id) { return block_state_ref.at(id) == tree_state_ref.at(id); }); +} + +} // namespace bb::world_state diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp b/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp new file mode 100644 index 00000000000..4ab6455131b --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp @@ -0,0 +1,381 @@ +#pragma once + +#include "barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp" +#include "barretenberg/crypto/merkle_tree/hash.hpp" +#include "barretenberg/crypto/merkle_tree/hash_path.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.hpp" +#include "barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp" +#include "barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp" +#include "barretenberg/crypto/merkle_tree/response.hpp" +#include "barretenberg/crypto/merkle_tree/signal.hpp" +#include "barretenberg/crypto/merkle_tree/types.hpp" +#include "barretenberg/ecc/curves/bn254/fr.hpp" +#include "barretenberg/serialize/msgpack.hpp" +#include "barretenberg/world_state/tree_with_store.hpp" +#include "barretenberg/world_state/types.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace bb::world_state { + +using crypto::merkle_tree::index_t; + +using HashPolicy = crypto::merkle_tree::Poseidon2HashPolicy; + +using FrStore = crypto::merkle_tree::CachedTreeStore; +using FrTree = crypto::merkle_tree::AppendOnlyTree; + +using NullifierStore = + crypto::merkle_tree::CachedTreeStore; +using NullifierTree = crypto::merkle_tree::IndexedTree; + +using PublicDataStore = + crypto::merkle_tree::CachedTreeStore; +using PublicDataTree = crypto::merkle_tree::IndexedTree; + +using Tree = std::variant, TreeWithStore, TreeWithStore>; + +template struct BatchInsertionResult { + std::vector> low_leaf_witness_data; + std::vector> sorted_leaves; + crypto::merkle_tree::fr_sibling_path subtree_path; + + MSGPACK_FIELDS(low_leaf_witness_data, sorted_leaves, subtree_path); +}; + +const uint NULLIFIER_TREE_HEIGHT = 20; +const uint NOTE_HASH_TREE_HEIGHT = 32; +const uint PUBLIC_DATA_TREE_HEIGHT = 40; +const uint L1_TO_L2_MSG_TREE_HEIGHT = 16; +const uint ARCHIVE_TREE_HEIGHT = 16; + +class WorldState { + public: + WorldState(uint threads, const std::string& data_dir, uint map_size_kb); + + /** + * @brief Get tree metadata for a particular tree + * + * @param revision The revision to query + * @param tree_id The ID of the tree + * @return TreeInfo + */ + crypto::merkle_tree::TreeMetaResponse get_tree_info(WorldStateRevision revision, MerkleTreeId tree_id) const; + + /** + * @brief Gets the state reference for all the trees in the world state + * + * @param revision The revision to query + * @return StateReference + */ + StateReference get_state_reference(WorldStateRevision revision) const; + + /** + * @brief Get the sibling path object for a leaf in a tree + * + * @param revision The revision to query + * @param tree_id The ID of the tree + * @param leaf_index The index of the leaf + * @return crypto::merkle_tree::fr_sibling_path + */ + crypto::merkle_tree::fr_sibling_path get_sibling_path(WorldStateRevision revision, + MerkleTreeId tree_id, + index_t leaf_index) const; + + /** + * @brief Get the leaf preimage object + * + * @tparam T the type of the leaf. Either NullifierLeafValue, PublicDataLeafValue + * @param revision The revision to query + * @param tree_id The ID of the tree + * @param leaf_index The index of the leaf + * @return std::optional The IndexedLeaf object or nullopt if the leaf does not exist + */ + template + std::optional> get_indexed_leaf(WorldStateRevision revision, + MerkleTreeId tree_id, + index_t leaf_index) const; + + /** + * @brief Gets the value of a leaf in a tree + * + * @tparam T the type of the leaf. Either bb::fr, NullifierLeafValue, PublicDataLeafValue + * @param revision The revision to query + * @param tree_id The ID of the tree + * @param leaf_index The index of the leaf + * @return std::optional The value of the leaf or nullopt if the leaf does not exist + */ + template + std::optional get_leaf(WorldStateRevision revision, MerkleTreeId tree_id, index_t leaf_index) const; + + /** + * @brief Finds the leaf that would have its nextIdx/nextValue fields modified if the target leaf were to be + * inserted into the tree. If the vlaue already exists in the tree, the leaf with the same value is returned. + * + * @param revision The revision to query + * @param tree_id The ID of the tree + * @param leaf_key The leaf to find the predecessor of + * @return PredecessorInfo + */ + std::pair find_low_leaf_index(WorldStateRevision revision, MerkleTreeId tree_id, fr leaf_key) const; + + /** + * @brief Finds the index of a leaf in a tree + * + * @param revision The revision to query + * @param tree_id The ID of the tree + * @param leaf The leaf to find + * @param start_index The index to start searching from + * @return std::optional + */ + template + std::optional find_leaf_index(WorldStateRevision revision, + MerkleTreeId tree_id, + const T& leaf, + index_t start_index = 0) const; + + /** + * @brief Appends a set of leaves to an existing Merkle Tree. + * + * @tparam T The type of the leaves. + * @param tree_id The ID of the Merkle Tree. + * @param leaves The leaves to append. + */ + template void append_leaves(MerkleTreeId tree_id, const std::vector& leaves); + + /** + * @brief Batch inserts a set of leaves into an indexed Merkle Tree. + * + * @tparam T The type of the leaves. + * @param tree_id The ID of the Merkle Tree. + * @param leaves The leaves to insert. + * @return BatchInsertionResult + */ + template + BatchInsertionResult batch_insert_indexed_leaves(MerkleTreeId tree_id, + const std::vector& leaves, + uint32_t subtree_depth); + + /** + * @brief Updates a leaf in an existing Merkle Tree. + * + * @param new_value The new value of the leaf. + */ + void update_public_data(const crypto::merkle_tree::PublicDataLeafValue& new_value); + + /** + * @brief Commits the current state of the world state. + */ + void commit(); + + /** + * @brief Rolls back any uncommitted changes made to the world state. + */ + void rollback(); + + /** + * @brief Synchronizes the world state with a new block. + * + * @param block The block to synchronize with. + */ + bool sync_block(StateReference& block_state_ref, + fr block_hash, + const std::vector& notes, + const std::vector& l1_to_l2_messages, + const std::vector& nullifiers, + const std::vector>& public_writes); + + private: + std::unique_ptr _lmdb_env; + std::unordered_map _trees; + bb::ThreadPool _workers; + + TreeStateReference get_tree_snapshot(MerkleTreeId id); + + static bool include_uncommitted(WorldStateRevision rev); + static bool block_state_matches_world_state(const StateReference& block_state_ref, + const StateReference& tree_state_ref); +}; + +template +std::optional> WorldState::get_indexed_leaf(const WorldStateRevision rev, + MerkleTreeId id, + index_t leaf) const +{ + using namespace crypto::merkle_tree; + using Store = CachedTreeStore; + using Tree = IndexedTree; + + if (auto* const wrapper = std::get_if>(&_trees.at(id))) { + std::optional> value; + Signal signal; + auto callback = [&](const TypedResponse>& response) { + if (response.inner.indexed_leaf.has_value()) { + value = response.inner.indexed_leaf; + } + + signal.signal_level(0); + }; + + wrapper->tree->get_leaf(leaf, include_uncommitted(rev), callback); + signal.wait_for_level(); + + return value; + } + + throw std::runtime_error("Invalid tree type"); +} + +template +std::optional WorldState::get_leaf(const WorldStateRevision revision, MerkleTreeId tree_id, index_t leaf_index) const +{ + using namespace crypto::merkle_tree; + + bool uncommitted = include_uncommitted(revision); + std::optional leaf; + Signal signal; + if constexpr (std::is_same_v) { + const auto& wrapper = std::get>(_trees.at(tree_id)); + wrapper.tree->get_leaf(leaf_index, uncommitted, [&signal, &leaf](const TypedResponse& resp) { + if (resp.inner.leaf.has_value()) { + leaf = resp.inner.leaf.value(); + } + signal.signal_level(); + }); + } else { + using Store = CachedTreeStore; + using Tree = IndexedTree; + + auto& wrapper = std::get>(_trees.at(tree_id)); + wrapper.tree->get_leaf( + leaf_index, uncommitted, [&signal, &leaf](const TypedResponse>& resp) { + if (resp.inner.indexed_leaf.has_value()) { + leaf = resp.inner.indexed_leaf.value().value; + } + signal.signal_level(); + }); + } + + signal.wait_for_level(); + return leaf; +} + +template +std::optional WorldState::find_leaf_index(const WorldStateRevision rev, + MerkleTreeId id, + const T& leaf, + index_t start_index) const +{ + using namespace crypto::merkle_tree; + bool uncommitted = include_uncommitted(rev); + std::optional index; + + Signal signal; + auto callback = [&](const TypedResponse& response) { + if (response.success) { + index = response.inner.leaf_index; + } + signal.signal_level(0); + }; + if constexpr (std::is_same_v) { + const auto& wrapper = std::get>(_trees.at(id)); + wrapper.tree->find_leaf_index_from(leaf, start_index, uncommitted, callback); + } else { + using Store = CachedTreeStore; + using Tree = IndexedTree; + + auto& wrapper = std::get>(_trees.at(id)); + wrapper.tree->find_leaf_index_from(leaf, start_index, uncommitted, callback); + } + + signal.wait_for_level(0); + return index; +} + +template void WorldState::append_leaves(MerkleTreeId id, const std::vector& leaves) +{ + using namespace crypto::merkle_tree; + + Signal signal; + + bool success = false; + auto callback = [&signal, &success](const auto& resp) { + success = resp.success; + signal.signal_level(0); + }; + + if constexpr (std::is_same_v) { + auto& wrapper = std::get>(_trees.at(id)); + wrapper.tree->add_values(leaves, callback); + } else { + using Store = CachedTreeStore; + using Tree = IndexedTree; + auto& wrapper = std::get>(_trees.at(id)); + wrapper.tree->add_or_update_values(leaves, 0, callback); + } + + signal.wait_for_level(0); + + if (!success) { + throw std::runtime_error("Failed to append leaves"); + } +} + +template +BatchInsertionResult WorldState::batch_insert_indexed_leaves(MerkleTreeId id, + const std::vector& leaves, + uint32_t subtree_depth) +{ + using namespace crypto::merkle_tree; + using Store = CachedTreeStore; + using Tree = IndexedTree; + + Signal signal; + BatchInsertionResult result; + const auto& wrapper = std::get>(_trees.at(id)); + bool success = false; + std::string error_msg; + + wrapper.tree->add_or_update_values( + leaves, subtree_depth, [&](const TypedResponse>& response) { + success = response.success; + if (!response.success) { + error_msg = response.message; + return; + } + + result.low_leaf_witness_data.reserve(response.inner.low_leaf_witness_data->size()); + std::copy(response.inner.low_leaf_witness_data->begin(), + response.inner.low_leaf_witness_data->end(), + std::back_inserter(result.low_leaf_witness_data)); + + result.sorted_leaves.reserve(response.inner.sorted_leaves->size()); + std::copy(response.inner.sorted_leaves->begin(), + response.inner.sorted_leaves->end(), + std::back_inserter(result.sorted_leaves)); + + result.subtree_path = response.inner.subtree_path; + + signal.signal_level(0); + }); + + signal.wait_for_level(); + + if (!success) { + throw std::runtime_error(error_msg); + } + + return result; +} +} // namespace bb::world_state + +MSGPACK_ADD_ENUM(bb::world_state::MerkleTreeId) diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp b/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp new file mode 100644 index 00000000000..f4cd18003a1 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp @@ -0,0 +1,505 @@ +#include "barretenberg/world_state/world_state.hpp" +#include "barretenberg/crypto/merkle_tree/fixtures.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/ecc/curves/bn254/fr.hpp" +#include "barretenberg/world_state/types.hpp" +#include +#include + +using namespace bb::world_state; +using namespace bb::crypto::merkle_tree; + +class WorldStateTest : public testing::Test { + protected: + void SetUp() override + { + // setup with 1MB max db size, 1 max database and 2 maximum concurrent readers + _directory = random_temp_directory(); + std::filesystem::create_directories(_directory); + } + + void TearDown() override { std::filesystem::remove_all(_directory); } + + static std::string _directory; +}; + +std::string WorldStateTest::_directory; + +template +void assert_leaf_status( + const WorldState& ws, WorldStateRevision revision, MerkleTreeId tree_id, index_t leaf_index, bool exists) +{ + std::optional leaf = ws.get_leaf(revision, tree_id, leaf_index); + EXPECT_EQ(leaf.has_value(), exists); +} + +template +void assert_leaf_value(const WorldState& ws, + WorldStateRevision revision, + MerkleTreeId tree_id, + index_t leaf_index, + const Leaf& expected_value) +{ + std::optional leaf = ws.get_leaf(revision, tree_id, leaf_index); + EXPECT_EQ(leaf.has_value(), true); + EXPECT_EQ(leaf.value(), expected_value); +} + +template +void assert_leaf_exists( + const WorldState& ws, WorldStateRevision revision, MerkleTreeId tree_id, const Leaf& expected_value, bool exists) +{ + std::optional index = ws.find_leaf_index(revision, tree_id, expected_value); + EXPECT_EQ(index.has_value(), exists); +} + +template +void assert_leaf_index( + const WorldState& ws, WorldStateRevision revision, MerkleTreeId tree_id, const Leaf& value, index_t expected_index) +{ + std::optional index = ws.find_leaf_index(revision, tree_id, value); + EXPECT_EQ(index.value(), expected_index); +} + +void assert_tree_size(const WorldState& ws, WorldStateRevision revision, MerkleTreeId tree_id, size_t expected_size) +{ + auto info = ws.get_tree_info(revision, tree_id); + EXPECT_EQ(info.size, expected_size); +} + +void assert_sibling_path( + const WorldState& ws, WorldStateRevision revision, MerkleTreeId tree_id, fr root, fr leaf, index_t index) +{ + auto sibling_path = ws.get_sibling_path(revision, tree_id, index); + fr left; + fr right; + fr hash = leaf; + for (const auto& node : sibling_path) { + if (index % 2 == 0) { + left = hash; + right = node; + } else { + left = node; + right = hash; + } + + hash = HashPolicy::hash_pair(left, right); + index >>= 1; + } + + EXPECT_EQ(hash, root); +} + +TEST_F(WorldStateTest, GetInitialTreeInfoForAllTrees) +{ + WorldState ws(1, _directory, 1024); + + { + auto info = ws.get_tree_info(WorldStateRevision::committed(), MerkleTreeId::NULLIFIER_TREE); + EXPECT_EQ(info.size, 128); + EXPECT_EQ(info.depth, NULLIFIER_TREE_HEIGHT); + EXPECT_EQ(info.root, bb::fr("0x19a8c197c12bb33da6314c4ef4f8f6fcb9e25250c085df8672adf67c8f1e3dbc")); + } + + { + auto info = ws.get_tree_info(WorldStateRevision::committed(), MerkleTreeId::NOTE_HASH_TREE); + EXPECT_EQ(info.size, 0); + EXPECT_EQ(info.depth, NOTE_HASH_TREE_HEIGHT); + EXPECT_EQ(info.root, bb::fr("0x0b59baa35b9dc267744f0ccb4e3b0255c1fc512460d91130c6bc19fb2668568d")); + } + + { + auto info = ws.get_tree_info(WorldStateRevision::committed(), MerkleTreeId::PUBLIC_DATA_TREE); + EXPECT_EQ(info.size, 128); + EXPECT_EQ(info.depth, PUBLIC_DATA_TREE_HEIGHT); + EXPECT_EQ(info.root, bb::fr("0x23c08a6b1297210c5e24c76b9a936250a1ce2721576c26ea797c7ec35f9e46a9")); + } + + { + auto info = ws.get_tree_info(WorldStateRevision::committed(), MerkleTreeId::L1_TO_L2_MESSAGE_TREE); + EXPECT_EQ(info.size, 0); + EXPECT_EQ(info.depth, L1_TO_L2_MSG_TREE_HEIGHT); + EXPECT_EQ(info.root, bb::fr("0x14f44d672eb357739e42463497f9fdac46623af863eea4d947ca00a497dcdeb3")); + } + + { + // TODO (alexg) this should be the tree _after_ we insert the initial header + // currently it's the root of an empty tree + auto info = ws.get_tree_info(WorldStateRevision::committed(), MerkleTreeId::ARCHIVE); + EXPECT_EQ(info.size, 0); + EXPECT_EQ(info.depth, ARCHIVE_TREE_HEIGHT); + EXPECT_EQ(info.root, bb::fr("0x14f44d672eb357739e42463497f9fdac46623af863eea4d947ca00a497dcdeb3")); + } +} + +TEST_F(WorldStateTest, GetInitialStateReference) +{ + WorldState ws(1, _directory, 1024); + auto state_ref = ws.get_state_reference(WorldStateRevision::committed()); + + EXPECT_EQ(state_ref.size(), 5); + + { + auto snapshot = state_ref.at(MerkleTreeId::NULLIFIER_TREE); + EXPECT_EQ(snapshot, + std::make_pair(bb::fr("0x19a8c197c12bb33da6314c4ef4f8f6fcb9e25250c085df8672adf67c8f1e3dbc"), 128UL)); + } + + { + auto snapshot = state_ref.at(MerkleTreeId::NOTE_HASH_TREE); + EXPECT_EQ(snapshot, + std::make_pair(bb::fr("0x0b59baa35b9dc267744f0ccb4e3b0255c1fc512460d91130c6bc19fb2668568d"), 0UL)); + } + + { + auto snapshot = state_ref.at(MerkleTreeId::PUBLIC_DATA_TREE); + EXPECT_EQ(snapshot, + std::make_pair(bb::fr("0x23c08a6b1297210c5e24c76b9a936250a1ce2721576c26ea797c7ec35f9e46a9"), 128UL)); + } + + { + auto snapshot = state_ref.at(MerkleTreeId::L1_TO_L2_MESSAGE_TREE); + EXPECT_EQ(snapshot, + std::make_pair(bb::fr("0x14f44d672eb357739e42463497f9fdac46623af863eea4d947ca00a497dcdeb3"), 0UL)); + } + + { + // TODO (alexg) this should be the tree _after_ we insert the initial header + // currently it's the root of an empty tree + auto snapshot = state_ref.at(MerkleTreeId::ARCHIVE); + EXPECT_EQ(snapshot, + std::make_pair(bb::fr("0x14f44d672eb357739e42463497f9fdac46623af863eea4d947ca00a497dcdeb3"), 0UL)); + } +} + +TEST_F(WorldStateTest, AppendOnlyTrees) +{ + WorldState ws(1, _directory, 1024); + + std::vector tree_ids{ MerkleTreeId::NOTE_HASH_TREE, MerkleTreeId::L1_TO_L2_MESSAGE_TREE, MerkleTreeId::ARCHIVE }; + + for (auto tree_id : tree_ids) { + auto initial = ws.get_tree_info(WorldStateRevision::committed(), tree_id); + assert_leaf_status(ws, WorldStateRevision::committed(), tree_id, 0, false); + + ws.append_leaves(tree_id, { fr(42) }); + assert_leaf_value(ws, WorldStateRevision::uncommitted(), tree_id, 0, fr(42)); + assert_leaf_status(ws, WorldStateRevision::committed(), tree_id, 0, false); + assert_leaf_index(ws, WorldStateRevision::uncommitted(), tree_id, fr(42), 0); + + auto uncommitted = ws.get_tree_info(WorldStateRevision::uncommitted(), tree_id); + // uncommitted state diverges from committed state + EXPECT_EQ(uncommitted.size, initial.size + 1); + EXPECT_NE(uncommitted.root, initial.root); + + assert_sibling_path(ws, WorldStateRevision::uncommitted(), tree_id, uncommitted.root, fr(42), 0); + + auto committed = ws.get_tree_info(WorldStateRevision::committed(), tree_id); + EXPECT_EQ(committed.size, initial.size); + EXPECT_EQ(committed.root, initial.root); + + ws.commit(); + assert_leaf_value(ws, WorldStateRevision::committed(), tree_id, 0, fr(42)); + assert_leaf_index(ws, WorldStateRevision::committed(), tree_id, fr(42), 0); + + auto after_commit = ws.get_tree_info(WorldStateRevision::committed(), tree_id); + // commiting updates the committed state + EXPECT_EQ(after_commit.size, uncommitted.size); + EXPECT_EQ(after_commit.root, uncommitted.root); + + assert_sibling_path(ws, WorldStateRevision::committed(), tree_id, after_commit.root, fr(42), 0); + + ws.append_leaves(tree_id, { fr(43) }); + assert_leaf_value(ws, WorldStateRevision::uncommitted(), tree_id, 1, fr(43)); + assert_leaf_status(ws, WorldStateRevision::committed(), tree_id, 1, false); + assert_leaf_index(ws, WorldStateRevision::uncommitted(), tree_id, fr(43), 1); + + auto before_rollback = ws.get_tree_info(WorldStateRevision::uncommitted(), tree_id); + EXPECT_EQ(before_rollback.size, after_commit.size + 1); + EXPECT_NE(before_rollback.root, after_commit.root); + + ws.rollback(); + assert_leaf_status(ws, WorldStateRevision::uncommitted(), tree_id, 1, false); + assert_leaf_status(ws, WorldStateRevision::committed(), tree_id, 1, false); + + auto after_rollback = ws.get_tree_info(WorldStateRevision::committed(), tree_id); + // rollback restores the committed state + EXPECT_EQ(after_rollback.size, after_commit.size); + EXPECT_EQ(after_rollback.root, after_commit.root); + } +} + +TEST_F(WorldStateTest, AppendOnlyAllowDuplicates) +{ + WorldState ws(1, _directory, 1024); + + std::vector tree_ids{ MerkleTreeId::NOTE_HASH_TREE, MerkleTreeId::L1_TO_L2_MESSAGE_TREE, MerkleTreeId::ARCHIVE }; + + for (auto tree_id : tree_ids) { + ws.append_leaves(tree_id, { fr(42), fr(42) }); + ws.append_leaves(tree_id, { fr(42) }); + + assert_leaf_value(ws, WorldStateRevision::uncommitted(), tree_id, 0, fr(42)); + assert_leaf_value(ws, WorldStateRevision::uncommitted(), tree_id, 1, fr(42)); + assert_leaf_value(ws, WorldStateRevision::uncommitted(), tree_id, 2, fr(42)); + + ws.commit(); + + assert_leaf_value(ws, WorldStateRevision::committed(), tree_id, 0, fr(42)); + assert_leaf_value(ws, WorldStateRevision::committed(), tree_id, 1, fr(42)); + assert_leaf_value(ws, WorldStateRevision::committed(), tree_id, 2, fr(42)); + } +} + +TEST_F(WorldStateTest, NullifierTree) +{ + WorldState ws(1, _directory, 1024); + auto tree_id = MerkleTreeId::NULLIFIER_TREE; + NullifierLeafValue test_nullifier(142); + + auto predecessor_of_142 = + ws.find_low_leaf_index(WorldStateRevision::committed(), tree_id, test_nullifier.get_key()); + EXPECT_EQ(predecessor_of_142, std::make_pair(false, 127UL)); + + ws.append_leaves(tree_id, { test_nullifier }); + assert_leaf_value(ws, WorldStateRevision::uncommitted(), tree_id, 128, test_nullifier); + + ws.commit(); + + auto test_leaf = ws.get_indexed_leaf(WorldStateRevision::committed(), tree_id, 128); + // at this point 142 should be the biggest leaf so it wraps back to 0 + EXPECT_EQ(test_leaf.value(), IndexedLeaf(test_nullifier, 0, 0)); + + auto predecessor_of_142_again = + ws.find_low_leaf_index(WorldStateRevision::committed(), tree_id, test_nullifier.get_key()); + EXPECT_EQ(predecessor_of_142_again, std::make_pair(true, 128UL)); + + auto predecessor_of_143 = ws.find_low_leaf_index(WorldStateRevision::committed(), tree_id, 143); + EXPECT_EQ(predecessor_of_143, std::make_pair(false, 128UL)); // predecessor is going to be nullifier 142 on slot 127 + + auto info = ws.get_tree_info(WorldStateRevision::committed(), tree_id); + assert_sibling_path(ws, + WorldStateRevision::committed(), + tree_id, + info.root, + HashPolicy::hash(test_leaf.value().get_hash_inputs()), + 128); +} + +TEST_F(WorldStateTest, NullifierTreeDuplicates) +{ + WorldState ws(1, _directory, 1024); + auto tree_id = MerkleTreeId::NULLIFIER_TREE; + NullifierLeafValue test_nullifier(142); + + ws.append_leaves(tree_id, { test_nullifier }); + ws.commit(); + + assert_tree_size(ws, WorldStateRevision::committed(), tree_id, 129); + EXPECT_THROW(ws.append_leaves(tree_id, { test_nullifier }), std::runtime_error); + assert_tree_size(ws, WorldStateRevision::committed(), tree_id, 129); +} + +TEST_F(WorldStateTest, NullifierBatchInsert) +{ + WorldState ws(1, _directory, 1024); + auto response = ws.batch_insert_indexed_leaves( + MerkleTreeId::NULLIFIER_TREE, { NullifierLeafValue(150), NullifierLeafValue(142), NullifierLeafValue(180) }, 2); + + std::vector> expected_sorted_leaves = { { NullifierLeafValue(180), 2 }, + { NullifierLeafValue(150), 0 }, + { NullifierLeafValue(142), 1 } }; + EXPECT_EQ(response.sorted_leaves, expected_sorted_leaves); + + { + // insertion happens in descending order, but keeping original indices + // first insert leaf 180, at index 130 (tree had size 127, 180 is the third item => 127 + 3) + // predecessor will be 127, currently linked to head of the list (0) + auto low_leaf = response.low_leaf_witness_data[0]; + auto expected_low_leaf = IndexedLeaf(NullifierLeafValue(127), 0, fr(0)); + EXPECT_EQ(low_leaf.index, 127); + EXPECT_EQ(low_leaf.leaf, expected_low_leaf); + } + + { + // insert 150 on position 128 (127 + 1) + // predecessor will be 127 linked to 180 + auto low_leaf = response.low_leaf_witness_data[1]; + auto expected_low_leaf = IndexedLeaf(NullifierLeafValue(127), 130, fr(180)); + EXPECT_EQ(low_leaf.index, 127); + EXPECT_EQ(low_leaf.leaf, expected_low_leaf); + } + + { + // finally, insert 142 on position 129(127 + 2) + // prededecessor will be 127 linked to 150 + auto low_leaf = response.low_leaf_witness_data[2]; + auto expected_low_leaf = IndexedLeaf(NullifierLeafValue(127), 128, fr(150)); + EXPECT_EQ(low_leaf.index, 127); + EXPECT_EQ(low_leaf.leaf, expected_low_leaf); + } +} + +TEST_F(WorldStateTest, PublicDataTree) +{ + WorldState ws(1, _directory, 1024); + + ws.append_leaves(MerkleTreeId::PUBLIC_DATA_TREE, std::vector{ PublicDataLeafValue(142, 0) }); + assert_tree_size(ws, WorldStateRevision::uncommitted(), MerkleTreeId::PUBLIC_DATA_TREE, 129); + + auto leaf = ws.get_indexed_leaf( + WorldStateRevision::uncommitted(), MerkleTreeId::PUBLIC_DATA_TREE, 128); + EXPECT_EQ(leaf.value().value, PublicDataLeafValue(142, 0)); + + ws.update_public_data(PublicDataLeafValue(142, 1)); + // updating insert a dummy leaf + assert_tree_size(ws, WorldStateRevision::uncommitted(), MerkleTreeId::PUBLIC_DATA_TREE, 130); + + leaf = ws.get_indexed_leaf( + WorldStateRevision::uncommitted(), MerkleTreeId::PUBLIC_DATA_TREE, 128); + EXPECT_EQ(leaf.value().value, PublicDataLeafValue(142, 1)); +} + +TEST_F(WorldStateTest, CommitsAndRollsBackAllTrees) +{ + WorldState ws(1, _directory, 1024); + + ws.append_leaves(MerkleTreeId::NOTE_HASH_TREE, { fr(42) }); + ws.append_leaves(MerkleTreeId::L1_TO_L2_MESSAGE_TREE, { fr(42) }); + ws.append_leaves(MerkleTreeId::ARCHIVE, { fr(42) }); + ws.append_leaves(MerkleTreeId::NULLIFIER_TREE, { NullifierLeafValue(142) }); + ws.append_leaves(MerkleTreeId::PUBLIC_DATA_TREE, { PublicDataLeafValue(142, 1) }); + + ws.commit(); + + assert_leaf_value(ws, WorldStateRevision::committed(), MerkleTreeId::NOTE_HASH_TREE, 0, fr(42)); + assert_leaf_value(ws, WorldStateRevision::committed(), MerkleTreeId::L1_TO_L2_MESSAGE_TREE, 0, fr(42)); + assert_leaf_value(ws, WorldStateRevision::committed(), MerkleTreeId::ARCHIVE, 0, fr(42)); + assert_leaf_value(ws, WorldStateRevision::committed(), MerkleTreeId::NULLIFIER_TREE, 128, NullifierLeafValue(142)); + assert_leaf_value( + ws, WorldStateRevision::committed(), MerkleTreeId::PUBLIC_DATA_TREE, 128, PublicDataLeafValue(142, 1)); + + ws.append_leaves(MerkleTreeId::NOTE_HASH_TREE, { fr(43) }); + ws.append_leaves(MerkleTreeId::L1_TO_L2_MESSAGE_TREE, { fr(43) }); + ws.append_leaves(MerkleTreeId::ARCHIVE, { fr(43) }); + ws.append_leaves(MerkleTreeId::NULLIFIER_TREE, { NullifierLeafValue(143) }); + ws.append_leaves(MerkleTreeId::PUBLIC_DATA_TREE, { PublicDataLeafValue(143, 1) }); + + ws.rollback(); + + assert_leaf_exists(ws, WorldStateRevision::committed(), MerkleTreeId::NOTE_HASH_TREE, fr(43), false); + assert_leaf_exists(ws, WorldStateRevision::committed(), MerkleTreeId::L1_TO_L2_MESSAGE_TREE, fr(43), false); + assert_leaf_exists(ws, WorldStateRevision::committed(), MerkleTreeId::ARCHIVE, fr(43), false); + assert_leaf_exists( + ws, WorldStateRevision::committed(), MerkleTreeId::NULLIFIER_TREE, NullifierLeafValue(143), false); + assert_leaf_exists( + ws, WorldStateRevision::committed(), MerkleTreeId::PUBLIC_DATA_TREE, PublicDataLeafValue(143, 1), false); +} + +TEST_F(WorldStateTest, SyncExternalBlockFromEmpty) +{ + WorldState ws(1, _directory, 1024); + StateReference block_state_ref = { + { MerkleTreeId::NULLIFIER_TREE, + { fr("0x0342578609a7358092788d0eed7d1ee0ec8e0c596c0b1e85ba980ddd5cc79d04"), 129 } }, + { MerkleTreeId::NOTE_HASH_TREE, + { fr("0x15dad063953d8d216c1db77739d6fb27e1b73a5beef748a1208898b3428781eb"), 1 } }, + { MerkleTreeId::PUBLIC_DATA_TREE, + { fr("0x0278dcf9ff541da255ee722aecfad849b66af0d42c2924d949b5a509f2e1aec9"), 129 } }, + { MerkleTreeId::L1_TO_L2_MESSAGE_TREE, + { fr("0x20ea8ca97f96508aaed2d6cdc4198a41c77c640bfa8785a51bb905b9a672ba0b"), 1 } }, + }; + + bool sync_res = ws.sync_block( + block_state_ref, fr(1), { 42 }, { 43 }, { NullifierLeafValue(144) }, { { PublicDataLeafValue(145, 1) } }); + EXPECT_EQ(sync_res, false); + + assert_leaf_value(ws, WorldStateRevision::committed(), MerkleTreeId::NOTE_HASH_TREE, 0, fr(42)); + assert_leaf_value(ws, WorldStateRevision::committed(), MerkleTreeId::L1_TO_L2_MESSAGE_TREE, 0, fr(43)); + assert_leaf_value(ws, WorldStateRevision::committed(), MerkleTreeId::NULLIFIER_TREE, 128, NullifierLeafValue(144)); + assert_leaf_value( + ws, WorldStateRevision::committed(), MerkleTreeId::PUBLIC_DATA_TREE, 128, PublicDataLeafValue(145, 1)); + assert_leaf_value(ws, WorldStateRevision::committed(), MerkleTreeId::ARCHIVE, 0, fr(1)); + + auto state_ref = ws.get_state_reference(WorldStateRevision::committed()); + for (const auto& [tree_id, snapshot] : block_state_ref) { + EXPECT_EQ(state_ref.at(tree_id), snapshot); + } +} + +TEST_F(WorldStateTest, SyncBlockFromDirtyState) +{ + WorldState ws(1, _directory, 1024); + StateReference block_state_ref = { + { MerkleTreeId::NULLIFIER_TREE, + { fr("0x0342578609a7358092788d0eed7d1ee0ec8e0c596c0b1e85ba980ddd5cc79d04"), 129 } }, + { MerkleTreeId::NOTE_HASH_TREE, + { fr("0x15dad063953d8d216c1db77739d6fb27e1b73a5beef748a1208898b3428781eb"), 1 } }, + { MerkleTreeId::PUBLIC_DATA_TREE, + { fr("0x0278dcf9ff541da255ee722aecfad849b66af0d42c2924d949b5a509f2e1aec9"), 129 } }, + { MerkleTreeId::L1_TO_L2_MESSAGE_TREE, + { fr("0x20ea8ca97f96508aaed2d6cdc4198a41c77c640bfa8785a51bb905b9a672ba0b"), 1 } }, + }; + + ws.append_leaves(MerkleTreeId::NOTE_HASH_TREE, { fr(142) }); + ws.append_leaves(MerkleTreeId::L1_TO_L2_MESSAGE_TREE, { fr(143) }); + ws.append_leaves(MerkleTreeId::NULLIFIER_TREE, { NullifierLeafValue(142) }); + ws.append_leaves(MerkleTreeId::PUBLIC_DATA_TREE, { PublicDataLeafValue(142, 1) }); + + auto uncommitted_state_ref = ws.get_state_reference(WorldStateRevision::uncommitted()); + for (const auto& [tree_id, snapshot] : block_state_ref) { + EXPECT_NE(uncommitted_state_ref.at(tree_id), snapshot); + } + + bool sync_res = ws.sync_block( + block_state_ref, fr(1), { 42 }, { 43 }, { NullifierLeafValue(144) }, { { PublicDataLeafValue(145, 1) } }); + EXPECT_EQ(sync_res, false); + + assert_leaf_value(ws, WorldStateRevision::committed(), MerkleTreeId::NOTE_HASH_TREE, 0, fr(42)); + assert_leaf_value(ws, WorldStateRevision::committed(), MerkleTreeId::L1_TO_L2_MESSAGE_TREE, 0, fr(43)); + assert_leaf_value(ws, WorldStateRevision::committed(), MerkleTreeId::NULLIFIER_TREE, 128, NullifierLeafValue(144)); + assert_leaf_value( + ws, WorldStateRevision::committed(), MerkleTreeId::PUBLIC_DATA_TREE, 128, PublicDataLeafValue(145, 1)); + assert_leaf_value(ws, WorldStateRevision::committed(), MerkleTreeId::ARCHIVE, 0, fr(1)); + + auto state_ref = ws.get_state_reference(WorldStateRevision::committed()); + for (const auto& [tree_id, snapshot] : block_state_ref) { + EXPECT_EQ(state_ref.at(tree_id), snapshot); + } +} + +TEST_F(WorldStateTest, SyncCurrentBlock) +{ + WorldState ws(1, _directory, 1024); + StateReference block_state_ref = { + { MerkleTreeId::NULLIFIER_TREE, + { fr("0x0342578609a7358092788d0eed7d1ee0ec8e0c596c0b1e85ba980ddd5cc79d04"), 129 } }, + { MerkleTreeId::NOTE_HASH_TREE, + { fr("0x15dad063953d8d216c1db77739d6fb27e1b73a5beef748a1208898b3428781eb"), 1 } }, + { MerkleTreeId::PUBLIC_DATA_TREE, + { fr("0x0278dcf9ff541da255ee722aecfad849b66af0d42c2924d949b5a509f2e1aec9"), 129 } }, + { MerkleTreeId::L1_TO_L2_MESSAGE_TREE, + { fr("0x20ea8ca97f96508aaed2d6cdc4198a41c77c640bfa8785a51bb905b9a672ba0b"), 1 } }, + }; + + ws.append_leaves(MerkleTreeId::NOTE_HASH_TREE, { 42 }); + ws.append_leaves(MerkleTreeId::L1_TO_L2_MESSAGE_TREE, { 43 }); + ws.append_leaves(MerkleTreeId::NULLIFIER_TREE, { NullifierLeafValue(144) }); + ws.append_leaves(MerkleTreeId::PUBLIC_DATA_TREE, { PublicDataLeafValue(145, 1) }); + + auto uncommitted_state_ref = ws.get_state_reference(WorldStateRevision::uncommitted()); + for (const auto& [tree_id, snapshot] : block_state_ref) { + EXPECT_EQ(uncommitted_state_ref.at(tree_id), snapshot); + } + + bool sync_res = ws.sync_block( + block_state_ref, fr(1), { 42 }, { 43 }, { NullifierLeafValue(144) }, { { PublicDataLeafValue(145, 1) } }); + EXPECT_EQ(sync_res, true); + + assert_leaf_value(ws, WorldStateRevision::uncommitted(), MerkleTreeId::ARCHIVE, 0, fr(1)); + + auto state_ref = ws.get_state_reference(WorldStateRevision::committed()); + for (const auto& [tree_id, snapshot] : block_state_ref) { + EXPECT_EQ(state_ref.at(tree_id), snapshot); + } +} diff --git a/barretenberg/cpp/src/barretenberg/world_state_napi/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/world_state_napi/CMakeLists.txt new file mode 100644 index 00000000000..5c1ba6d9159 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/world_state_napi/CMakeLists.txt @@ -0,0 +1,30 @@ +# see https://nodejs.org/dist/latest/docs/api/n-api.html#node-api-version-matrix +add_definitions(-DNAPI_VERSION=9) + +file(GLOB_RECURSE SOURCE_FILES *.cpp) +file(GLOB_RECURSE HEADER_FILES *.hpp *.tcc) + +execute_process( + COMMAND yarn --frozen-lockfile --prefer-offline + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} +) + +execute_process( + COMMAND node -p "require('node-addon-api').include" + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + OUTPUT_VARIABLE NODE_ADDON_API_DIR +) + +execute_process( + COMMAND node -p "require('node-api-headers').include_dir" + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + OUTPUT_VARIABLE NODE_API_HEADERS_DIR +) + +string(REGEX REPLACE "[\r\n\"]" "" NODE_ADDON_API_DIR ${NODE_ADDON_API_DIR}) +string(REGEX REPLACE "[\r\n\"]" "" NODE_API_HEADERS_DIR ${NODE_API_HEADERS_DIR}) + +add_library(world_state_napi SHARED ${SOURCE_FILES}) +set_target_properties(world_state_napi PROPERTIES PREFIX "" SUFFIX ".node") +target_include_directories(world_state_napi PRIVATE ${NODE_API_HEADERS_DIR} ${NODE_ADDON_API_DIR}) +target_link_libraries(world_state_napi PRIVATE world_state) diff --git a/barretenberg/cpp/src/barretenberg/world_state_napi/addon.cpp b/barretenberg/cpp/src/barretenberg/world_state_napi/addon.cpp new file mode 100644 index 00000000000..d7b87eb53b6 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/world_state_napi/addon.cpp @@ -0,0 +1,485 @@ +#include "barretenberg/world_state_napi/addon.hpp" +#include "barretenberg/crypto/merkle_tree/hash_path.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/ecc/curves/bn254/fr.hpp" +#include "barretenberg/messaging/header.hpp" +#include "barretenberg/world_state/types.hpp" +#include "barretenberg/world_state/world_state.hpp" +#include "barretenberg/world_state_napi/async_op.hpp" +#include "barretenberg/world_state_napi/message.hpp" +#include "msgpack/v3/pack_decl.hpp" +#include "msgpack/v3/sbuffer_decl.hpp" +#include "napi.h" +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace bb::world_state; +using namespace bb::crypto::merkle_tree; +using namespace bb::messaging; + +WorldStateAddon::WorldStateAddon(const Napi::CallbackInfo& info) + : ObjectWrap(info) +{ + Napi::Env env = info.Env(); + + if (info.Length() < 1) { + throw Napi::TypeError::New(env, "Wrong number of arguments"); + } + + if (!info[0].IsString()) { + throw Napi::TypeError::New(env, "Directory needs to be a string"); + } + + std::string data_dir = info[0].As(); + _ws = std::make_unique(16, data_dir, 1024 * 1024); // 1 GiB + + _dispatcher.registerTarget( + WorldStateMessageType::GET_TREE_INFO, + [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return get_tree_info(obj, buffer); }); + + _dispatcher.registerTarget( + WorldStateMessageType::GET_STATE_REFERENCE, + [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return get_state_reference(obj, buffer); }); + + _dispatcher.registerTarget( + WorldStateMessageType::GET_LEAF_VALUE, + [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return get_leaf_value(obj, buffer); }); + + _dispatcher.registerTarget( + WorldStateMessageType::GET_LEAF_PREIMAGE, + [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return get_leaf_preimage(obj, buffer); }); + + _dispatcher.registerTarget( + WorldStateMessageType::GET_SIBLING_PATH, + [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return get_sibling_path(obj, buffer); }); + + _dispatcher.registerTarget( + WorldStateMessageType::FIND_LEAF_INDEX, + [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return find_leaf_index(obj, buffer); }); + + _dispatcher.registerTarget( + WorldStateMessageType::FIND_LOW_LEAF, + [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return find_low_leaf(obj, buffer); }); + + _dispatcher.registerTarget( + WorldStateMessageType::APPEND_LEAVES, + [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return append_leaves(obj, buffer); }); + + _dispatcher.registerTarget( + WorldStateMessageType::BATCH_INSERT, + [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return batch_insert(obj, buffer); }); + + _dispatcher.registerTarget( + WorldStateMessageType::UPDATE_ARCHIVE, + [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return update_archive(obj, buffer); }); + + _dispatcher.registerTarget(WorldStateMessageType::COMMIT, + [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return commit(obj, buffer); }); + + _dispatcher.registerTarget(WorldStateMessageType::ROLLBACK, [this](msgpack::object& obj, msgpack::sbuffer& buffer) { + return rollback(obj, buffer); + }); + + _dispatcher.registerTarget( + WorldStateMessageType::SYNC_BLOCK, + [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return sync_block(obj, buffer); }); +} + +Napi::Value WorldStateAddon::call(const Napi::CallbackInfo& info) +{ + Napi::Env env = info.Env(); + // keep this in a shared pointer so that AsyncOperation can resolve/reject the promise once the execution is + // complete on an separate thread + auto deferred = std::make_shared(env); + + if (info.Length() < 1) { + deferred->Reject(Napi::TypeError::New(env, "Wrong number of arguments").Value()); + } else if (!info[0].IsBuffer()) { + deferred->Reject(Napi::TypeError::New(env, "Argument must be a buffer").Value()); + } else { + auto buffer = info[0].As>(); + size_t length = buffer.Length(); + // we mustn't access the Napi::Env outside of this top-level function + // so copy the data to a variable we own + // and make it a shared pointer so that it doesn't get destroyed as soon as we exit this code block + auto data = std::make_shared>(length); + std::copy_n(buffer.Data(), length, data->data()); + + auto* op = new AsyncOperation(env, deferred, [=, this](msgpack::sbuffer& buf) { + msgpack::object_handle obj_handle = msgpack::unpack(data->data(), length); + msgpack::object obj = obj_handle.get(); + _dispatcher.onNewData(obj, buf); + }); + + // Napi is now responsible for destroying this object + op->Queue(); + } + + return deferred->Promise(); +} + +bool WorldStateAddon::get_tree_info(msgpack::object& obj, msgpack::sbuffer& buffer) const +{ + TypedMessage request; + obj.convert(request); + auto info = _ws->get_tree_info(revision_from_input(request.value.revision), request.value.treeId); + + MsgHeader header(request.header.messageId); + messaging::TypedMessage resp_msg( + WorldStateMessageType::GET_TREE_INFO, header, { request.value.treeId, info.root, info.size, info.depth }); + + msgpack::pack(buffer, resp_msg); + + return true; +} + +bool WorldStateAddon::get_state_reference(msgpack::object& obj, msgpack::sbuffer& buffer) const +{ + TypedMessage request; + obj.convert(request); + auto state = _ws->get_state_reference(revision_from_input(request.value.revision)); + + MsgHeader header(request.header.messageId); + messaging::TypedMessage resp_msg( + WorldStateMessageType::GET_STATE_REFERENCE, header, { state }); + + msgpack::pack(buffer, resp_msg); + + return true; +} + +bool WorldStateAddon::get_leaf_value(msgpack::object& obj, msgpack::sbuffer& buffer) const +{ + TypedMessage request; + obj.convert(request); + + switch (request.value.treeId) { + case MerkleTreeId::NOTE_HASH_TREE: + case MerkleTreeId::L1_TO_L2_MESSAGE_TREE: + case MerkleTreeId::ARCHIVE: { + auto leaf = _ws->get_leaf( + revision_from_input(request.value.revision), request.value.treeId, request.value.leafIndex); + + MsgHeader header(request.header.messageId); + messaging::TypedMessage> resp_msg(WorldStateMessageType::GET_LEAF_VALUE, header, leaf); + msgpack::pack(buffer, resp_msg); + break; + } + + case MerkleTreeId::PUBLIC_DATA_TREE: { + auto leaf = _ws->get_leaf( + revision_from_input(request.value.revision), request.value.treeId, request.value.leafIndex); + MsgHeader header(request.header.messageId); + messaging::TypedMessage> resp_msg( + WorldStateMessageType::GET_LEAF_VALUE, header, leaf); + msgpack::pack(buffer, resp_msg); + break; + } + + case MerkleTreeId::NULLIFIER_TREE: { + auto leaf = _ws->get_leaf( + revision_from_input(request.value.revision), request.value.treeId, request.value.leafIndex); + MsgHeader header(request.header.messageId); + messaging::TypedMessage> resp_msg( + WorldStateMessageType::GET_LEAF_VALUE, header, leaf); + msgpack::pack(buffer, resp_msg); + break; + } + + default: + throw std::runtime_error("Unsupported tree type"); + } + + return true; +} + +bool WorldStateAddon::get_leaf_preimage(msgpack::object& obj, msgpack::sbuffer& buffer) const +{ + TypedMessage request; + obj.convert(request); + + MsgHeader header(request.header.messageId); + + switch (request.value.treeId) { + case MerkleTreeId::NULLIFIER_TREE: { + auto leaf = _ws->get_indexed_leaf( + revision_from_input(request.value.revision), request.value.treeId, request.value.leafIndex); + messaging::TypedMessage>> resp_msg( + WorldStateMessageType::GET_LEAF_PREIMAGE, header, leaf); + msgpack::pack(buffer, resp_msg); + break; + } + + case MerkleTreeId::PUBLIC_DATA_TREE: { + auto leaf = _ws->get_indexed_leaf( + revision_from_input(request.value.revision), request.value.treeId, request.value.leafIndex); + + messaging::TypedMessage>> resp_msg( + WorldStateMessageType::GET_LEAF_PREIMAGE, header, leaf); + msgpack::pack(buffer, resp_msg); + break; + } + + default: + throw std::runtime_error("Unsupported tree type"); + } + + return true; +} + +bool WorldStateAddon::get_sibling_path(msgpack::object& obj, msgpack::sbuffer& buffer) const +{ + TypedMessage request; + obj.convert(request); + + fr_sibling_path path = _ws->get_sibling_path( + revision_from_input(request.value.revision), request.value.treeId, request.value.leafIndex); + + MsgHeader header(request.header.messageId); + messaging::TypedMessage resp_msg(WorldStateMessageType::GET_SIBLING_PATH, header, path); + + msgpack::pack(buffer, resp_msg); + + return true; +} + +bool WorldStateAddon::find_leaf_index(msgpack::object& obj, msgpack::sbuffer& buffer) const +{ + TypedMessage request; + obj.convert(request); + + std::optional index; + switch (request.value.treeId) { + case MerkleTreeId::NOTE_HASH_TREE: + case MerkleTreeId::L1_TO_L2_MESSAGE_TREE: + case MerkleTreeId::ARCHIVE: { + TypedMessage> r1; + obj.convert(r1); + index = _ws->find_leaf_index( + revision_from_input(request.value.revision), request.value.treeId, r1.value.leaf); + break; + } + + case MerkleTreeId::PUBLIC_DATA_TREE: { + TypedMessage> r2; + obj.convert(r2); + index = _ws->find_leaf_index( + revision_from_input(request.value.revision), request.value.treeId, r2.value.leaf); + break; + } + case MerkleTreeId::NULLIFIER_TREE: { + TypedMessage> r3; + obj.convert(r3); + index = _ws->find_leaf_index( + revision_from_input(request.value.revision), request.value.treeId, r3.value.leaf); + break; + } + } + + MsgHeader header(request.header.messageId); + messaging::TypedMessage> resp_msg(WorldStateMessageType::FIND_LEAF_INDEX, header, index); + msgpack::pack(buffer, resp_msg); + + return true; +} + +bool WorldStateAddon::find_low_leaf(msgpack::object& obj, msgpack::sbuffer& buffer) const +{ + TypedMessage request; + obj.convert(request); + + std::pair low_leaf_info = + _ws->find_low_leaf_index(revision_from_input(request.value.revision), request.value.treeId, request.value.key); + + MsgHeader header(request.header.messageId); + TypedMessage response( + WorldStateMessageType::FIND_LOW_LEAF, header, { low_leaf_info.first, low_leaf_info.second }); + msgpack::pack(buffer, response); + + return true; +} + +bool WorldStateAddon::append_leaves(msgpack::object& obj, msgpack::sbuffer& buf) +{ + TypedMessage request; + obj.convert(request); + + switch (request.value.treeId) { + case MerkleTreeId::NOTE_HASH_TREE: + case MerkleTreeId::L1_TO_L2_MESSAGE_TREE: + case MerkleTreeId::ARCHIVE: { + TypedMessage> r1; + obj.convert(r1); + _ws->append_leaves(r1.value.treeId, r1.value.leaves); + break; + } + case MerkleTreeId::PUBLIC_DATA_TREE: { + TypedMessage> r2; + obj.convert(r2); + _ws->append_leaves(r2.value.treeId, r2.value.leaves); + break; + } + case MerkleTreeId::NULLIFIER_TREE: { + TypedMessage> r3; + obj.convert(r3); + _ws->append_leaves(r3.value.treeId, r3.value.leaves); + break; + } + } + + MsgHeader header(request.header.messageId); + messaging::TypedMessage resp_msg(WorldStateMessageType::APPEND_LEAVES, header, {}); + msgpack::pack(buf, resp_msg); + + return true; +} + +bool WorldStateAddon::batch_insert(msgpack::object& obj, msgpack::sbuffer& buffer) +{ + TypedMessage request; + obj.convert(request); + + switch (request.value.treeId) { + case MerkleTreeId::PUBLIC_DATA_TREE: { + TypedMessage> r1; + obj.convert(r1); + auto result = _ws->batch_insert_indexed_leaves( + request.value.treeId, r1.value.leaves, r1.value.subtreeDepth); + MsgHeader header(request.header.messageId); + messaging::TypedMessage> resp_msg( + WorldStateMessageType::BATCH_INSERT, header, result); + msgpack::pack(buffer, resp_msg); + + break; + } + case MerkleTreeId::NULLIFIER_TREE: { + TypedMessage> r2; + obj.convert(r2); + auto result = _ws->batch_insert_indexed_leaves( + request.value.treeId, r2.value.leaves, r2.value.subtreeDepth); + MsgHeader header(request.header.messageId); + messaging::TypedMessage> resp_msg( + WorldStateMessageType::BATCH_INSERT, header, result); + msgpack::pack(buffer, resp_msg); + break; + } + default: + throw std::runtime_error("Unsupported tree type"); + } + + return true; +} + +bool WorldStateAddon::update_archive(msgpack::object& obj, msgpack::sbuffer& buf) +{ + TypedMessage request; + obj.convert(request); + + // TODO (alexg) move this to world state + auto world_state_ref = _ws->get_state_reference(WorldStateRevision::uncommitted()); + auto block_state_ref = request.value.blockStateRef; + + if (block_state_ref[MerkleTreeId::NULLIFIER_TREE] == world_state_ref[MerkleTreeId::NULLIFIER_TREE] && + block_state_ref[MerkleTreeId::NOTE_HASH_TREE] == world_state_ref[MerkleTreeId::NOTE_HASH_TREE] && + block_state_ref[MerkleTreeId::PUBLIC_DATA_TREE] == world_state_ref[MerkleTreeId::PUBLIC_DATA_TREE] && + block_state_ref[MerkleTreeId::L1_TO_L2_MESSAGE_TREE] == world_state_ref[MerkleTreeId::L1_TO_L2_MESSAGE_TREE]) { + _ws->append_leaves(MerkleTreeId::ARCHIVE, { request.value.blockHash }); + } else { + throw std::runtime_error("Block state reference does not match current state"); + } + + MsgHeader header(request.header.messageId); + messaging::TypedMessage resp_msg(WorldStateMessageType::APPEND_LEAVES, header, {}); + msgpack::pack(buf, resp_msg); + + return true; +} + +bool WorldStateAddon::commit(msgpack::object& obj, msgpack::sbuffer& buf) +{ + HeaderOnlyMessage request; + obj.convert(request); + + _ws->commit(); + + MsgHeader header(request.header.messageId); + messaging::TypedMessage resp_msg(WorldStateMessageType::COMMIT, header, {}); + msgpack::pack(buf, resp_msg); + + return true; +} + +bool WorldStateAddon::rollback(msgpack::object& obj, msgpack::sbuffer& buf) +{ + HeaderOnlyMessage request; + obj.convert(request); + + _ws->rollback(); + + MsgHeader header(request.header.messageId); + messaging::TypedMessage resp_msg(WorldStateMessageType::ROLLBACK, header, {}); + msgpack::pack(buf, resp_msg); + + return true; +} + +bool WorldStateAddon::sync_block(msgpack::object& obj, msgpack::sbuffer& buf) +{ + TypedMessage request; + obj.convert(request); + + bool is_block_ours = _ws->sync_block(request.value.blockStateRef, + request.value.blockHash, + request.value.paddedNoteHashes, + request.value.paddedL1ToL2Messages, + request.value.paddedNullifiers, + request.value.batchesOfPaddedPublicDataWrites); + + MsgHeader header(request.header.messageId); + messaging::TypedMessage resp_msg(WorldStateMessageType::SYNC_BLOCK, header, { is_block_ours }); + msgpack::pack(buf, resp_msg); + + return true; +} + +WorldStateRevision WorldStateAddon::revision_from_input(int input) +{ + if (input == -1) { + return WorldStateRevision::uncommitted(); + } + + if (input == 0) { + return WorldStateRevision::committed(); + } + + if (input > 0) { + return WorldStateRevision::finalised_block(static_cast(input)); + } + + throw std::runtime_error("Revision must be -1, 0, or a positive integer"); +} + +Napi::Function WorldStateAddon::get_class(Napi::Env env) +{ + return DefineClass(env, + "WorldState", + { + WorldStateAddon::InstanceMethod("call", &WorldStateAddon::call), + }); +} + +Napi::Object Init(Napi::Env env, Napi::Object exports) +{ + Napi::String name = Napi::String::New(env, "WorldState"); + exports.Set(name, WorldStateAddon::get_class(env)); + return exports; +} + +// NOLINTNEXTLINE +NODE_API_MODULE(addon, Init) diff --git a/barretenberg/cpp/src/barretenberg/world_state_napi/addon.hpp b/barretenberg/cpp/src/barretenberg/world_state_napi/addon.hpp new file mode 100644 index 00000000000..c80f544f845 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/world_state_napi/addon.hpp @@ -0,0 +1,57 @@ +#pragma once + +#include "barretenberg/messaging/dispatcher.hpp" +#include "barretenberg/world_state/types.hpp" +#include "barretenberg/world_state/world_state.hpp" +#include "barretenberg/world_state_napi/message.hpp" +#include +#include +#include + +namespace bb::world_state { + +/** + * @brief Manages the interaction between the JavaScript runtime and the WorldState class. + */ +class WorldStateAddon : public Napi::ObjectWrap { + public: + WorldStateAddon(const Napi::CallbackInfo&); + + /** + * @brief The only instance method exposed to JavaScript. Takes a msgpack Message and returns a Promise + */ + Napi::Value call(const Napi::CallbackInfo&); + + /** + * @brief Register the WorldStateAddon class with the JavaScript runtime. + */ + static Napi::Function get_class(Napi::Env); + + private: + std::unique_ptr _ws; + bb::messaging::MessageDispatcher _dispatcher; + + bool get_tree_info(msgpack::object& obj, msgpack::sbuffer& buffer) const; + bool get_state_reference(msgpack::object& obj, msgpack::sbuffer& buffer) const; + + bool get_leaf_value(msgpack::object& obj, msgpack::sbuffer& buffer) const; + bool get_leaf_preimage(msgpack::object& obj, msgpack::sbuffer& buffer) const; + bool get_sibling_path(msgpack::object& obj, msgpack::sbuffer& buffer) const; + + bool find_leaf_index(msgpack::object& obj, msgpack::sbuffer& buffer) const; + bool find_low_leaf(msgpack::object& obj, msgpack::sbuffer& buffer) const; + + bool append_leaves(msgpack::object& obj, msgpack::sbuffer& buffer); + bool batch_insert(msgpack::object& obj, msgpack::sbuffer& buffer); + + bool update_archive(msgpack::object& obj, msgpack::sbuffer& buffer); + + bool commit(msgpack::object& obj, msgpack::sbuffer& buffer); + bool rollback(msgpack::object& obj, msgpack::sbuffer& buffer); + + bool sync_block(msgpack::object& obj, msgpack::sbuffer& buffer); + + static WorldStateRevision revision_from_input(int input); +}; + +} // namespace bb::world_state diff --git a/barretenberg/cpp/src/barretenberg/world_state_napi/async_op.hpp b/barretenberg/cpp/src/barretenberg/world_state_napi/async_op.hpp new file mode 100644 index 00000000000..e5a4849b38a --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/world_state_napi/async_op.hpp @@ -0,0 +1,64 @@ +#pragma once + +#include "barretenberg/serialize/cbind.hpp" +#include +#include +#include + +namespace bb::world_state { + +using async_fn = std::function; + +/** + * @brief Encapsulatest some work that can be done off the JavaScript main thread + * + * This class takes a Deferred instance (i.e. a Promise to JS), execute some work in a separate thread, and then report + * back on the result. The async execution _must not_ touch the JS environment. Everything that's needed to complete the + * work must be copied into memory owned by the C++ code. The same has to be done when reporting back the result: keep + * the result in memory owned by the C++ code and copy it back to the JS environment in the OnOK/OnError methods. + * + * OnOK/OnError will be called on the main JS thread, so it's safe to interact with the JS environment there. + * + * Instances of this class are managed by the NodeJS environment and execute on a libuv thread. + * Docs + * . - https://github.com/nodejs/node-addon-api/blob/cc06369aa4dd29e585600b8b47839c1297df962d/doc/async_worker.md + * . - https://nodejs.github.io/node-addon-examples/special-topics/asyncworker + */ +class AsyncOperation : public Napi::AsyncWorker { + public: + AsyncOperation(Napi::Env env, std::shared_ptr deferred, async_fn fn) + : Napi::AsyncWorker(env) + , _fn(std::move(fn)) + , _deferred(std::move(deferred)) + {} + + AsyncOperation(const AsyncOperation&) = delete; + AsyncOperation& operator=(const AsyncOperation&) = delete; + AsyncOperation(AsyncOperation&&) = delete; + AsyncOperation& operator=(AsyncOperation&&) = delete; + + ~AsyncOperation() override = default; + + void Execute() override + { + try { + _fn(_result); + } catch (const std::exception& e) { + SetError(e.what()); + } + } + + void OnOK() override + { + auto buf = Napi::Buffer::Copy(Env(), _result.data(), _result.size()); + _deferred->Resolve(buf); + } + void OnError(const Napi::Error& e) override { _deferred->Reject(e.Value()); } + + private: + async_fn _fn; + std::shared_ptr _deferred; + msgpack::sbuffer _result; +}; + +} // namespace bb::world_state diff --git a/barretenberg/cpp/src/barretenberg/world_state_napi/message.hpp b/barretenberg/cpp/src/barretenberg/world_state_napi/message.hpp new file mode 100644 index 00000000000..e809028f7a9 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/world_state_napi/message.hpp @@ -0,0 +1,162 @@ +#pragma once +#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/ecc/curves/bn254/fr.hpp" +#include "barretenberg/messaging/header.hpp" +#include "barretenberg/serialize/msgpack.hpp" +#include "barretenberg/world_state/types.hpp" +#include "barretenberg/world_state/world_state.hpp" +#include +#include + +namespace bb::world_state { + +using namespace bb::messaging; + +enum WorldStateMessageType { + GET_TREE_INFO = FIRST_APP_MSG_TYPE, + GET_STATE_REFERENCE, + + GET_LEAF_VALUE, + GET_LEAF_PREIMAGE, + GET_SIBLING_PATH, + + FIND_LEAF_INDEX, + FIND_LOW_LEAF, + + APPEND_LEAVES, + BATCH_INSERT, + + UPDATE_ARCHIVE, + + COMMIT, + ROLLBACK, + + SYNC_BLOCK +}; + +struct TreeIdOnlyRequest { + MerkleTreeId treeId; + MSGPACK_FIELDS(treeId); +}; + +struct TreeIdAndRevisionRequest { + MerkleTreeId treeId; + // -1 uncomitted state + // 0 latest committed state + // > 0 specific block number + int revision; + MSGPACK_FIELDS(treeId, revision); +}; + +struct EmptyResponse { + bool ok{ true }; + MSGPACK_FIELDS(ok); +}; + +struct GetTreeInfoRequest { + MerkleTreeId treeId; + int revision; + MSGPACK_FIELDS(treeId, revision); +}; + +struct GetTreeInfoResponse { + MerkleTreeId treeId; + fr root; + index_t size; + uint32_t depth; + MSGPACK_FIELDS(treeId, root, size, depth); +}; + +struct GetStateReferenceRequest { + int revision; + MSGPACK_FIELDS(revision); +}; + +struct GetStateReferenceResponse { + StateReference state; + MSGPACK_FIELDS(state); +}; + +struct GetLeafValueRequest { + MerkleTreeId treeId; + int revision; + index_t leafIndex; + MSGPACK_FIELDS(treeId, revision, leafIndex); +}; + +struct GetLeafPreimageRequest { + MerkleTreeId treeId; + int revision; + index_t leafIndex; + MSGPACK_FIELDS(treeId, revision, leafIndex); +}; + +struct GetSiblingPathRequest { + MerkleTreeId treeId; + int revision; + index_t leafIndex; + MSGPACK_FIELDS(treeId, revision, leafIndex); +}; + +template struct FindLeafIndexRequest { + MerkleTreeId treeId; + int revision; + T leaf; + MSGPACK_FIELDS(treeId, revision, leaf); +}; + +struct FindLowLeafRequest { + MerkleTreeId treeId; + int revision; + fr key; + MSGPACK_FIELDS(treeId, revision, key); +}; + +struct FindLowLeafResponse { + bool alreadyPresent; + index_t index; + MSGPACK_FIELDS(alreadyPresent, index); +}; + +template struct AppendLeavesRequest { + MerkleTreeId treeId; + std::vector leaves; + MSGPACK_FIELDS(treeId, leaves); +}; + +template struct BatchInsertRequest { + MerkleTreeId treeId; + std::vector leaves; + uint32_t subtreeDepth; + MSGPACK_FIELDS(treeId, leaves, subtreeDepth); +}; + +struct UpdateArchiveRequest { + StateReference blockStateRef; + bb::fr blockHash; + MSGPACK_FIELDS(blockStateRef, blockHash); +}; + +struct SyncBlockRequest { + StateReference blockStateRef; + bb::fr blockHash; + std::vector paddedNoteHashes, paddedL1ToL2Messages; + std::vector paddedNullifiers; + std::vector> batchesOfPaddedPublicDataWrites; + + MSGPACK_FIELDS(blockStateRef, + blockHash, + paddedNoteHashes, + paddedL1ToL2Messages, + paddedNullifiers, + batchesOfPaddedPublicDataWrites); +}; + +struct SyncBlockResponse { + bool isBlockOurs; + MSGPACK_FIELDS(isBlockOurs); +}; + +} // namespace bb::world_state + +MSGPACK_ADD_ENUM(bb::world_state::WorldStateMessageType) diff --git a/barretenberg/cpp/src/barretenberg/world_state_napi/package.json b/barretenberg/cpp/src/barretenberg/world_state_napi/package.json new file mode 100644 index 00000000000..d812caf6171 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/world_state_napi/package.json @@ -0,0 +1,15 @@ +{ + "name": "world_state_napi", + "private": true, + "version": "0.0.0", + "packageManager": "yarn@1.22.22+sha512.a6b2f7906b721bba3d67d4aff083df04dad64c399707841b7acf00f6b133b7ac24255f2652fa22ae3534329dc6180534e98d17432037ff6fd140556e2bb3137e", + "dependencies": { + "node-addon-api": "^8.0.0", + "node-api-headers": "^1.1.0" + }, + "binary": { + "napi_versions": [ + 9 + ] + } +} diff --git a/barretenberg/cpp/src/barretenberg/world_state_napi/yarn.lock b/barretenberg/cpp/src/barretenberg/world_state_napi/yarn.lock new file mode 100644 index 00000000000..6a671ec7eec --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/world_state_napi/yarn.lock @@ -0,0 +1,13 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +node-addon-api@^8.0.0: + version "8.0.0" + resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-8.0.0.tgz#5453b7ad59dd040d12e0f1a97a6fa1c765c5c9d2" + integrity sha512-ipO7rsHEBqa9STO5C5T10fj732ml+5kLN1cAG8/jdHd56ldQeGj3Q7+scUS+VHK/qy1zLEwC4wMK5+yM0btPvw== + +node-api-headers@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/node-api-headers/-/node-api-headers-1.1.0.tgz#3f9dd7bb10b29e1c3e3db675979605a308b2373c" + integrity sha512-ucQW+SbYCUPfprvmzBsnjT034IGRB2XK8rRc78BgjNKhTdFKgAwAmgW704bKIBmcYW48it0Gkjpkd39Azrwquw== diff --git a/noir-projects/Earthfile b/noir-projects/Earthfile index 7a3b55ef995..d679b5c72c5 100644 --- a/noir-projects/Earthfile +++ b/noir-projects/Earthfile @@ -5,6 +5,10 @@ source: # Install nargo COPY ../noir/+nargo/nargo /usr/bin/nargo + # Install bb + COPY ../barretenberg/cpp/+preset-release/bin/bb /usr/src/barretenberg/cpp/build/bin/bb + # Install world state napi + COPY ../barretenberg/cpp/+preset-release-world-state/bin/world_state_napi.node /usr/src/barretenberg/cpp/build/bin/world_state_napi.node WORKDIR /usr/src/noir-projects diff --git a/yarn-project/Earthfile b/yarn-project/Earthfile index 07be9672103..ec298067247 100644 --- a/yarn-project/Earthfile +++ b/yarn-project/Earthfile @@ -6,7 +6,8 @@ deps: LET tsconfigs = $(git ls-files "**/tsconfig*.json" tsconfig*.json) FROM ../build-images+from-registry # copy bb, bb-js and noir-packages - COPY ../barretenberg/cpp/+preset-release/bin /usr/src/barretenberg/cpp/build/ + COPY ../barretenberg/cpp/+preset-release/bin /usr/src/barretenberg/cpp/build/bin + COPY ../barretenberg/cpp/+preset-release-world-state/bin /usr/src/barretenberg/cpp/build/bin COPY ../barretenberg/ts/+build/build /usr/src/barretenberg/ts COPY ../noir/+packages/packages /usr/src/noir/packages WORKDIR /usr/src/yarn-project @@ -41,6 +42,7 @@ build: FROM +deps COPY ../barretenberg/cpp/+preset-release/bin/bb /usr/src/barretenberg/cpp/build/bin/bb + COPY ../barretenberg/cpp/+preset-release-world-state/bin/world_state_napi.node /usr/src/barretenberg/cpp/build/bin/world_state_napi.node COPY ../noir/+nargo/acvm /usr/src/noir/noir-repo/target/release/acvm COPY --dir ../noir-projects/+build/. /usr/src/noir-projects COPY ../l1-contracts/+build/usr/src/l1-contracts /usr/src/l1-contracts @@ -156,7 +158,8 @@ aztec: ENV BB_BINARY_PATH=/usr/src/barretenberg/cpp/build/bin/bb ENV ACVM_WORKING_DIRECTORY=/usr/src/acvm ENV ACVM_BINARY_PATH=/usr/src/noir/noir-repo/target/release/acvm - RUN mkdir -p $BB_WORKING_DIRECTORY $ACVM_WORKING_DIRECTORY + RUN mkdir -p $BB_WORKING_DIRECTORY $ACVM_WORKING_DIRECTORY /usr/src/yarn-project/world-state/build + RUN cp /usr/src/barretenberg/cpp/build/bin/world_state_napi.node /usr/src/yarn-project/world-state/build ENTRYPOINT ["node", "--no-warnings", "/usr/src/yarn-project/aztec/dest/bin/index.js"] LET port=8080 ENV PORT=$port @@ -216,8 +219,7 @@ end-to-end-base: ENV ACVM_WORKING_DIRECTORY=/usr/src/acvm ENV ACVM_BINARY_PATH=/usr/src/noir/noir-repo/target/release/acvm ENV PROVER_AGENT_CONCURRENCY=8 - RUN mkdir -p $BB_WORKING_DIRECTORY $ACVM_WORKING_DIRECTORY - + RUN mkdir -p $BB_WORKING_DIRECTORY $ACVM_WORKING_DIRECTORY /usr/src/yarn-project/world-state/build RUN ln -s /usr/src/yarn-project/.yarn/releases/yarn-3.6.3.cjs /usr/local/bin/yarn end-to-end: @@ -225,6 +227,7 @@ end-to-end: COPY +anvil/anvil /opt/foundry/bin/anvil COPY +end-to-end-prod/usr/src /usr/src + RUN cp /usr/src/barretenberg/cpp/build/bin/world_state_napi.node /usr/src/yarn-project/world-state/build WORKDIR /usr/src/yarn-project/end-to-end ENTRYPOINT ["yarn", "test"] @@ -263,6 +266,9 @@ format-check: test: FROM +build + + RUN mkdir -p /usr/src/yarn-project/world-state/build + RUN cp /usr/src/barretenberg/cpp/build/bin/world_state_napi.node /usr/src/yarn-project/world-state/build RUN yarn test run-e2e: diff --git a/yarn-project/circuit-types/src/interfaces/merkle_tree_operations.ts b/yarn-project/circuit-types/src/interfaces/merkle_tree_operations.ts index af16730d969..3a72f58d731 100644 --- a/yarn-project/circuit-types/src/interfaces/merkle_tree_operations.ts +++ b/yarn-project/circuit-types/src/interfaces/merkle_tree_operations.ts @@ -11,6 +11,8 @@ import { type SiblingPath } from '../sibling_path/sibling_path.js'; */ export type IndexedTreeId = MerkleTreeId.NULLIFIER_TREE | MerkleTreeId.PUBLIC_DATA_TREE; +export type FrTreeId = Exclude; + /** * All of the data to be return during batch insertion. */ diff --git a/yarn-project/prover-client/src/mocks/test_context.ts b/yarn-project/prover-client/src/mocks/test_context.ts index 16e19745b0a..e0d6e170685 100644 --- a/yarn-project/prover-client/src/mocks/test_context.ts +++ b/yarn-project/prover-client/src/mocks/test_context.ts @@ -25,9 +25,12 @@ import { } from '@aztec/simulator'; import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; import { MerkleTrees } from '@aztec/world-state'; +import { NativeWorldStateService } from '@aztec/world-state/native'; import * as fs from 'fs/promises'; import { type MockProxy, mock } from 'jest-mock-extended'; +import { tmpdir } from 'os'; +import { join } from 'path'; import { TestCircuitProver } from '../../../bb-prover/src/test/test_circuit_prover.js'; import { ProvingOrchestrator } from '../orchestrator/orchestrator.js'; @@ -58,11 +61,13 @@ export class TestContext { static async new( logger: DebugLogger, + worldState: 'native' | 'legacy' = 'legacy', proverCount = 4, createProver: (bbConfig: BBProverConfig) => Promise = _ => Promise.resolve(new TestCircuitProver(new NoopTelemetryClient(), new WASMSimulator())), blockNumber = 3, ) { + const directoriesToCleanup: string[] = []; const globalVariables = makeGlobals(blockNumber); const publicExecutor = mock(); @@ -70,7 +75,19 @@ export class TestContext { const publicWorldStateDB = mock(); const publicKernel = new RealPublicKernelCircuitSimulator(new WASMSimulator()); const telemetry = new NoopTelemetryClient(); - const actualDb = await MerkleTrees.new(openTmpStore(), telemetry).then(t => t.asLatest()); + + let actualDb: MerkleTreeAdminOperations; + + if (worldState === 'native') { + const dir = await fs.mkdtemp(join(tmpdir(), 'prover-client-world-state-')); + directoriesToCleanup.push(dir); + const ws = await NativeWorldStateService.create(dir); + actualDb = ws.asLatest(); + } else { + const ws = await MerkleTrees.new(openTmpStore(), telemetry); + actualDb = ws.asLatest(); + } + const processor = new PublicProcessor( actualDb, publicExecutor, @@ -100,6 +117,10 @@ export class TestContext { localProver = await createProver(bbConfig); } + if (config?.directoryToCleanup) { + directoriesToCleanup.push(config.directoryToCleanup); + } + const queue = new MemoryProvingQueue(telemetry); const orchestrator = new ProvingOrchestrator(actualDb, queue, telemetry); const agent = new ProverAgent(localProver, proverCount); @@ -119,7 +140,7 @@ export class TestContext { agent, orchestrator, blockNumber, - [config?.directoryToCleanup ?? ''], + directoriesToCleanup, logger, ); } diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator.ts b/yarn-project/prover-client/src/orchestrator/orchestrator.ts index bb336b19735..b0ea8d36e60 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator.ts @@ -598,7 +598,7 @@ export class ProvingOrchestrator implements BlockProver { return; } - logger.error(`Error thrown when proving job`); + logger.error(`Error thrown when proving job`, err); provingState!.reject(`${err}`); } finally { const index = this.pendingProvingJobs.indexOf(controller); diff --git a/yarn-project/prover-client/src/test/bb_prover_base_rollup.test.ts b/yarn-project/prover-client/src/test/bb_prover_base_rollup.test.ts index 411960efede..8954177e5a5 100644 --- a/yarn-project/prover-client/src/test/bb_prover_base_rollup.test.ts +++ b/yarn-project/prover-client/src/test/bb_prover_base_rollup.test.ts @@ -23,7 +23,7 @@ describe('prover/bb_prover/base-rollup', () => { prover = await BBNativeRollupProver.new(bbConfig, new NoopTelemetryClient()); return prover; }; - context = await TestContext.new(logger, 1, buildProver); + context = await TestContext.new(logger, 'legacy', 1, buildProver); }); afterAll(async () => { diff --git a/yarn-project/prover-client/src/test/bb_prover_full_rollup.test.ts b/yarn-project/prover-client/src/test/bb_prover_full_rollup.test.ts index b5a7722a69e..8ffad1ac338 100644 --- a/yarn-project/prover-client/src/test/bb_prover_full_rollup.test.ts +++ b/yarn-project/prover-client/src/test/bb_prover_full_rollup.test.ts @@ -20,7 +20,7 @@ describe('prover/bb_prover/full-rollup', () => { return prover; }; logger = createDebugLogger('aztec:bb-prover-full-rollup'); - context = await TestContext.new(logger, 1, buildProver); + context = await TestContext.new(logger, 'legacy', 1, buildProver); }); afterAll(async () => { diff --git a/yarn-project/prover-client/src/test/bb_prover_parity.test.ts b/yarn-project/prover-client/src/test/bb_prover_parity.test.ts index 9272d74a78b..fbb69326c0f 100644 --- a/yarn-project/prover-client/src/test/bb_prover_parity.test.ts +++ b/yarn-project/prover-client/src/test/bb_prover_parity.test.ts @@ -32,7 +32,7 @@ describe('prover/bb_prover/parity', () => { bbProver = await BBNativeRollupProver.new(bbConfig, new NoopTelemetryClient()); return bbProver; }; - context = await TestContext.new(logger, 1, buildProver); + context = await TestContext.new(logger, 'legacy', 1, buildProver); }); afterAll(async () => { diff --git a/yarn-project/watch.sh b/yarn-project/watch.sh index d52f682f692..55fd43e58e6 100755 --- a/yarn-project/watch.sh +++ b/yarn-project/watch.sh @@ -6,6 +6,7 @@ INOTIFY_EVENTS="modify,create,delete,move" NOIR_CONTRACTS_OUT_DIR="../noir-projects/noir-contracts/target/" NOIR_CIRCUITS_OUT_DIR="../noir-projects/noir-protocol-circuits/target/" L1_CONTRACTS_OUT_DIR="../l1-contracts/out/" +BARRETENBERG_OUT_DIR="../barretenberg/cpp/build/" # Debounce any command sent here. Grouped by command name and first arg. debounce() { @@ -60,6 +61,11 @@ run_generate() { rm .generating.lock } +cp_barretenberg_artifacts() { + mkdir -p world-state/build + cp $BARRETENBERG_OUT_DIR/lib/world_state_napi.node world-state/build/world_state_napi.node +} + # Remove all temp files with process or run ids on exit cleanup() { rm -f .tsc.pid || true @@ -74,7 +80,7 @@ start_tsc_watch # Watch for changes in the output directories while true; do - folder=$(inotifywait --format '%w' --quiet --recursive --event $INOTIFY_EVENTS $NOIR_CONTRACTS_OUT_DIR $NOIR_CIRCUITS_OUT_DIR $L1_CONTRACTS_OUT_DIR) + folder=$(inotifywait --format '%w' --quiet --recursive --event $INOTIFY_EVENTS $NOIR_CONTRACTS_OUT_DIR $NOIR_CIRCUITS_OUT_DIR $L1_CONTRACTS_OUT_DIR $BARRETENBERG_OUT_DIR) case $folder in "$NOIR_CONTRACTS_OUT_DIR") debounce run_generate "noir-contracts" @@ -85,6 +91,9 @@ while true; do "$L1_CONTRACTS_OUT_DIR"*) debounce run_generate "l1-contracts" ;; + "$BARRETENBERG_OUT_DIR"*) + debounce cp_barretenberg_artifacts + ;; *) echo "Error: change at $folder not matched with any project" exit 1 diff --git a/yarn-project/world-state/package.json b/yarn-project/world-state/package.json index e731926eca0..8c7a1f793e7 100644 --- a/yarn-project/world-state/package.json +++ b/yarn-project/world-state/package.json @@ -2,7 +2,10 @@ "name": "@aztec/world-state", "version": "0.1.0", "type": "module", - "exports": "./dest/index.js", + "exports": { + ".": "./dest/index.js", + "./native": "./dest/native/index.js" + }, "typedocOptions": { "entryPoints": [ "./src/index.ts" @@ -12,6 +15,7 @@ }, "scripts": { "build": "yarn clean && tsc -b", + "build:cpp": "./scripts/build.sh cpp", "build:dev": "tsc -b --watch", "clean": "rm -rf ./dest .tsbuildinfo", "formatting": "run -T prettier --check ./src && run -T eslint ./src", @@ -60,10 +64,13 @@ "@aztec/merkle-tree": "workspace:^", "@aztec/telemetry-client": "workspace:^", "@aztec/types": "workspace:^", + "bindings": "^1.5.0", + "msgpackr": "^1.10.2", "tslib": "^2.4.0" }, "devDependencies": { "@jest/globals": "^29.5.0", + "@types/bindings": "^1.5.5", "@types/jest": "^29.5.0", "@types/levelup": "^5.1.2", "@types/memdown": "^3.0.0", diff --git a/yarn-project/world-state/package.local.json b/yarn-project/world-state/package.local.json new file mode 100644 index 00000000000..110d4ecded7 --- /dev/null +++ b/yarn-project/world-state/package.local.json @@ -0,0 +1,5 @@ +{ + "scripts": { + "build": "yarn clean && ./scripts/build.sh ts" + } +} diff --git a/yarn-project/world-state/scripts/build.sh b/yarn-project/world-state/scripts/build.sh new file mode 100755 index 00000000000..9ac0f1c305a --- /dev/null +++ b/yarn-project/world-state/scripts/build.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +set -e + +cd "$(dirname "$0")/.." + +# relatiev path from the directory containing package.json +WORLD_STATE_LIB_PATH=../../barretenberg/cpp/build/bin/world_state_napi.node + +build_addon() { + (cd ../../barretenberg/cpp; cmake --preset clang16-pic -DCMAKE_BUILD_TYPE=RelWithAssert; cmake --build --preset clang16-pic --target world_state_napi; echo $PWD; mkdir -p build/bin; cp ./build-pic/lib/world_state_napi.node ./build/bin/world_state_napi.node) +} + +cp_addon_lib() { + if [ -f $WORLD_STATE_LIB_PATH ]; then + echo "Copying world_state_napi.node to build directory" + rm -rf build + mkdir build + cp $WORLD_STATE_LIB_PATH build/world_state_napi.node + else + echo "world_state_napi.node not found at $WORLD_STATE_LIB_PATH" + echo "Skipping copy to build directory" + echo "NativeWorldStateService will not work without this file" + fi +} + +build_ts() { + tsc -b . +} + +case $1 in + cpp) + build_addon + cp_addon_lib + ;; + ts) + cp_addon_lib + build_ts + ;; + *) + echo "Usage: $0 {cpp|ts}" + exit 1 + ;; +esac diff --git a/yarn-project/world-state/src/native/index.ts b/yarn-project/world-state/src/native/index.ts new file mode 100644 index 00000000000..4cd3ce4f304 --- /dev/null +++ b/yarn-project/world-state/src/native/index.ts @@ -0,0 +1 @@ +export * from './native_world_state.js'; diff --git a/yarn-project/world-state/src/native/message.ts b/yarn-project/world-state/src/native/message.ts new file mode 100644 index 00000000000..2e073079d3b --- /dev/null +++ b/yarn-project/world-state/src/native/message.ts @@ -0,0 +1,247 @@ +import { MerkleTreeId } from '@aztec/circuit-types'; +import { AppendOnlyTreeSnapshot, Fr, INITIAL_L2_BLOCK_NUM, type StateReference, type UInt32 } from '@aztec/circuits.js'; +import { type Tuple } from '@aztec/foundation/serialize'; + +export type MessageHeaderInit = { + /** The message ID. Optional, if not set defaults to 0 */ + messageId?: number; + /** Identifies the original request. Optional */ + requestId?: number; +}; + +export class MessageHeader { + /** An number to identify this message */ + public readonly messageId: number; + /** If this message is a response to a request, the messageId of the request */ + public readonly requestId: number; + + constructor({ messageId, requestId }: MessageHeaderInit) { + this.messageId = messageId ?? 0; + this.requestId = requestId ?? 0; + } + + static fromMessagePack(data: object): MessageHeader { + return new MessageHeader(data as MessageHeaderInit); + } +} + +interface TypedMessageLike { + msgType: number; + header: { + messageId?: number; + requestId?: number; + }; + value: any; +} + +export class TypedMessage { + public constructor(public readonly msgType: T, public readonly header: MessageHeader, public readonly value: B) {} + + static fromMessagePack(data: TypedMessageLike): TypedMessage { + return new TypedMessage(data['msgType'] as T, MessageHeader.fromMessagePack(data['header']), data['value']); + } + + static isTypedMessageLike(obj: any): obj is TypedMessageLike { + return typeof obj === 'object' && obj !== null && 'msgType' in obj && 'header' in obj && 'value' in obj; + } +} + +export enum WorldStateMessageType { + GET_TREE_INFO = 100, + GET_STATE_REFERENCE, + + GET_LEAF_VALUE, + GET_LEAF_PREIMAGE, + GET_SIBLING_PATH, + + FIND_LEAF_INDEX, + FIND_LOW_LEAF, + + APPEND_LEAVES, + BATCH_INSERT, + + UPDATE_ARCHIVE, + + COMMIT, + ROLLBACK, + + SYNC_BLOCK, +} + +interface WithTreeId { + treeId: MerkleTreeId; +} + +interface WithWorldStateRevision { + revision: WorldStateRevision; +} + +interface WithLeafIndex { + leafIndex: bigint; +} + +export type SerializedLeafValue = + | Buffer // Fr + | { value: Buffer } // NullifierLeaf + | { value: Buffer; slot: Buffer }; // PublicDataTreeLeaf + +export type SerializedIndexedLeaf = { + value: Exclude; + nextIndex: bigint | number; + nextValue: Buffer; // Fr +}; + +interface WithLeafValue { + leaf: SerializedLeafValue; +} + +interface WithLeaves { + leaves: SerializedLeafValue[]; +} + +interface GetTreeInfoRequest extends WithTreeId, WithWorldStateRevision {} +interface GetTreeInfoResponse { + treeId: MerkleTreeId; + depth: UInt32; + size: bigint | number; + root: Buffer; +} + +interface GetSiblingPathRequest extends WithTreeId, WithLeafIndex, WithWorldStateRevision {} +type GetSiblingPathResponse = Buffer[]; + +interface GetStateReferenceRequest extends WithWorldStateRevision {} +interface GetStateReferenceResponse { + state: Record; +} + +interface GetLeafRequest extends WithTreeId, WithWorldStateRevision, WithLeafIndex {} +type GetLeafResponse = SerializedLeafValue | undefined; + +interface GetLeafPreImageRequest extends WithTreeId, WithLeafIndex, WithWorldStateRevision {} +type GetLeafPreImageResponse = SerializedIndexedLeaf | undefined; + +interface FindLeafIndexRequest extends WithTreeId, WithLeafValue, WithWorldStateRevision { + startIndex: bigint; +} +type FindLeafIndexResponse = bigint | null; + +interface FindLowLeafRequest extends WithTreeId, WithWorldStateRevision { + key: Fr; +} +interface FindLowLeafResponse { + index: bigint | number; + alreadyPresent: boolean; +} + +interface AppendLeavesRequest extends WithTreeId, WithLeaves {} + +interface BatchInsertRequest extends WithTreeId, WithLeaves { + subtreeDepth: number; +} +interface BatchInsertResponse { + low_leaf_witness_data: ReadonlyArray<{ + leaf: SerializedIndexedLeaf; + index: bigint | number; + path: Tuple; + }>; + sorted_leaves: ReadonlyArray<[SerializedLeafValue, UInt32]>; + subtree_path: Tuple; +} + +interface UpdateArchiveRequest { + blockStateRef: BlockStateReference; + blockHash: Buffer; +} + +interface SyncBlockRequest { + blockStateRef: BlockStateReference; + blockHash: Fr; + paddedNoteHashes: readonly SerializedLeafValue[]; + paddedL1ToL2Messages: readonly SerializedLeafValue[]; + paddedNullifiers: readonly SerializedLeafValue[]; + batchesOfPaddedPublicDataWrites: readonly SerializedLeafValue[][]; +} + +interface SyncBlockResponse { + isBlockOurs: boolean; +} + +export type WorldStateRequest = { + [WorldStateMessageType.GET_TREE_INFO]: GetTreeInfoRequest; + [WorldStateMessageType.GET_STATE_REFERENCE]: GetStateReferenceRequest; + + [WorldStateMessageType.GET_LEAF_VALUE]: GetLeafRequest; + [WorldStateMessageType.GET_LEAF_PREIMAGE]: GetLeafPreImageRequest; + [WorldStateMessageType.GET_SIBLING_PATH]: GetSiblingPathRequest; + + [WorldStateMessageType.FIND_LEAF_INDEX]: FindLeafIndexRequest; + [WorldStateMessageType.FIND_LOW_LEAF]: FindLowLeafRequest; + + [WorldStateMessageType.APPEND_LEAVES]: AppendLeavesRequest; + [WorldStateMessageType.BATCH_INSERT]: BatchInsertRequest; + + [WorldStateMessageType.UPDATE_ARCHIVE]: UpdateArchiveRequest; + + [WorldStateMessageType.COMMIT]: void; + [WorldStateMessageType.ROLLBACK]: void; + + [WorldStateMessageType.SYNC_BLOCK]: SyncBlockRequest; +}; + +export type WorldStateResponse = { + [WorldStateMessageType.GET_TREE_INFO]: GetTreeInfoResponse; + [WorldStateMessageType.GET_STATE_REFERENCE]: GetStateReferenceResponse; + + [WorldStateMessageType.GET_LEAF_VALUE]: GetLeafResponse; + [WorldStateMessageType.GET_LEAF_PREIMAGE]: GetLeafPreImageResponse; + [WorldStateMessageType.GET_SIBLING_PATH]: GetSiblingPathResponse; + + [WorldStateMessageType.FIND_LEAF_INDEX]: FindLeafIndexResponse; + [WorldStateMessageType.FIND_LOW_LEAF]: FindLowLeafResponse; + + [WorldStateMessageType.APPEND_LEAVES]: void; + [WorldStateMessageType.BATCH_INSERT]: BatchInsertResponse; + + [WorldStateMessageType.UPDATE_ARCHIVE]: void; + + [WorldStateMessageType.COMMIT]: void; + [WorldStateMessageType.ROLLBACK]: void; + + [WorldStateMessageType.SYNC_BLOCK]: SyncBlockResponse; +}; + +export type WorldStateRevision = -1 | 0 | UInt32; +export function worldStateRevision(includeUncommittedOrBlock: false | true | number): WorldStateRevision { + if (typeof includeUncommittedOrBlock === 'number') { + if (includeUncommittedOrBlock < INITIAL_L2_BLOCK_NUM || !Number.isInteger(includeUncommittedOrBlock)) { + throw new TypeError('Invalid block number: ' + includeUncommittedOrBlock); + } + + return includeUncommittedOrBlock; + } else if (includeUncommittedOrBlock) { + return -1; + } else { + return 0; + } +} + +type TreeStateReference = readonly [Buffer, number | bigint]; +type BlockStateReference = Map, TreeStateReference>; + +export function treeStateReferenceToSnapshot([root, size]: TreeStateReference): AppendOnlyTreeSnapshot { + return new AppendOnlyTreeSnapshot(Fr.fromBuffer(root), Number(size)); +} + +export function treeStateReference(snapshot: AppendOnlyTreeSnapshot) { + return [snapshot.root.toBuffer(), BigInt(snapshot.nextAvailableLeafIndex)] as const; +} + +export function blockStateReference(state: StateReference): BlockStateReference { + return new Map([ + [MerkleTreeId.NULLIFIER_TREE, treeStateReference(state.partial.nullifierTree)], + [MerkleTreeId.NOTE_HASH_TREE, treeStateReference(state.partial.noteHashTree)], + [MerkleTreeId.PUBLIC_DATA_TREE, treeStateReference(state.partial.publicDataTree)], + [MerkleTreeId.L1_TO_L2_MESSAGE_TREE, treeStateReference(state.l1ToL2MessageTree)], + ]); +} diff --git a/yarn-project/world-state/src/native/native_world_state.test.ts b/yarn-project/world-state/src/native/native_world_state.test.ts new file mode 100644 index 00000000000..6033a827336 --- /dev/null +++ b/yarn-project/world-state/src/native/native_world_state.test.ts @@ -0,0 +1,102 @@ +import { MerkleTreeId } from '@aztec/circuit-types'; +import { Fr } from '@aztec/circuits.js'; + +import { jest } from '@jest/globals'; +import { decode, encode } from 'msgpackr'; + +import { + MessageHeader, + TypedMessage, + WorldStateMessageType, + type WorldStateRequest, + type WorldStateResponse, + worldStateRevision, +} from './message.js'; +import { type NativeInstance, NativeWorldStateService } from './native_world_state.js'; + +describe('NativeWorldState', () => { + let call: jest.MockedFunction; + let worldState: WorldStateWithMockedInstance; + + beforeEach(() => { + call = jest.fn(); + worldState = new WorldStateWithMockedInstance({ call }); + }); + + afterEach(async () => { + await worldState.stop(); + }); + + it('encodes responses', async () => { + const expectedResponse: WorldStateResponse[WorldStateMessageType.GET_TREE_INFO] = { + treeId: MerkleTreeId.NULLIFIER_TREE, + depth: 20, + root: new Fr(42).toBuffer(), + size: 128n, + }; + + call.mockResolvedValueOnce( + encode( + new TypedMessage(WorldStateMessageType.GET_TREE_INFO, new MessageHeader({ requestId: 1 }), expectedResponse), + ), + ); + + const treeInfo = await worldState.getTreeInfo(MerkleTreeId.NULLIFIER_TREE, false); + expect(treeInfo).toEqual(expectedResponse); + }); + + it.each([true, false])('encodes messages with includeUncommitted=%s', async includeUncommitted => { + const expectedResponse: WorldStateResponse[WorldStateMessageType.GET_TREE_INFO] = { + treeId: MerkleTreeId.NULLIFIER_TREE, + depth: 20, + root: new Fr(42).toBuffer(), + size: 128n, + }; + + call.mockResolvedValueOnce( + encode( + new TypedMessage(WorldStateMessageType.GET_TREE_INFO, new MessageHeader({ requestId: 1 }), expectedResponse), + ), + ); + + await worldState.getTreeInfo(MerkleTreeId.NULLIFIER_TREE, includeUncommitted); + expect(call).toHaveBeenCalled(); + + const msg: TypedMessage< + WorldStateMessageType.GET_TREE_INFO, + WorldStateRequest[WorldStateMessageType.GET_TREE_INFO] + > = decode(call.mock.calls[0][0]); + + expect(msg.msgType).toBe(WorldStateMessageType.GET_TREE_INFO); + expect(msg.header.messageId).toBeGreaterThan(0); + expect(msg.value).toEqual({ + treeId: MerkleTreeId.NULLIFIER_TREE, + revision: worldStateRevision(includeUncommitted), + }); + }); + + it.each([ + [encode(undefined), 'Invalid response: expected TypedMessageLike, got undefined'], + [undefined, 'Invalid encoded response: expected Buffer or ArrayBuffer, got undefined'], + [encode(null), 'Invalid response: expected TypedMessageLike, got null'], + [null, 'Invalid encoded response: expected Buffer or ArrayBuffer, got null'], + [encode({}), 'Invalid response: expected TypedMessageLike, got object'], + [ + encode(new TypedMessage(WorldStateMessageType.GET_TREE_INFO, new MessageHeader({ requestId: 100 }), {})), + 'Response ID does not match request', + ], + [ + encode(new TypedMessage(WorldStateMessageType.COMMIT, new MessageHeader({ requestId: 1 }), {})), + 'Invalid response message type', + ], + ])('rejects invalid responses', async (resp, expectedError) => { + call.mockResolvedValueOnce(resp); + await expect(worldState.getTreeInfo(MerkleTreeId.NULLIFIER_TREE, false)).rejects.toThrow(expectedError); + }); +}); + +class WorldStateWithMockedInstance extends NativeWorldStateService { + constructor(instance: NativeInstance) { + super(instance); + } +} diff --git a/yarn-project/world-state/src/native/native_world_state.ts b/yarn-project/world-state/src/native/native_world_state.ts new file mode 100644 index 00000000000..482bbc341e6 --- /dev/null +++ b/yarn-project/world-state/src/native/native_world_state.ts @@ -0,0 +1,462 @@ +import { + type BatchInsertionResult, + type HandleL2BlockAndMessagesResult, + type IndexedTreeId, + type L2Block, + type MerkleTreeAdminOperations, + MerkleTreeId, + type MerkleTreeLeafType, + SiblingPath, + type TreeInfo, + TxEffect, +} from '@aztec/circuit-types'; +import { + Fr, + Header, + MAX_NOTE_HASHES_PER_TX, + MAX_NULLIFIERS_PER_TX, + MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, + NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, + NullifierLeaf, + NullifierLeafPreimage, + PartialStateReference, + PublicDataTreeLeaf, + PublicDataTreeLeafPreimage, + StateReference, +} from '@aztec/circuits.js'; +import { padArrayEnd } from '@aztec/foundation/collection'; +import { SerialQueue } from '@aztec/foundation/queue'; +import { serializeToBuffer } from '@aztec/foundation/serialize'; +import type { IndexedTreeLeafPreimage } from '@aztec/foundation/trees'; + +import bindings from 'bindings'; +import { Decoder, Encoder, addExtension } from 'msgpackr'; +import { isAnyArrayBuffer } from 'util/types'; + +import { type MerkleTreeDb, type TreeSnapshots } from '../world-state-db/merkle_tree_db.js'; +import { MerkleTreeAdminOperationsFacade } from '../world-state-db/merkle_tree_operations_facade.js'; +import { + MessageHeader, + type SerializedIndexedLeaf, + type SerializedLeafValue, + TypedMessage, + WorldStateMessageType, + type WorldStateRequest, + type WorldStateResponse, + blockStateReference, + treeStateReferenceToSnapshot, + worldStateRevision, +} from './message.js'; + +// small extension to pack an NodeJS Fr instance to a representation that the C++ code can understand +// this only works for writes. Unpacking from C++ can't create Fr instances because the data is passed +// as raw, untagged, buffers. On the NodeJS side we don't know what the buffer represents +// Adding a tag would be a solution, but it would have to be done on both sides and it's unclear where else +// C++ fr instances are sent/received/stored. +addExtension({ + Class: Fr, + write: fr => fr.toBuffer(), +}); + +export interface NativeInstance { + call(msg: Buffer | Uint8Array): Promise; +} + +export class NativeWorldStateService implements MerkleTreeDb { + private nextMessageId = 1; + + private encoder = new Encoder({ + // always encode JS objects as MessagePack maps + // this makes it compatible with other MessagePack decoders + useRecords: false, + int64AsType: 'bigint', + }); + + private decoder = new Decoder({ + useRecords: false, + int64AsType: 'bigint', + }); + + private queue = new SerialQueue(); + + protected constructor(private instance: NativeInstance) { + this.queue.start(); + } + + static async create( + dataDir: string, + libraryName = 'world_state_napi', + className = 'WorldState', + ): Promise { + const library = bindings(libraryName); + const instance = new library[className](dataDir); + const worldState = new NativeWorldStateService(instance); + await worldState.init(); + return worldState; + } + + protected async init() { + const archive = await this.getTreeInfo(MerkleTreeId.ARCHIVE, false); + if (archive.size === 0n) { + // TODO (alexg) move this to the native module + // const header = await this.buildInitialHeader(true); + // await this.appendLeaves(MerkleTreeId.ARCHIVE, [header.hash()]); + // await this.commit(); + } + } + + public asLatest(): MerkleTreeAdminOperations { + return new MerkleTreeAdminOperationsFacade(this, true); + } + + // async buildInitialHeader(ic: boolean = false): Promise
{ + // const state = await this.getStateReference(ic); + // return new Header( + // AppendOnlyTreeSnapshot.zero(), + // ContentCommitment.empty(), + // state, + // GlobalVariables.empty(), + // Fr.ZERO, + // ); + // } + + public getInitialHeader(): Header { + // TODO (alexg) implement this + return Header.empty(); + } + + async appendLeaves(treeId: ID, leaves: MerkleTreeLeafType[]): Promise { + await this.call(WorldStateMessageType.APPEND_LEAVES, { + leaves: leaves.map(leaf => leaf as any), + treeId, + }); + } + + async batchInsert( + treeId: ID, + rawLeaves: Buffer[], + subtreeHeight: number, + ): Promise> { + const leaves = rawLeaves.map((leaf: Buffer) => hydrateLeaf(treeId, leaf)).map(serializeLeaf); + const resp = await this.call(WorldStateMessageType.BATCH_INSERT, { leaves, treeId, subtreeDepth: subtreeHeight }); + + return { + newSubtreeSiblingPath: new SiblingPath( + resp.subtree_path.length as any, + resp.subtree_path, + ), + sortedNewLeaves: resp.sorted_leaves + .map(([leaf]) => leaf) + .map(deserializeLeafValue) + .map(serializeToBuffer), + sortedNewLeavesIndexes: resp.sorted_leaves.map(([, index]) => index), + lowLeavesWitnessData: resp.low_leaf_witness_data.map(data => ({ + index: BigInt(data.index), + leafPreimage: deserializeIndexedLeaf(data.leaf), + siblingPath: new SiblingPath(data.path.length as any, data.path), + })), + }; + } + + async commit(): Promise { + await this.call(WorldStateMessageType.COMMIT, void 0); + } + + findLeafIndex( + treeId: MerkleTreeId, + value: MerkleTreeLeafType, + includeUncommitted: boolean, + ): Promise { + return this.findLeafIndexAfter(treeId, value, 0n, includeUncommitted); + } + + async findLeafIndexAfter( + treeId: MerkleTreeId, + leaf: MerkleTreeLeafType, + startIndex: bigint, + includeUncommitted: boolean, + ): Promise { + const index = await this.call(WorldStateMessageType.FIND_LEAF_INDEX, { + leaf: serializeLeaf(hydrateLeaf(treeId, leaf)), + revision: worldStateRevision(includeUncommitted), + treeId, + startIndex, + }); + + if (typeof index === 'number' || typeof index === 'bigint') { + return BigInt(index); + } else { + return undefined; + } + } + + async getLeafPreimage( + treeId: IndexedTreeId, + leafIndex: bigint, + args: boolean, + ): Promise { + const resp = await this.call(WorldStateMessageType.GET_LEAF_PREIMAGE, { + leafIndex, + revision: worldStateRevision(args), + treeId, + }); + + return resp ? deserializeIndexedLeaf(resp) : undefined; + } + + async getLeafValue( + treeId: MerkleTreeId, + leafIndex: bigint, + includeUncommitted: boolean, + ): Promise | undefined> { + const resp = await this.call(WorldStateMessageType.GET_LEAF_VALUE, { + leafIndex, + revision: worldStateRevision(includeUncommitted), + treeId, + }); + + if (!resp) { + return undefined; + } + + const leaf = deserializeLeafValue(resp); + if (leaf instanceof Fr) { + return leaf; + } else { + return leaf.toBuffer(); + } + } + + async getPreviousValueIndex( + treeId: IndexedTreeId, + value: bigint, + includeUncommitted: boolean, + ): Promise<{ index: bigint; alreadyPresent: boolean } | undefined> { + const resp = await this.call(WorldStateMessageType.FIND_LOW_LEAF, { + key: new Fr(value), + revision: worldStateRevision(includeUncommitted), + treeId, + }); + return { + alreadyPresent: resp.alreadyPresent, + index: BigInt(resp.index), + }; + } + + async getSiblingPath( + treeId: MerkleTreeId, + leafIndex: bigint, + includeUncommitted: boolean, + ): Promise> { + const siblingPath = await this.call(WorldStateMessageType.GET_SIBLING_PATH, { + leafIndex, + revision: worldStateRevision(includeUncommitted), + treeId, + }); + + return new SiblingPath(siblingPath.length, siblingPath); + } + + getSnapshot(_block: number): Promise { + return Promise.reject(new Error('getSnapshot not implemented')); + } + + async getStateReference(includeUncommitted: boolean): Promise { + const resp = await this.call(WorldStateMessageType.GET_STATE_REFERENCE, { + revision: worldStateRevision(includeUncommitted), + }); + + return new StateReference( + treeStateReferenceToSnapshot(resp.state[MerkleTreeId.L1_TO_L2_MESSAGE_TREE]), + new PartialStateReference( + treeStateReferenceToSnapshot(resp.state[MerkleTreeId.NOTE_HASH_TREE]), + treeStateReferenceToSnapshot(resp.state[MerkleTreeId.NULLIFIER_TREE]), + treeStateReferenceToSnapshot(resp.state[MerkleTreeId.PUBLIC_DATA_TREE]), + ), + ); + } + + async getTreeInfo(treeId: MerkleTreeId, includeUncommitted: boolean): Promise { + const resp = await this.call(WorldStateMessageType.GET_TREE_INFO, { + treeId: treeId, + revision: worldStateRevision(includeUncommitted), + }); + + return { + depth: resp.depth, + root: resp.root, + size: BigInt(resp.size), + treeId, + }; + } + + async handleL2BlockAndMessages(l2Block: L2Block, l1ToL2Messages: Fr[]): Promise { + // We have to pad both the tx effects and the values within tx effects because that's how the trees are built + // by circuits. + const paddedTxEffects = padArrayEnd( + l2Block.body.txEffects, + TxEffect.empty(), + l2Block.body.numberOfTxsIncludingPadded, + ); + + const paddedNoteHashes = paddedTxEffects.flatMap(txEffect => + padArrayEnd(txEffect.noteHashes, Fr.ZERO, MAX_NOTE_HASHES_PER_TX), + ); + const paddedL1ToL2Messages = padArrayEnd(l1ToL2Messages, Fr.ZERO, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP); + + const paddedNullifiers = paddedTxEffects + .flatMap(txEffect => padArrayEnd(txEffect.nullifiers, Fr.ZERO, MAX_NULLIFIERS_PER_TX)) + .map(nullifier => new NullifierLeaf(nullifier)); + + // We insert the public data tree leaves with one batch per tx to avoid updating the same key twice + const batchesOfPaddedPublicDataWrites: PublicDataTreeLeaf[][] = []; + for (const txEffect of paddedTxEffects) { + const batch: PublicDataTreeLeaf[] = Array(MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX).fill( + PublicDataTreeLeaf.empty(), + ); + for (const [i, write] of txEffect.publicDataWrites.entries()) { + batch[i] = new PublicDataTreeLeaf(write.leafIndex, write.newValue); + } + + batchesOfPaddedPublicDataWrites.push(batch); + } + + return await this.call(WorldStateMessageType.SYNC_BLOCK, { + blockHash: l2Block.hash(), + paddedL1ToL2Messages: paddedL1ToL2Messages.map(serializeLeaf), + paddedNoteHashes: paddedNoteHashes.map(serializeLeaf), + paddedNullifiers: paddedNullifiers.map(serializeLeaf), + batchesOfPaddedPublicDataWrites: batchesOfPaddedPublicDataWrites.map(batch => batch.map(serializeLeaf)), + blockStateRef: blockStateReference(l2Block.header.state), + }); + } + + async rollback(): Promise { + await this.call(WorldStateMessageType.ROLLBACK, void 0); + } + + async updateArchive(header: Header, _includeUncommitted: boolean): Promise { + await this.call(WorldStateMessageType.UPDATE_ARCHIVE, { + blockHash: header.hash().toBuffer(), + blockStateRef: blockStateReference(header.state), + }); + } + + updateLeaf( + _treeId: ID, + _leaf: NullifierLeafPreimage | Buffer, + _index: bigint, + ): Promise { + return Promise.reject(new Error('Method not implemented')); + } + + private call( + messageType: T, + body: WorldStateRequest[T], + ): Promise { + return this.queue.put(async () => { + const request = new TypedMessage(messageType, new MessageHeader({ messageId: this.nextMessageId++ }), body); + + const encodedRequest = this.encoder.encode(request); + const encodedResponse = await this.instance.call(encodedRequest); + + const buf = Buffer.isBuffer(encodedResponse) + ? encodedResponse + : isAnyArrayBuffer(encodedResponse) + ? Buffer.from(encodedResponse) + : encodedResponse; + + if (!Buffer.isBuffer(buf)) { + throw new TypeError( + 'Invalid encoded response: expected Buffer or ArrayBuffer, got ' + + (encodedResponse === null ? 'null' : typeof encodedResponse), + ); + } + + const decodedResponse = this.decoder.unpack(buf); + if (!TypedMessage.isTypedMessageLike(decodedResponse)) { + throw new TypeError( + 'Invalid response: expected TypedMessageLike, got ' + + (decodedResponse === null ? 'null' : typeof decodedResponse), + ); + } + + const response = TypedMessage.fromMessagePack(decodedResponse); + + if (response.header.requestId !== request.header.messageId) { + throw new Error( + 'Response ID does not match request: ' + response.header.requestId + ' != ' + request.header.messageId, + ); + } + + if (response.msgType !== messageType) { + throw new Error('Invalid response message type: ' + response.msgType + ' != ' + messageType); + } + + return response.value; + }); + } + + public async stop(): Promise { + await this.queue.end(); + } + + public delete(): Promise { + return Promise.reject(new Error('Method not implemented')); + } + + public fork(): Promise { + return Promise.reject(new Error('Method not implemented')); + } +} + +function hydrateLeaf(treeId: ID, leaf: Fr | Buffer) { + if (leaf instanceof Fr) { + return leaf; + } else if (treeId === MerkleTreeId.NULLIFIER_TREE) { + return NullifierLeaf.fromBuffer(leaf); + } else if (treeId === MerkleTreeId.PUBLIC_DATA_TREE) { + return PublicDataTreeLeaf.fromBuffer(leaf); + } else { + return Fr.fromBuffer(leaf); + } +} + +function serializeLeaf(leaf: Fr | NullifierLeaf | PublicDataTreeLeaf): SerializedLeafValue { + if (leaf instanceof Fr) { + return leaf.toBuffer(); + } else if (leaf instanceof NullifierLeaf) { + return { value: leaf.nullifier.toBuffer() }; + } else { + return { value: leaf.value.toBuffer(), slot: leaf.slot.toBuffer() }; + } +} + +function deserializeLeafValue(leaf: SerializedLeafValue): Fr | NullifierLeaf | PublicDataTreeLeaf { + if (Buffer.isBuffer(leaf)) { + return Fr.fromBuffer(leaf); + } else if ('slot' in leaf) { + return new PublicDataTreeLeaf(Fr.fromBuffer(leaf.slot), Fr.fromBuffer(leaf.value)); + } else { + return new NullifierLeaf(Fr.fromBuffer(leaf.value)); + } +} + +function deserializeIndexedLeaf(leaf: SerializedIndexedLeaf): IndexedTreeLeafPreimage { + if ('slot' in leaf.value) { + return new PublicDataTreeLeafPreimage( + Fr.fromBuffer(leaf.value.slot), + Fr.fromBuffer(leaf.value.value), + Fr.fromBuffer(leaf.nextValue), + BigInt(leaf.nextIndex), + ); + } else if ('value' in leaf.value) { + return new NullifierLeafPreimage( + Fr.fromBuffer(leaf.value.value), + Fr.fromBuffer(leaf.nextValue), + BigInt(leaf.nextIndex), + ); + } else { + throw new Error('Invalid leaf type'); + } +} diff --git a/yarn-project/world-state/src/native/native_world_state_integration.test.ts b/yarn-project/world-state/src/native/native_world_state_integration.test.ts new file mode 100644 index 00000000000..4342a6c3e0f --- /dev/null +++ b/yarn-project/world-state/src/native/native_world_state_integration.test.ts @@ -0,0 +1,295 @@ +import { + type FrTreeId, + type IndexedTreeId, + L2Block, + MerkleTreeId, + PublicDataWrite, + TxEffect, +} from '@aztec/circuit-types'; +import { + AppendOnlyTreeSnapshot, + Fr, + MAX_NOTE_HASHES_PER_TX, + MAX_NULLIFIERS_PER_TX, + MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, + NULLIFIER_SUBTREE_HEIGHT, + NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, + NullifierLeaf, + PUBLIC_DATA_SUBTREE_HEIGHT, + PublicDataTreeLeaf, +} from '@aztec/circuits.js'; +import { padArrayEnd } from '@aztec/foundation/collection'; +import { elapsed } from '@aztec/foundation/timer'; +import { AztecLmdbStore } from '@aztec/kv-store/lmdb'; +import { openTmpStore } from '@aztec/kv-store/utils'; +import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; + +import { mkdtemp, rm } from 'fs/promises'; +import { tmpdir } from 'os'; +import { join } from 'path'; + +import { type MerkleTreeDb } from '../world-state-db/merkle_tree_db.js'; +import { MerkleTrees } from '../world-state-db/merkle_trees.js'; +import { NativeWorldStateService } from './native_world_state.js'; + +describe('NativeWorldState', () => { + let nativeDataDir: string; + let legacyDataDir: string; + + let nativeWS: NativeWorldStateService; + let legacyWS: MerkleTrees; + + const allTrees = Object.values(MerkleTreeId) + .filter((x): x is MerkleTreeId => typeof x === 'number') + .map(x => [MerkleTreeId[x], x] as const); + + beforeAll(async () => { + nativeDataDir = await mkdtemp(join(tmpdir(), 'native_world_state_test-')); + legacyDataDir = await mkdtemp(join(tmpdir(), 'js_world_state_test-')); + }); + + afterAll(async () => { + await rm(nativeDataDir, { recursive: true }); + await rm(legacyDataDir, { recursive: true }); + }); + + beforeAll(async () => { + nativeWS = await NativeWorldStateService.create(nativeDataDir); + legacyWS = await MerkleTrees.new(AztecLmdbStore.open(legacyDataDir), new NoopTelemetryClient()); + }); + + describe('Initial state', () => { + it.each( + allTrees + .filter(x => x[1] != MerkleTreeId.ARCHIVE) + .flatMap(t => [ + [...t, false], + [...t, true], + ]), + )('tree %s is the same', async (_, treeId, includeUncommitted) => { + await assertSameTree(treeId, includeUncommitted); + }); + + it.each([false, true])('includeUncommitted=%s state is the same', async includeUncommitted => { + await assertSameState(includeUncommitted); + }); + }); + + describe('Uncommitted state', () => { + afterEach(async () => { + await Promise.all([nativeWS.rollback(), legacyWS.rollback()]); + }); + + it.each<[string, IndexedTreeId, Buffer[]]>([ + [ + MerkleTreeId[MerkleTreeId.NULLIFIER_TREE], + MerkleTreeId.NULLIFIER_TREE, + Array(64).fill(new NullifierLeaf(Fr.ZERO).toBuffer()), + ], + [ + MerkleTreeId[MerkleTreeId.PUBLIC_DATA_TREE], + MerkleTreeId.PUBLIC_DATA_TREE, + Array(64).fill(new PublicDataTreeLeaf(Fr.ZERO, Fr.ZERO).toBuffer()), + ], + ])('inserting 0 leaves into %s', async (_, treeId, leaves) => { + const height = Math.ceil(Math.log2(leaves.length) | 0); + const [native, js] = await Promise.all([ + nativeWS.batchInsert(treeId, leaves, height), + legacyWS.batchInsert(treeId, leaves, height), + ]); + + expect(native.sortedNewLeaves.map(Fr.fromBuffer)).toEqual(js.sortedNewLeaves.map(Fr.fromBuffer)); + expect(native.sortedNewLeavesIndexes).toEqual(js.sortedNewLeavesIndexes); + expect(native.newSubtreeSiblingPath.toFields()).toEqual(js.newSubtreeSiblingPath.toFields()); + expect(native.lowLeavesWitnessData).toEqual(js.lowLeavesWitnessData); + + await assertSameTree(treeId, true); + }); + + it.each<[string, FrTreeId, Fr[]]>([ + [MerkleTreeId[MerkleTreeId.NOTE_HASH_TREE], MerkleTreeId.NOTE_HASH_TREE, Array(64).fill(Fr.ZERO)], + [MerkleTreeId[MerkleTreeId.L1_TO_L2_MESSAGE_TREE], MerkleTreeId.NOTE_HASH_TREE, Array(64).fill(Fr.ZERO)], + ])('inserting 0 leaves into %s', async (_, treeId, leaves) => { + await Promise.all([nativeWS.appendLeaves(treeId, leaves), legacyWS.appendLeaves(treeId, leaves)]); + + await assertSameTree(treeId, true); + }); + + it.each<[string, IndexedTreeId, Buffer[]]>([ + [ + MerkleTreeId[MerkleTreeId.NULLIFIER_TREE], + MerkleTreeId.NULLIFIER_TREE, + Array(64) + .fill(0) + .map(() => new NullifierLeaf(Fr.random()).toBuffer()), + ], + [ + MerkleTreeId[MerkleTreeId.PUBLIC_DATA_TREE], + MerkleTreeId.PUBLIC_DATA_TREE, + Array(64) + .fill(0) + .map(() => new PublicDataTreeLeaf(Fr.random(), Fr.random()).toBuffer()), + ], + ])('inserting real leaves into %s', async (_, treeId, leaves) => { + const height = Math.ceil(Math.log2(leaves.length) | 0); + const [native, js] = await Promise.all([ + nativeWS.batchInsert(treeId, leaves, height), + legacyWS.batchInsert(treeId, leaves, height), + ]); + + expect(native.sortedNewLeaves.map(Fr.fromBuffer)).toEqual(js.sortedNewLeaves.map(Fr.fromBuffer)); + expect(native.sortedNewLeavesIndexes).toEqual(js.sortedNewLeavesIndexes); + expect(native.newSubtreeSiblingPath.toFields()).toEqual(js.newSubtreeSiblingPath.toFields()); + expect(native.lowLeavesWitnessData).toEqual(js.lowLeavesWitnessData); + + await assertSameTree(treeId, true); + }); + + it.each<[string, FrTreeId, Fr[]]>([ + [MerkleTreeId[MerkleTreeId.NOTE_HASH_TREE], MerkleTreeId.NOTE_HASH_TREE, Array(64).fill(0).map(Fr.random)], + [MerkleTreeId[MerkleTreeId.L1_TO_L2_MESSAGE_TREE], MerkleTreeId.NOTE_HASH_TREE, Array(64).fill(0).map(Fr.random)], + ])('inserting real leaves into %s', async (_, treeId, leaves) => { + await Promise.all([nativeWS.appendLeaves(treeId, leaves), legacyWS.appendLeaves(treeId, leaves)]); + + await assertSameTree(treeId, true); + }); + + it.each<[string, MerkleTreeId, number[]]>([ + [MerkleTreeId[MerkleTreeId.NULLIFIER_TREE], MerkleTreeId.NULLIFIER_TREE, [0, 1, 10, 127, 128, 256]], + [MerkleTreeId[MerkleTreeId.NOTE_HASH_TREE], MerkleTreeId.NOTE_HASH_TREE, [0, 1, 10, 127, 128, 256]], + [MerkleTreeId[MerkleTreeId.PUBLIC_DATA_TREE], MerkleTreeId.PUBLIC_DATA_TREE, [0, 1, 10, 127, 128, 256]], + [MerkleTreeId[MerkleTreeId.L1_TO_L2_MESSAGE_TREE], MerkleTreeId.NOTE_HASH_TREE, [0, 1, 10, 127, 128, 256]], + ])('sibling paths to initial leaves match', async (_, treeId, leafIndices) => { + for (const index of leafIndices) { + const [nativeSB, legacySB] = await Promise.all([ + nativeWS.getSiblingPath(treeId, BigInt(index), true), + legacyWS.getSiblingPath(treeId, BigInt(index), true), + ]); + expect(nativeSB.toFields()).toEqual(legacySB.toFields()); + } + }); + + it.each<[string, IndexedTreeId, Buffer, (b: Buffer) => bigint]>([ + [ + MerkleTreeId[MerkleTreeId.NULLIFIER_TREE], + MerkleTreeId.NULLIFIER_TREE, + new NullifierLeaf(Fr.random()).toBuffer(), + b => NullifierLeaf.fromBuffer(b).getKey(), + ], + [ + MerkleTreeId[MerkleTreeId.PUBLIC_DATA_TREE], + MerkleTreeId.PUBLIC_DATA_TREE, + new PublicDataTreeLeaf(Fr.random(), Fr.random()).toBuffer(), + b => PublicDataTreeLeaf.fromBuffer(b).getKey(), + ], + ])('inserting real leaves into %s', async (_, treeId, leaf, getKey) => { + await Promise.all([nativeWS.batchInsert(treeId, [leaf], 0), legacyWS.batchInsert(treeId, [leaf], 0)]); + + const [nativeLeafIndex, legacyLeafIndex] = await Promise.all([ + nativeWS.getPreviousValueIndex(treeId, getKey(leaf), true), + legacyWS.getPreviousValueIndex(treeId, getKey(leaf), true), + ]); + expect(nativeLeafIndex).toEqual(legacyLeafIndex); + + const [nativeSB, legacySB] = await Promise.all([ + nativeWS.getSiblingPath(treeId, nativeLeafIndex!.index, true), + legacyWS.getSiblingPath(treeId, legacyLeafIndex!.index, true), + ]); + + expect(nativeSB.toFields()).toEqual(legacySB.toFields()); + }); + }); + + describe('Block synch', () => { + it('syncs a new block from empty state', async () => { + await assertSameState(false); + const [_blockMS, { block, messages }] = await elapsed(mockBlock(1)); + + const [_nativeMs] = await elapsed(nativeWS.handleL2BlockAndMessages(block, messages)); + const [_legacyMs] = await elapsed(legacyWS.handleL2BlockAndMessages(block, messages)); + + // console.log(`Native: ${nativeMs} ms, Legacy: ${legacyMs} ms. Generating mock block took ${blockMS} ms`); + + await assertSameState(false); + }, 15_000); + }); + + async function assertSameTree(treeId: MerkleTreeId, includeUncommitted = false) { + const nativeInfo = await nativeWS.getTreeInfo(treeId, includeUncommitted); + const jsInfo = await legacyWS.getTreeInfo(treeId, includeUncommitted); + expect(nativeInfo.treeId).toBe(jsInfo.treeId); + expect(nativeInfo.depth).toBe(jsInfo.depth); + expect(nativeInfo.size).toBe(jsInfo.size); + expect(Fr.fromBuffer(nativeInfo.root)).toEqual(Fr.fromBuffer(jsInfo.root)); + } + + async function assertSameState(includeUncommitted = false) { + const nativeStateRef = await nativeWS.getStateReference(includeUncommitted); + const legacyStateRef = await legacyWS.getStateReference(includeUncommitted); + expect(nativeStateRef).toEqual(legacyStateRef); + } + + async function mockBlock(blockNum = 1, merkleTrees?: MerkleTreeDb) { + merkleTrees ??= await MerkleTrees.new(openTmpStore(), new NoopTelemetryClient()); + const l2Block = L2Block.random(blockNum, 2); // 2 txs + const l1ToL2Messages = Array(16).fill(0).map(Fr.random); + + const paddedTxEffects = padArrayEnd( + l2Block.body.txEffects, + TxEffect.empty(), + l2Block.body.numberOfTxsIncludingPadded, + ); + + // Sync the append only trees + { + const noteHashesPadded = paddedTxEffects.flatMap(txEffect => + padArrayEnd(txEffect.noteHashes, Fr.ZERO, MAX_NOTE_HASHES_PER_TX), + ); + await merkleTrees.appendLeaves(MerkleTreeId.NOTE_HASH_TREE, noteHashesPadded); + + const l1ToL2MessagesPadded = padArrayEnd(l1ToL2Messages, Fr.ZERO, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP); + await merkleTrees.appendLeaves(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, l1ToL2MessagesPadded); + } + + // Sync the indexed trees + { + const nullifiersPadded = paddedTxEffects.flatMap(txEffect => + padArrayEnd(txEffect.nullifiers, Fr.ZERO, MAX_NULLIFIERS_PER_TX), + ); + await merkleTrees.batchInsert( + MerkleTreeId.NULLIFIER_TREE, + nullifiersPadded.map(nullifier => nullifier.toBuffer()), + NULLIFIER_SUBTREE_HEIGHT, + ); + + // We insert the public data tree leaves with one batch per tx to avoid updating the same key twice + for (const txEffect of paddedTxEffects) { + const publicDataWrites = padArrayEnd( + txEffect.publicDataWrites, + PublicDataWrite.empty(), + MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, + ); + + await merkleTrees.batchInsert( + MerkleTreeId.PUBLIC_DATA_TREE, + publicDataWrites.map(write => new PublicDataTreeLeaf(write.leafIndex, write.newValue).toBuffer()), + PUBLIC_DATA_SUBTREE_HEIGHT, + ); + } + } + + // await merkleTrees.appendLeaves(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, l1ToL2Messages); + const state = await merkleTrees.getStateReference(true); + l2Block.header.state = state; + await merkleTrees.updateArchive(l2Block.header, true); + + const archiveState = await merkleTrees.getTreeInfo(MerkleTreeId.ARCHIVE, true); + + l2Block.archive = new AppendOnlyTreeSnapshot(Fr.fromBuffer(archiveState.root), Number(archiveState.size)); + + return { + block: l2Block, + messages: l1ToL2Messages, + }; + } +}); diff --git a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts index ce188d1eaa1..c0235d8fbf5 100644 --- a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts +++ b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts @@ -17,12 +17,12 @@ import { type AztecKVStore, type AztecSingleton } from '@aztec/kv-store'; import { openTmpStore } from '@aztec/kv-store/utils'; import { SHA256Trunc, StandardTree } from '@aztec/merkle-tree'; -import { type MerkleTrees } from '../world-state-db/index.js'; import { MerkleTreeAdminOperationsFacade, MerkleTreeOperationsFacade, } from '../world-state-db/merkle_tree_operations_facade.js'; import { MerkleTreeSnapshotOperationsFacade } from '../world-state-db/merkle_tree_snapshot_operations_facade.js'; +import { type MerkleTrees } from '../world-state-db/merkle_trees.js'; import { type WorldStateConfig } from './config.js'; import { WorldStateRunningState, diff --git a/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts b/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts index bf4d97e3924..3fa9dab9591 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts @@ -77,4 +77,7 @@ export type MerkleTreeAdminDb = { /** Deletes this database. */ delete(): Promise; + + /** Stops the database */ + stop(): Promise; }; diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index f1a9a1fb7f0..beac450610c 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -1247,13 +1247,16 @@ __metadata: "@aztec/telemetry-client": "workspace:^" "@aztec/types": "workspace:^" "@jest/globals": ^29.5.0 + "@types/bindings": ^1.5.5 "@types/jest": ^29.5.0 "@types/levelup": ^5.1.2 "@types/memdown": ^3.0.0 "@types/node": ^18.7.23 + bindings: ^1.5.0 jest: ^29.5.0 jest-mock-extended: ^3.0.5 memdown: ^6.1.1 + msgpackr: ^1.10.2 ts-node: ^10.9.1 tslib: ^2.4.0 typescript: ^5.0.4 @@ -4099,6 +4102,15 @@ __metadata: languageName: node linkType: hard +"@types/bindings@npm:^1.5.5": + version: 1.5.5 + resolution: "@types/bindings@npm:1.5.5" + dependencies: + "@types/node": "*" + checksum: 0205bd031677e9af479437b4f358d4ea96d9d1a7d57132657915201d9a8be968d5f8341f526cc79481142fe9456f82e093cf62a0b2bcf6c3cfd4577962603b22 + languageName: node + linkType: hard + "@types/bn.js@npm:*, @types/bn.js@npm:^5.1.3": version: 5.1.5 resolution: "@types/bn.js@npm:5.1.5" @@ -5943,7 +5955,7 @@ __metadata: languageName: node linkType: hard -"bindings@npm:^1.3.0": +"bindings@npm:^1.3.0, bindings@npm:^1.5.0": version: 1.5.0 resolution: "bindings@npm:1.5.0" dependencies: @@ -12385,6 +12397,18 @@ __metadata: languageName: node linkType: hard +"msgpackr@npm:^1.10.2": + version: 1.10.2 + resolution: "msgpackr@npm:1.10.2" + dependencies: + msgpackr-extract: ^3.0.2 + dependenciesMeta: + msgpackr-extract: + optional: true + checksum: f7d9aae68196b3612e522a81c939b226045d115e97233541bf8475d170ca686f814bfdb6255865912d052f87f37466dbadf09ca17541b315f03d9ad9f6b439ef + languageName: node + linkType: hard + "msgpackr@npm:^1.9.9": version: 1.10.1 resolution: "msgpackr@npm:1.10.1"