feat: native world state (#7516)

This PR adds a new CPP module for managing the merkle trees that make up
the world state. A new implementetion of `MerkleTreeDb` is provided to
interact with the native code.

msgpack is used to pass messages across the js<->cpp boundary. Tests
have been added to assert that the two world state implementations work
in the same way (more tests would be better).

This PR builds on top of #7037.

PS: I'm not as experienced with C++

---------

Co-authored-by: PhilWindle <[email protected]>

barretenberg/cpp/CMakeLists.txt

@@ -38,6 +38,7 @@ option(ENABLE_HEAVY_TESTS "Enable heavy tests when collecting coverage" OFF)
 # Note: Must do 'sudo apt-get install libdw-dev' or equivalent
 option(CHECK_CIRCUIT_STACKTRACES "Enable (slow) stack traces for check circuit" OFF)
 option(ENABLE_TRACY "Enable low-medium overhead profiling for memory and performance with tracy" OFF)
+option(ENABLE_PIC "Builds with position independent code" OFF)
 
 if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64" OR CMAKE_SYSTEM_PROCESSOR MATCHES "arm64")
     message(STATUS "Compiling for ARM.")

barretenberg/cpp/CMakePresets.json

@@ -21,6 +21,23 @@
         "TARGET_ARCH": "skylake"
       }
     },
+    {
+      "name": "default-pic",
+      "displayName": "Build with Clang with Position Independent Code",
+      "description": "Build with globally installed Clang with Position Independent Code",
+      "binaryDir": "build-pic",
+      "generator": "Ninja",
+      "environment": {
+        "CC": "clang",
+        "CXX": "clang++",
+        "CMAKE_EXPORT_COMPILE_COMMANDS": "ON",
+        "LDFLAGS": "-Wl,-undefined,dynamic_lookup"
+      },
+      "cacheVariables": {
+        "TARGET_ARCH": "skylake",
+        "ENABLE_PIC": "ON"
+      }
+    },
     {
       "name": "homebrew",
       "displayName": "Homebrew + Clang",
@@ -66,6 +83,16 @@
         "CXX": "clang++-16"
       }
     },
+    {
+      "name": "clang16-pic",
+      "displayName": "Release build with Position Independent Code",
+      "description": "Build with globally installed Clang-16 using Position Independent Code",
+      "inherits": "clang16",
+      "binaryDir": "build-pic",
+      "cacheVariables": {
+        "ENABLE_PIC": "ON"
+      }
+    },
     {
       "name": "clang16-dbg",
       "displayName": "Debugging build with Clang-16",
@@ -446,6 +473,16 @@
       "inherits": "default",
       "configurePreset": "tracy"
     },
+    {
+      "name": "clang16-pic",
+      "inherits": "default",
+      "configurePreset": "clang16-pic"
+    },
+    {
+      "name": "default-pic",
+      "inherits": "default",
+      "configurePreset": "default-pic"
+    },
     {
       "name": "tracy-gates",
       "inherits": "default",

barretenberg/cpp/Earthfile

@@ -29,6 +29,11 @@ preset-release:
     RUN cmake --preset clang16 -Bbuild && cmake --build build --target bb && rm -rf build/{deps,lib,src}
     SAVE ARTIFACT build/bin
 
+preset-release-world-state:
+  FROM +source
+  RUN cmake --preset clang16-pic -Bbuild && cmake --build build --target world_state_napi && cp ./build/lib/world_state_napi.node ./build/bin && rm -rf build/{deps,lib,src}
+  SAVE ARTIFACT build/bin
+
 preset-release-assert:
     FROM +source
     RUN cmake --preset clang16-assert -Bbuild && cmake --build build --target bb crypto_merkle_tree_tests && rm -rf build/{deps,lib,src}
@@ -264,3 +269,4 @@ build:
     BUILD +preset-wasm
     BUILD +preset-wasm-threads
     BUILD +preset-release
+    BUILD +preset-release-world-state

barretenberg/cpp/bootstrap.sh

@@ -46,17 +46,27 @@ else
   fi
 fi
 
+PIC_PRESET="$PRESET-pic"
+
 # Remove cmake cache files.
 rm -f {build,build-wasm,build-wasm-threads}/CMakeCache.txt
 
+(cd src/barretenberg/world_state_napi && yarn --frozen-lockfile --prefer-offline)
+
 echo "#################################"
 echo "# Building with preset: $PRESET"
 echo "# When running cmake directly, remember to use: --build --preset $PRESET"
 echo "#################################"
 
 function build_native {
+  # Build bb with standard preset and world_state_napi with Position Independent code variant
   cmake --preset $PRESET -DCMAKE_BUILD_TYPE=RelWithAssert
+  cmake --preset $PIC_PRESET -DCMAKE_BUILD_TYPE=RelWithAssert
   cmake --build --preset $PRESET --target bb
+  cmake --build --preset $PIC_PRESET --target world_state_napi
+  # copy the world_state_napi build artifact over to the world state in yarn-project
+  mkdir -p ../../yarn-project/world-state/build/
+  cp ./build-pic/lib/world_state_napi.node ../../yarn-project/world-state/build/
 }
 
 function build_wasm {
@@ -106,7 +116,7 @@ fi
 
 if [ ! -d ./srs_db/grumpkin ]; then
   # The Grumpkin SRS is generated manually at the moment, only up to a large enough size for tests
-  # If tests require more points, the parameter can be increased here. Note: IPA requires 
+  # If tests require more points, the parameter can be increased here. Note: IPA requires
   # dyadic_circuit_size + 1 points so in general this number will be a power of two plus 1
   cd ./build && cmake --build . --parallel --target grumpkin_srs_gen && ./bin/grumpkin_srs_gen 8193
 fi

barretenberg/cpp/scripts/world_state_tests.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -e
+
+# run commands relative to parent directory
+cd $(dirname $0)/..
+
+TEST=${1:-*}
+PRESET=${PRESET:-clang16}
+
+cmake --build --preset $PRESET --target world_state_tests
+./build/bin/world_state_tests --gtest_filter=WorldStateTest.${TEST}

barretenberg/cpp/src/CMakeLists.txt

@@ -1,5 +1,6 @@
 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
 set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
+set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
 
 # Enable the following warnings project wide.
 # If any compilation issues arise in the future, they should not be silenced here but rather in the
@@ -53,6 +54,13 @@ else()
     message(STATUS "Using optimized assembly for field arithmetic.")
 endif()
 
+if (ENABLE_PIC AND CMAKE_CXX_COMPILER_ID MATCHES "Clang")
+  message("Building with Position Independent Code")
+  set(CMAKE_CXX_FLAGS  "${CMAKE_CXX_FLAGS} -fPIC")
+  set(CMAKE_C_FLAGS  "${CMAKE_C_FLAGS} -fPIC")
+  add_subdirectory(barretenberg/world_state_napi)
+endif()
+
 add_subdirectory(barretenberg/aztec_ivc)
 add_subdirectory(barretenberg/bb)
 add_subdirectory(barretenberg/circuit_checker)
@@ -87,6 +95,7 @@ add_subdirectory(barretenberg/translator_vm)
 add_subdirectory(barretenberg/ultra_honk)
 add_subdirectory(barretenberg/vm)
 add_subdirectory(barretenberg/wasi)
+add_subdirectory(barretenberg/world_state)
 
 if(SMT)
     add_subdirectory(barretenberg/smt_verification)

barretenberg/cpp/src/barretenberg/messaging/dispatcher.hpp

@@ -0,0 +1,41 @@
+#pragma once
+
+#include "barretenberg/messaging/header.hpp"
+#include "barretenberg/serialize/cbind.hpp"
+#include <cstdint>
+#include <functional>
+#include <stdexcept>
+#include <utility>
+#include <vector>
+
+namespace bb::messaging {
+
+using message_handler = std::function<bool(msgpack::object&, msgpack::sbuffer&)>;
+
+class MessageDispatcher {
+  private:
+    std::unordered_map<uint32_t, message_handler> messageHandlers;
+
+  public:
+    MessageDispatcher() = default;
+
+    bool onNewData(msgpack::object& obj, msgpack::sbuffer& buffer)
+    {
+        bb::messaging::HeaderOnlyMessage header;
+        obj.convert(header);
+
+        auto iter = messageHandlers.find(header.msgType);
+        if (iter == messageHandlers.end()) {
+            throw std::runtime_error("No registered handler for message of type " + std::to_string(header.msgType));
+        }
+
+        return (iter->second)(obj, buffer);
+    }
+
+    void registerTarget(uint32_t msgType, const message_handler& handler)
+    {
+        messageHandlers.insert({ msgType, handler });
+    }
+};
+
+} // namespace bb::messaging

barretenberg/cpp/src/barretenberg/world_state/CMakeLists.txt

@@ -0,0 +1 @@
+barretenberg_module(world_state crypto_merkle_tree stdlib_poseidon2)

barretenberg/cpp/src/barretenberg/world_state/tree_with_store.hpp

@@ -0,0 +1,34 @@
+#pragma once
+
+#include <memory>
+
+namespace bb::world_state {
+
+template <typename Tree> struct TreeWithStore {
+    using TreeType = Tree;
+    std::unique_ptr<Tree> tree;
+    std::unique_ptr<typename Tree::StoreType> store;
+    std::unique_ptr<typename Tree::StoreType::PersistedStoreType> persisted_store;
+
+    TreeWithStore(std::unique_ptr<Tree> t,
+                  std::unique_ptr<typename Tree::StoreType> s,
+                  std::unique_ptr<typename Tree::StoreType::PersistedStoreType> p)
+        : tree(std::move(t))
+        , store(std::move(s))
+        , persisted_store(std::move(p))
+    {}
+
+    TreeWithStore(TreeWithStore&& other) noexcept
+        : tree(std::move(other.tree))
+        , store(std::move(other.store))
+        , persisted_store(std::move(other.persisted_store))
+    {}
+
+    TreeWithStore(const TreeWithStore& other) = delete;
+    ~TreeWithStore() = default;
+
+    TreeWithStore& operator=(TreeWithStore&& other) = delete;
+    TreeWithStore& operator=(const TreeWithStore& other) = delete;
+};
+
+} // namespace bb::world_state

barretenberg/cpp/src/barretenberg/world_state/types.hpp

@@ -0,0 +1,38 @@
+#pragma once
+
+#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp"
+#include "barretenberg/crypto/merkle_tree/types.hpp"
+#include "barretenberg/ecc/curves/bn254/fr.hpp"
+#include <cstdint>
+#include <variant>
+
+namespace bb::world_state {
+
+enum MerkleTreeId {
+    NULLIFIER_TREE = 0,
+    NOTE_HASH_TREE = 1,
+    PUBLIC_DATA_TREE = 2,
+    L1_TO_L2_MESSAGE_TREE = 3,
+    ARCHIVE = 4,
+};
+
+using TreeStateReference = std::pair<bb::fr, bb::crypto::merkle_tree::index_t>;
+using StateReference = std::unordered_map<MerkleTreeId, TreeStateReference>;
+
+struct WorldStateRevision {
+    struct FinalisedBlock {
+        uint32_t block;
+    };
+
+    struct CurrentState {
+        bool uncommitted;
+    };
+
+    using Revision = std::variant<WorldStateRevision::FinalisedBlock, WorldStateRevision::CurrentState>;
+    Revision inner;
+
+    static WorldStateRevision committed() { return { CurrentState{ false } }; }
+    static WorldStateRevision uncommitted() { return { CurrentState{ true } }; }
+    static WorldStateRevision finalised_block(uint32_t block_number) { return { FinalisedBlock{ block_number } }; }
+};
+} // namespace bb::world_state