feat: native world state

.github/workflows/ci.yml

@@ -284,6 +284,24 @@ jobs:
         # limit our parallelism to half our cores
         run: earthly-ci --no-output +preset-gcc
 
+  bb-gcc-pic:
+    needs: [build-images, changes]
+    runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
+    if: ${{ needs.changes.outputs.barretenberg-cpp == 'true' }}
+    steps:
+      - uses: actions/checkout@v4
+        with: { ref: "${{ env.GIT_COMMIT }}" }
+      # Only allow one memory-hunger prover test to use this runner
+      - uses: ./.github/ci-setup-action
+        with:
+          # must be globally unique for build x runner
+          concurrency_key: bb-gcc-pic
+      - name: "Ensure GCC PIC Builds"
+        working-directory: ./barretenberg/cpp/
+        timeout-minutes: 40
+        # limit our parallelism to half our cores
+        run: earthly-ci --no-output +preset-gcc-pic
+
   # barretenberg (prover) native, AVM (public VM) and Merkle tree (world state) tests
   # only ran on x86 for resource reasons (memory intensive)
   bb-native-tests:
@@ -688,6 +706,7 @@ jobs:
       # - acir-bench # non-blocking
       # - bench-summary # non-blocking
       - bb-gcc
+      - bb-gcc-pic
       - bb-native-tests
       - bb-js-test
       - noir-build-acir-tests

barretenberg/cpp/CMakeLists.txt

@@ -38,6 +38,7 @@ option(ENABLE_HEAVY_TESTS "Enable heavy tests when collecting coverage" OFF)
 # Note: Must do 'sudo apt-get install libdw-dev' or equivalent
 option(CHECK_CIRCUIT_STACKTRACES "Enable (slow) stack traces for check circuit" OFF)
 option(ENABLE_TRACY "Enable low-medium overhead profiling for memory and performance with tracy" OFF)
+option(ENABLE_PIC "Builds with position independent code" OFF)
 
 if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64" OR CMAKE_SYSTEM_PROCESSOR MATCHES "arm64")
     message(STATUS "Compiling for ARM.")

barretenberg/cpp/CMakePresets.json

@@ -21,6 +21,22 @@
         "TARGET_ARCH": "skylake"
       }
     },
+    {
+      "name": "default-pic",
+      "displayName": "Build with Clang with Position Independent Code",
+      "description": "Build with globally installed Clang with Position Independent Code",
+      "binaryDir": "build-pic",
+      "generator": "Ninja",
+      "environment": {
+        "CC": "clang",
+        "CXX": "clang++",
+        "CMAKE_EXPORT_COMPILE_COMMANDS": "ON"
+      },
+      "cacheVariables": {
+        "TARGET_ARCH": "skylake",
+        "ENABLE_PIC": "ON"
+      }
+    },
     {
       "name": "homebrew",
       "displayName": "Homebrew + Clang",
@@ -66,6 +82,16 @@
         "CXX": "clang++-16"
       }
     },
+    {
+      "name": "clang16-pic",
+      "displayName": "Release build with Position Independent Code",
+      "description": "Build with globally installed Clang-16 using Position Independent Code",
+      "inherits": "clang16",
+      "binaryDir": "build-pic",
+      "cacheVariables": {
+        "ENABLE_PIC": "ON"
+      }
+    },
     {
       "name": "clang16-dbg",
       "displayName": "Debugging build with Clang-16",
@@ -172,6 +198,20 @@
         "CXX": "g++"
       }
     },
+    {
+      "name": "gcc-pic",
+      "displayName": "Build with GCC using PIC",
+      "description": "Build with globally installed GCC using PIC",
+      "inherits": "gcc",
+      "binaryDir": "build-gcc-pic",
+      "environment": {
+        "CC": "gcc",
+        "CXX": "g++"
+      },
+      "cacheVariables": {
+        "ENABLE_PIC": "ON"
+      }
+    },
     {
       "name": "gcc10",
       "displayName": "Build with GCC-10",
@@ -446,6 +486,11 @@
       "inherits": "default",
       "configurePreset": "tracy"
     },
+    {
+      "name": "clang16-pic",
+      "inherits": "default",
+      "configurePreset": "clang16-pic"
+    },
     {
       "name": "tracy-gates",
       "inherits": "default",
@@ -476,6 +521,11 @@
       "inherits": "default",
       "configurePreset": "gcc"
     },
+    {
+      "name": "gcc-pic",
+      "inherits": "default",
+      "configurePreset": "gcc-pic"
+    },
     {
       "name": "gcc10",
       "inherits": "default",

barretenberg/cpp/Earthfile

@@ -29,6 +29,11 @@ preset-release:
     RUN cmake --preset clang16 -Bbuild && cmake --build build --target bb && rm -rf build/{deps,lib,src}
     SAVE ARTIFACT build/bin
 
+preset-release-pic:
+  FROM +source
+  RUN cmake --preset clang16-pic -Bbuild && cmake --build build --target world_state_napi && cp ./build/lib/world_state_napi.node ./build/bin && rm -rf build/{deps,lib,src}
+  SAVE ARTIFACT build/bin
+
 preset-release-assert:
     FROM +source
     RUN cmake --preset clang16-assert -Bbuild && cmake --build build --target bb crypto_merkle_tree_tests && rm -rf build/{deps,lib,src}
@@ -102,6 +107,11 @@ preset-gcc:
     RUN cmake --preset gcc -Bbuild && cmake --build build && rm -rf build/{deps,lib,src}
     SAVE ARTIFACT build/bin
 
+preset-gcc-pic:
+    FROM +source
+    RUN cmake --preset gcc-pic -Bbuild && cmake --build build && rm -rf build/{deps,lib,src}
+    SAVE ARTIFACT build/bin
+
 preset-fuzzing:
     FROM +source
     RUN cmake --preset fuzzing -Bbuild && cmake --build build && rm -rf build/{deps,lib,src}
@@ -264,3 +274,4 @@ build:
     BUILD +preset-wasm
     BUILD +preset-wasm-threads
     BUILD +preset-release
+    BUILD +preset-release-pic

barretenberg/cpp/bootstrap.sh

@@ -46,17 +46,27 @@ else
   fi
 fi
 
+PIC_PRESET="$PRESET-pic"
+
 # Remove cmake cache files.
 rm -f {build,build-wasm,build-wasm-threads}/CMakeCache.txt
 
+(cd src/barretenberg/world_state_napi && yarn --frozen-lockfile --prefer-offline)
+
 echo "#################################"
 echo "# Building with preset: $PRESET"
 echo "# When running cmake directly, remember to use: --build --preset $PRESET"
 echo "#################################"
 
 function build_native {
+  # Build bb with standard preset and world_state_napi with Position Independent code variant
   cmake --preset $PRESET -DCMAKE_BUILD_TYPE=RelWithAssert
+  cmake --preset $PIC_PRESET -DCMAKE_BUILD_TYPE=RelWithAssert
   cmake --build --preset $PRESET --target bb
+  cmake --build --preset $PIC_PRESET --target world_state_napi
+  # copy the world_state_napi build artifact over to the world state in yarh-project
+  mkdir -p ../../yarn-project/world-state/build/
+  cp ./build-pic/lib/world_state_napi.node ../../yarn-project/world-state/build/
 }
 
 function build_wasm {
@@ -106,7 +116,7 @@ fi
 
 if [ ! -d ./srs_db/grumpkin ]; then
   # The Grumpkin SRS is generated manually at the moment, only up to a large enough size for tests
-  # If tests require more points, the parameter can be increased here. Note: IPA requires 
+  # If tests require more points, the parameter can be increased here. Note: IPA requires
   # dyadic_circuit_size + 1 points so in general this number will be a power of two plus 1
   cd ./build && cmake --build . --parallel --target grumpkin_srs_gen && ./bin/grumpkin_srs_gen 8193
 fi

barretenberg/cpp/scripts/world_state_tests.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -e
+
+# run commands relative to parent directory
+cd $(dirname $0)/..
+
+TEST=${1:-*}
+PRESET=${PRESET:-clang16}
+
+cmake --build --preset $PRESET --target world_state_tests
+./build/bin/world_state_tests --gtest_filter=WorldStateTest.${TEST}

barretenberg/cpp/src/CMakeLists.txt

@@ -1,5 +1,6 @@
 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
 set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
+set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
 
 # Enable the following warnings project wide.
 # If any compilation issues arise in the future, they should not be silenced here but rather in the
@@ -53,6 +54,13 @@ else()
     message(STATUS "Using optimized assembly for field arithmetic.")
 endif()
 
+if (ENABLE_PIC AND CMAKE_CXX_COMPILER_ID MATCHES "Clang")
+  message("Building with Position Independent Code")
+  set(CMAKE_CXX_FLAGS  "${CMAKE_CXX_FLAGS} -fPIC")
+  set(CMAKE_C_FLAGS  "${CMAKE_C_FLAGS} -fPIC")
+  add_subdirectory(barretenberg/world_state_napi)
+endif()
+
 add_subdirectory(barretenberg/aztec_ivc)
 add_subdirectory(barretenberg/bb)
 add_subdirectory(barretenberg/circuit_checker)
@@ -87,6 +95,7 @@ add_subdirectory(barretenberg/translator_vm)
 add_subdirectory(barretenberg/ultra_honk)
 add_subdirectory(barretenberg/vm)
 add_subdirectory(barretenberg/wasi)
+add_subdirectory(barretenberg/world_state)
 
 if(SMT)
     add_subdirectory(barretenberg/smt_verification)

barretenberg/cpp/src/barretenberg/messaging/dispatcher.hpp

@@ -0,0 +1,41 @@
+#pragma once
+
+#include "barretenberg/messaging/header.hpp"
+#include "barretenberg/serialize/cbind.hpp"
+#include <cstdint>
+#include <functional>
+#include <stdexcept>
+#include <utility>
+#include <vector>
+
+namespace bb::messaging {
+
+using message_handler = std::function<bool(msgpack::object&, msgpack::sbuffer&)>;
+
+class MessageDispatcher {
+  private:
+    std::unordered_map<uint32_t, message_handler> messageHandlers;
+
+  public:
+    MessageDispatcher() = default;
+
+    bool onNewData(msgpack::object& obj, msgpack::sbuffer& buffer)
+    {
+        bb::messaging::HeaderOnlyMessage header;
+        obj.convert(header);
+
+        auto iter = messageHandlers.find(header.msgType);
+        if (iter == messageHandlers.end()) {
+            throw std::runtime_error("No registered handler for message of type " + std::to_string(header.msgType));
+        }
+
+        return (iter->second)(obj, buffer);
+    }
+
+    void registerTarget(uint32_t msgType, const message_handler& handler)
+    {
+        messageHandlers.insert({ msgType, handler });
+    }
+};
+
+} // namespace bb::messaging

barretenberg/cpp/src/barretenberg/world_state/CMakeLists.txt

@@ -0,0 +1 @@
+barretenberg_module(world_state crypto_merkle_tree stdlib_poseidon2)

barretenberg/cpp/src/barretenberg/world_state/tree_with_store.hpp

@@ -0,0 +1,34 @@
+#pragma once
+
+#include <memory>
+
+namespace bb::world_state {
+
+template <typename Tree> struct TreeWithStore {
+    using TreeType = Tree;
+    std::unique_ptr<Tree> tree;
+    std::unique_ptr<typename Tree::StoreType> store;
+    std::unique_ptr<typename Tree::StoreType::PersistedStoreType> persisted_store;
+
+    TreeWithStore(std::unique_ptr<Tree> t,
+                  std::unique_ptr<typename Tree::StoreType> s,
+                  std::unique_ptr<typename Tree::StoreType::PersistedStoreType> p)
+        : tree(std::move(t))
+        , store(std::move(s))
+        , persisted_store(std::move(p))
+    {}
+
+    TreeWithStore(TreeWithStore&& other) noexcept
+        : tree(std::move(other.tree))
+        , store(std::move(other.store))
+        , persisted_store(std::move(other.persisted_store))
+    {}
+
+    TreeWithStore(const TreeWithStore& other) = delete;
+    ~TreeWithStore() = default;
+
+    TreeWithStore& operator=(TreeWithStore&& other) = delete;
+    TreeWithStore& operator=(const TreeWithStore& other) = delete;
+};
+
+} // namespace bb::world_state

barretenberg/cpp/src/barretenberg/world_state/types.hpp

@@ -0,0 +1,38 @@
+#pragma once
+
+#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp"
+#include "barretenberg/crypto/merkle_tree/types.hpp"
+#include "barretenberg/ecc/curves/bn254/fr.hpp"
+#include <cstdint>
+#include <variant>
+
+namespace bb::world_state {
+
+enum MerkleTreeId {
+    NULLIFIER_TREE = 0,
+    NOTE_HASH_TREE = 1,
+    PUBLIC_DATA_TREE = 2,
+    L1_TO_L2_MESSAGE_TREE = 3,
+    ARCHIVE = 4,
+};
+
+using TreeStateReference = std::pair<bb::fr, bb::crypto::merkle_tree::index_t>;
+using StateReference = std::unordered_map<MerkleTreeId, TreeStateReference>;
+
+struct WorldStateRevision {
+    struct FinalisedBlock {
+        uint32_t block;
+    };
+
+    struct CurrentState {
+        bool uncommitted;
+    };
+
+    using Revision = std::variant<WorldStateRevision::FinalisedBlock, WorldStateRevision::CurrentState>;
+    Revision inner;
+
+    static WorldStateRevision committed() { return { CurrentState{ false } }; }
+    static WorldStateRevision uncommitted() { return { CurrentState{ true } }; }
+    static WorldStateRevision finalised_block(uint32_t block_number) { return { FinalisedBlock{ block_number } }; }
+};
+} // namespace bb::world_state