chore: reduce redundancy with msgpack (#1108)

* chore(bb): reduced msgpack redundancy

* chore: Use msgpack to lower redundancy.

* chore(bb): remove msgpack redundancy

* chore: bump bb module

* fix: build

* chore: submodule chores

circuits/cpp/CMakeLists.txt

@@ -23,7 +23,6 @@ option(MULTITHREADING "Enable multi-threading" ON)
 option(TESTING "Build tests" ON)
 option(ENABLE_ASAN "Address sanitizer for debugging tricky memory corruption" OFF)
 option(BENCHMARKS "Build benchmarks" ON)
-option(SERIALIZE_CANARY "Enable serialization checks" OFF)
 option(FUZZING "Build fuzzing harnesses" OFF)
 option(DISABLE_TBB "Intel Thread Building Blocks" ON)
 option(COVERAGE "Enable collecting coverage from tests" OFF)
@@ -46,10 +45,6 @@ if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64" OR CMAKE_SYSTEM_PROCESSOR MATCHES "a
     set(DISABLE_TBB 0)
 endif()
 
-if(SERIALIZE_CANARY)
-    add_definitions(-DENABLE_SERIALIZE_CANARY)
-endif()
-
 if(FUZZING)
     add_definitions(-DFUZZING=1)
 

circuits/cpp/cmake/barretenberg.cmake

@@ -38,7 +38,6 @@ ExternalProject_Add(Barretenberg
         ${CMAKE_COMMAND}
         --preset ${CMAKE_BBERG_PRESET}
         -DCMAKE_CXX_FLAGS=${CMAKE_BBERG_CXX_FLAGS}
-        -DSERIALIZE_CANARY=${SERIALIZE_CANARY}
         -DMULTITHREADING=${MULTITHREADING}
         -DENABLE_ASAN=${ENABLE_ASAN}
         -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}

circuits/cpp/src/aztec3/circuits/abis/append_only_tree_snapshot.hpp

@@ -15,26 +15,10 @@ template <typename NCT> struct AppendOnlyTreeSnapshot {
     bool operator==(AppendOnlyTreeSnapshot<NCT> const&) const = default;
 };
 
-template <typename NCT> void read(uint8_t const*& it, AppendOnlyTreeSnapshot<NCT>& obj)
-{
-    using serialize::read;
-
-    read(it, obj.root);
-    read(it, obj.next_available_leaf_index);
-};
-
-template <typename NCT> void write(std::vector<uint8_t>& buf, AppendOnlyTreeSnapshot<NCT> const& obj)
-{
-    using serialize::write;
-
-    write(buf, obj.root);
-    write(buf, obj.next_available_leaf_index);
-};
-
 template <typename NCT> std::ostream& operator<<(std::ostream& os, AppendOnlyTreeSnapshot<NCT> const& obj)
 {
     return os << "root: " << obj.root << "\n"
               << "next_available_leaf_index: " << obj.next_available_leaf_index << "\n";
 }
 
-}  // namespace aztec3::circuits::abis
+}  // namespace aztec3::circuits::abis

circuits/cpp/src/aztec3/circuits/abis/c_bind.cpp

@@ -96,6 +96,7 @@ static const char* bbmalloc_copy_string(const char* data, size_t len)
  * For testing only. Take this object, write it to a buffer, then output it. */
 template <typename T> static const char* as_string_output(uint8_t const* input_buf, uint32_t* size)
 {
+    using serialize::read;
     T obj;
     read(input_buf, obj);
     std::ostringstream stream;
@@ -109,6 +110,7 @@ template <typename T> static const char* as_string_output(uint8_t const* input_b
  * For testing only. Take this object, serialize it to a buffer, then output it. */
 template <typename T> static const char* as_serialized_output(uint8_t const* input_buf, uint32_t* size)
 {
+    using serialize::read;
     T obj;
     read(input_buf, obj);
     std::vector<uint8_t> stream;
@@ -282,7 +284,7 @@ WASM_EXPORT void abis__hash_constructor(uint8_t const* function_data_buf,
     NT::fr args_hash;
     NT::fr constructor_vk_hash;
 
-    read(function_data_buf, function_data);
+    serialize::read(function_data_buf, function_data);
     read(args_hash_buf, args_hash);
     read(constructor_vk_hash_buf, constructor_vk_hash);
 
@@ -315,7 +317,7 @@ WASM_EXPORT void abis__compute_contract_address(uint8_t const* point_data_buf,
     NT::fr function_tree_root;
     NT::fr constructor_hash;
 
-    read(point_data_buf, deployer_public_key);
+    serialize::read(point_data_buf, deployer_public_key);
     read(contract_address_salt_buf, contract_address_salt);
     read(function_tree_root_buf, function_tree_root);
     read(constructor_hash_buf, constructor_hash);
@@ -344,7 +346,7 @@ WASM_EXPORT void abis__compute_contract_address_from_partial(uint8_t const* poin
     Point<NT> deployer_public_key;
     NT::fr partial_address;
 
-    read(point_data_buf, deployer_public_key);
+    serialize::read(point_data_buf, deployer_public_key);
     read(partial_address_data_buf, partial_address);
 
     NT::fr const contract_address =
@@ -412,7 +414,7 @@ WASM_EXPORT void abis__compute_var_args_hash(uint8_t const* args_buf, uint8_t* o
 WASM_EXPORT void abis__compute_contract_leaf(uint8_t const* contract_leaf_preimage_buf, uint8_t* output)
 {
     NewContractData<NT> leaf_preimage;
-    read(contract_leaf_preimage_buf, leaf_preimage);
+    serialize::read(contract_leaf_preimage_buf, leaf_preimage);
     // as per the circuit implementation, if contract address == zero then return a zero leaf
     auto to_write = leaf_preimage.hash();
     NT::fr::serialize_to_buffer(to_write, output);
@@ -460,7 +462,7 @@ WASM_EXPORT void abis__compute_transaction_hash(uint8_t const* tx_request_buf, u
 WASM_EXPORT void abis__compute_call_stack_item_hash(uint8_t const* call_stack_item_buf, uint8_t* output)
 {
     CallStackItem<NT, PublicTypes> call_stack_item;
-    read(call_stack_item_buf, call_stack_item);
+    serialize::read(call_stack_item_buf, call_stack_item);
     NT::fr::serialize_to_buffer(get_call_stack_item_hash(call_stack_item), output);
 }
 

circuits/cpp/src/aztec3/circuits/abis/c_bind.test.cpp

@@ -1,4 +1,5 @@
 #include "c_bind.h"
+
 #include "function_leaf_preimage.hpp"
 #include "tx_request.hpp"
 
@@ -89,7 +90,7 @@ TEST(abi_tests, compute_contract_address)
     write(contract_address_salt_buf, contract_address_salt);
     write(function_tree_root_buf, function_tree_root);
     write(constructor_hash_buf, constructor_hash);
-    write(point_buf, point);
+    serialize::write(point_buf, point);
     abis__compute_contract_address(point_buf.data(),
                                    contract_address_salt_buf.data(),
                                    function_tree_root_buf.data(),
@@ -287,7 +288,7 @@ TEST(abi_tests, hash_constructor)
 
     // Write the function data and args to a buffer
     std::vector<uint8_t> func_data_buf;
-    write(func_data_buf, func_data);
+    serialize::write(func_data_buf, func_data);
 
     std::vector<uint8_t> args_hash_buf;
     write(args_hash_buf, args_hash);
@@ -346,7 +347,7 @@ TEST(abi_tests, compute_contract_leaf)
 
     // Write the leaf preimage to a buffer
     std::vector<uint8_t> preimage_buf;
-    write(preimage_buf, preimage);
+    serialize::write(preimage_buf, preimage);
 
     std::array<uint8_t, sizeof(NT::fr)> output = { 0 };
     abis__compute_contract_leaf(preimage_buf.data(), output.data());

circuits/cpp/src/aztec3/circuits/abis/call_context.hpp

@@ -104,34 +104,10 @@ template <typename NCT> struct CallContext {
     }
 };
 
-template <typename NCT> void read(uint8_t const*& it, CallContext<NCT>& call_context)
-{
-    using serialize::read;
-
-    read(it, call_context.msg_sender);
-    read(it, call_context.storage_contract_address);
-    read(it, call_context.portal_contract_address);
-    read(it, call_context.is_delegate_call);
-    read(it, call_context.is_static_call);
-    read(it, call_context.is_contract_deployment);
-};
-
-template <typename NCT> void write(std::vector<uint8_t>& buf, CallContext<NCT> const& call_context)
-{
-    using serialize::write;
-
-    write(buf, call_context.msg_sender);
-    write(buf, call_context.storage_contract_address);
-    write(buf, call_context.portal_contract_address);
-    write(buf, call_context.is_delegate_call);
-    write(buf, call_context.is_static_call);
-    write(buf, call_context.is_contract_deployment);
-};
-
 template <typename NCT> std::ostream& operator<<(std::ostream& os, CallContext<NCT> const& call_context)
 {
     utils::msgpack_derived_output(os, call_context);
     return os;
 }
 
-}  // namespace aztec3::circuits::abis
+}  // namespace aztec3::circuits::abis

circuits/cpp/src/aztec3/circuits/abis/call_stack_item.hpp

@@ -82,28 +82,6 @@ template <typename NCT, template <class> typename PrivatePublic> struct CallStac
     }
 };
 
-template <typename NCT, template <class> typename PrivatePublic>
-void read(uint8_t const*& it, CallStackItem<NCT, PrivatePublic>& call_stack_item)
-{
-    using serialize::read;
-
-    read(it, call_stack_item.contract_address);
-    read(it, call_stack_item.function_data);
-    read(it, call_stack_item.public_inputs);
-    read(it, call_stack_item.is_execution_request);
-};
-
-template <typename NCT, template <class> typename PrivatePublic>
-void write(std::vector<uint8_t>& buf, CallStackItem<NCT, PrivatePublic> const& call_stack_item)
-{
-    using serialize::write;
-
-    write(buf, call_stack_item.contract_address);
-    write(buf, call_stack_item.function_data);
-    write(buf, call_stack_item.public_inputs);
-    write(buf, call_stack_item.is_execution_request);
-};
-
 template <typename NCT, template <class> typename PrivatePublic>
 std::ostream& operator<<(std::ostream& os, CallStackItem<NCT, PrivatePublic> const& call_stack_item)
 {

circuits/cpp/src/aztec3/circuits/abis/combined_accumulated_data.hpp

@@ -245,50 +245,6 @@ template <typename NCT> struct CombinedAccumulatedData {
     }
 };
 
-template <typename NCT> void read(uint8_t const*& it, CombinedAccumulatedData<NCT>& accum_data)
-{
-    using serialize::read;
-
-    read(it, accum_data.aggregation_object);
-    read(it, accum_data.read_requests);
-    read(it, accum_data.read_request_membership_witnesses);
-    read(it, accum_data.new_commitments);
-    read(it, accum_data.new_nullifiers);
-    read(it, accum_data.private_call_stack);
-    read(it, accum_data.public_call_stack);
-    read(it, accum_data.new_l2_to_l1_msgs);
-    read(it, accum_data.encrypted_logs_hash);
-    read(it, accum_data.unencrypted_logs_hash);
-    read(it, accum_data.encrypted_log_preimages_length);
-    read(it, accum_data.unencrypted_log_preimages_length);
-    read(it, accum_data.new_contracts);
-    read(it, accum_data.optionally_revealed_data);
-    read(it, accum_data.public_data_update_requests);
-    read(it, accum_data.public_data_reads);
-};
-
-template <typename NCT> void write(std::vector<uint8_t>& buf, CombinedAccumulatedData<NCT> const& accum_data)
-{
-    using serialize::write;
-
-    write(buf, accum_data.aggregation_object);
-    write(buf, accum_data.read_requests);
-    write(buf, accum_data.read_request_membership_witnesses);
-    write(buf, accum_data.new_commitments);
-    write(buf, accum_data.new_nullifiers);
-    write(buf, accum_data.private_call_stack);
-    write(buf, accum_data.public_call_stack);
-    write(buf, accum_data.new_l2_to_l1_msgs);
-    write(buf, accum_data.encrypted_logs_hash);
-    write(buf, accum_data.unencrypted_logs_hash);
-    write(buf, accum_data.encrypted_log_preimages_length);
-    write(buf, accum_data.unencrypted_log_preimages_length);
-    write(buf, accum_data.new_contracts);
-    write(buf, accum_data.optionally_revealed_data);
-    write(buf, accum_data.public_data_update_requests);
-    write(buf, accum_data.public_data_reads);
-};
-
 template <typename NCT> std::ostream& operator<<(std::ostream& os, CombinedAccumulatedData<NCT> const& accum_data)
 {
     return os << "aggregation_object:\n"

circuits/cpp/src/aztec3/circuits/abis/combined_constant_data.hpp

@@ -65,26 +65,10 @@ template <typename NCT> struct CombinedConstantData {
     }
 };
 
-template <typename NCT> void read(uint8_t const*& it, CombinedConstantData<NCT>& constant_data)
-{
-    using serialize::read;
-
-    read(it, constant_data.historic_tree_roots);
-    read(it, constant_data.tx_context);
-};
-
-template <typename NCT> void write(std::vector<uint8_t>& buf, CombinedConstantData<NCT> const& constant_data)
-{
-    using serialize::write;
-
-    write(buf, constant_data.historic_tree_roots);
-    write(buf, constant_data.tx_context);
-};
-
 template <typename NCT> std::ostream& operator<<(std::ostream& os, CombinedConstantData<NCT> const& constant_data)
 {
     return os << "historic_tree_roots: " << constant_data.historic_tree_roots << "\n"
               << "tx_context: " << constant_data.tx_context << "\n";
 }
 
-}  // namespace aztec3::circuits::abis
+}  // namespace aztec3::circuits::abis

circuits/cpp/src/aztec3/circuits/abis/combined_historic_tree_roots.hpp

@@ -61,24 +61,10 @@ template <typename NCT> struct CombinedHistoricTreeRoots {
     }
 };
 
-template <typename NCT> void read(uint8_t const*& it, CombinedHistoricTreeRoots<NCT>& historic_tree_roots)
-{
-    using serialize::read;
-
-    read(it, historic_tree_roots.private_historic_tree_roots);
-};
-
-template <typename NCT> void write(std::vector<uint8_t>& buf, CombinedHistoricTreeRoots<NCT> const& historic_tree_roots)
-{
-    using serialize::write;
-
-    write(buf, historic_tree_roots.private_historic_tree_roots);
-};
-
 template <typename NCT>
 std::ostream& operator<<(std::ostream& os, CombinedHistoricTreeRoots<NCT> const& historic_tree_roots)
 {
     return os << "private_historic_tree_roots: " << historic_tree_roots.private_historic_tree_roots << "\n";
 }
 
-}  // namespace aztec3::circuits::abis
+}  // namespace aztec3::circuits::abis

circuits/cpp/src/aztec3/circuits/abis/contract_deployment_data.hpp

@@ -102,28 +102,6 @@ template <typename NCT> struct ContractDeploymentData {
     }
 };
 
-template <typename NCT> void read(uint8_t const*& it, ContractDeploymentData<NCT>& data)
-{
-    using serialize::read;
-
-    read(it, data.deployer_public_key);
-    read(it, data.constructor_vk_hash);
-    read(it, data.function_tree_root);
-    read(it, data.contract_address_salt);
-    read(it, data.portal_contract_address);
-};
-
-template <typename NCT> void write(std::vector<uint8_t>& buf, ContractDeploymentData<NCT> const& data)
-{
-    using serialize::write;
-
-    write(buf, data.deployer_public_key);
-    write(buf, data.constructor_vk_hash);
-    write(buf, data.function_tree_root);
-    write(buf, data.contract_address_salt);
-    write(buf, data.portal_contract_address);
-};
-
 template <typename NCT> std::ostream& operator<<(std::ostream& os, ContractDeploymentData<NCT> const& data)
 {
     return os << "deployer_public_key: " << data.deployer_public_key << "\n"
@@ -133,4 +111,4 @@ template <typename NCT> std::ostream& operator<<(std::ostream& os, ContractDeplo
               << "portal_contract_address: " << data.portal_contract_address << "\n";
 }
 
-}  // namespace aztec3::circuits::abis
+}  // namespace aztec3::circuits::abis

circuits/cpp/src/aztec3/circuits/abis/contract_storage_read.hpp

@@ -73,22 +73,6 @@ template <typename NCT> struct ContractStorageRead {
     boolean is_empty() const { return storage_slot == 0; }
 };
 
-template <typename NCT> void read(uint8_t const*& it, ContractStorageRead<NCT>& contract_storage_read)
-{
-    using serialize::read;
-
-    read(it, contract_storage_read.storage_slot);
-    read(it, contract_storage_read.current_value);
-};
-
-template <typename NCT> void write(std::vector<uint8_t>& buf, ContractStorageRead<NCT> const& contract_storage_read)
-{
-    using serialize::write;
-
-    write(buf, contract_storage_read.storage_slot);
-    write(buf, contract_storage_read.current_value);
-};
-
 template <typename NCT>
 std::ostream& operator<<(std::ostream& os, ContractStorageRead<NCT> const& contract_storage_read)
 {