diff --git a/barretenberg/.gitmodules b/barretenberg/.gitmodules index ae63c99f6fce..7f651c8fde7c 100644 --- a/barretenberg/.gitmodules +++ b/barretenberg/.gitmodules @@ -1,3 +1,6 @@ +[submodule "cpp/src/msgpack-c"] + path = cpp/src/msgpack-c + url = https://github.com/AztecProtocol/msgpack-c [submodule "foundation"] path = foundation url = git@github.com:AztecProtocol/foundation.git diff --git a/barretenberg/cpp/src/CMakeLists.txt b/barretenberg/cpp/src/CMakeLists.txt index 69d867a2f7c8..7f12367c670a 100644 --- a/barretenberg/cpp/src/CMakeLists.txt +++ b/barretenberg/cpp/src/CMakeLists.txt @@ -27,7 +27,7 @@ if(CMAKE_CXX_COMPILER_ID MATCHES "GNU") add_compile_options(-fconstexpr-ops-limit=100000000) endif() -include_directories(${CMAKE_CURRENT_SOURCE_DIR}) +include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/msgpack-c/include) # I feel this should be limited to ecc, however it's currently used in headers that go across libraries, # and there currently isn't an easy way to inherit the DDISABLE_SHENANIGANS parameter. @@ -52,6 +52,7 @@ add_subdirectory(barretenberg/plonk) add_subdirectory(barretenberg/stdlib) add_subdirectory(barretenberg/join_split_example) add_subdirectory(barretenberg/dsl) +add_subdirectory(barretenberg/serialize) add_subdirectory(barretenberg/solidity_helpers) if(BENCHMARKS) diff --git a/barretenberg/cpp/src/barretenberg/common/log.hpp b/barretenberg/cpp/src/barretenberg/common/log.hpp index 729ffcb28c11..f58164588348 100644 --- a/barretenberg/cpp/src/barretenberg/common/log.hpp +++ b/barretenberg/cpp/src/barretenberg/common/log.hpp @@ -18,23 +18,10 @@ namespace { -inline void format_chain(std::ostream&) {} - -template void format_chain(std::ostream& os, T const& first) -{ - os << first; -} - -template void format_chain(std::ostream& os, T const& first, Args const&... args) -{ - os << first; - format_chain(os, args...); -} - template std::string format(Args... args) { std::ostringstream os; - format_chain(os, args...); + ((os << args), ...); return os.str(); } diff --git a/barretenberg/cpp/src/barretenberg/common/serialize.hpp b/barretenberg/cpp/src/barretenberg/common/serialize.hpp index 622dc919f73c..126097470ce1 100644 --- a/barretenberg/cpp/src/barretenberg/common/serialize.hpp +++ b/barretenberg/cpp/src/barretenberg/common/serialize.hpp @@ -1,5 +1,6 @@ /** - * This is the core serialization library. + * This is a non-msgpack flat buffer serialization library. + * It is currently used alongside msgpack, with hope to eventually move to msgpack. * It enables the reading and writing of big-endian formatted integers and various standard library types * to and from the following supported types: * - uint8_t* @@ -40,6 +41,7 @@ #ifndef __i386__ __extension__ using uint128_t = unsigned __int128; #endif + namespace serialize { // Basic integer read / write, to / from raw buffers. // Pointers to buffers are advanced by length of type. diff --git a/barretenberg/cpp/src/barretenberg/common/throw_or_abort.hpp b/barretenberg/cpp/src/barretenberg/common/throw_or_abort.hpp index 02f57437ce9a..623258724639 100644 --- a/barretenberg/cpp/src/barretenberg/common/throw_or_abort.hpp +++ b/barretenberg/cpp/src/barretenberg/common/throw_or_abort.hpp @@ -7,7 +7,7 @@ inline void throw_or_abort [[noreturn]] (std::string const& err) #ifndef __wasm__ throw std::runtime_error(err); #else - info(err); + info("abort: ", err); std::abort(); #endif -} +} \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/ecdsa/ecdsa.hpp b/barretenberg/cpp/src/barretenberg/crypto/ecdsa/ecdsa.hpp index 57e9074a1bd1..df1e887fdc5e 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/ecdsa/ecdsa.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/ecdsa/ecdsa.hpp @@ -3,6 +3,7 @@ #include #include #include "barretenberg/ecc/curves/secp256k1/secp256k1.hpp" +#include "barretenberg/serialize/msgpack.hpp" namespace crypto { namespace ecdsa { @@ -15,6 +16,8 @@ struct signature { std::array r; std::array s; uint8_t v; + // for serialization, update with any new fields + MSGPACK_FIELDS(r, s, v); }; template @@ -70,4 +73,4 @@ template inline void write(B& buf, key_pair using namespace barretenberg; +TEST(ecdsa, msgpack) +{ + auto [actual, expected] = msgpack_roundtrip(crypto::ecdsa::signature{}); + EXPECT_EQ(actual, expected); +} + TEST(ecdsa, verify_signature_grumpkin_sha256) { std::string message = "The quick brown dog jumped over the lazy fox."; diff --git a/barretenberg/cpp/src/barretenberg/crypto/pedersen_commitment/c_bind.cpp b/barretenberg/cpp/src/barretenberg/crypto/pedersen_commitment/c_bind.cpp index 1b58491c3ce9..f503481ce7d0 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/pedersen_commitment/c_bind.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/pedersen_commitment/c_bind.cpp @@ -30,7 +30,6 @@ WASM_EXPORT void pedersen_plookup_compress_fields(uint8_t const* left, uint8_t c barretenberg::fr::serialize_to_buffer(r, result); } - WASM_EXPORT void pedersen__compress(uint8_t const* inputs_buffer, uint8_t* output) { std::vector to_compress; diff --git a/barretenberg/cpp/src/barretenberg/dsl/acir_format/acir_format.test.cpp b/barretenberg/cpp/src/barretenberg/dsl/acir_format/acir_format.test.cpp index 3c10153cedd1..a7b39363d4c9 100644 --- a/barretenberg/cpp/src/barretenberg/dsl/acir_format/acir_format.test.cpp +++ b/barretenberg/cpp/src/barretenberg/dsl/acir_format/acir_format.test.cpp @@ -3,8 +3,13 @@ #include #include #include "barretenberg/common/streams.hpp" +#include "barretenberg/serialize/test_helper.hpp" #include "ecdsa_secp256k1.hpp" - +TEST(acir_format, msgpack_logic_constraint) +{ + auto [actual, expected] = msgpack_roundtrip(acir_format::LogicConstraint {}); + EXPECT_EQ(actual, expected); +} TEST(acir_format, test_logic_gate_from_noir_circuit) { /** diff --git a/barretenberg/cpp/src/barretenberg/dsl/acir_format/logic_constraint.hpp b/barretenberg/cpp/src/barretenberg/dsl/acir_format/logic_constraint.hpp index 964f84f5af90..d2ed943398c4 100644 --- a/barretenberg/cpp/src/barretenberg/dsl/acir_format/logic_constraint.hpp +++ b/barretenberg/cpp/src/barretenberg/dsl/acir_format/logic_constraint.hpp @@ -12,6 +12,9 @@ struct LogicConstraint { uint32_t is_xor_gate; friend bool operator==(LogicConstraint const& lhs, LogicConstraint const& rhs) = default; + + // for serialization, update with any new fields + MSGPACK_FIELDS(a, b, result, num_bits, is_xor_gate); }; void create_logic_gate(Composer& composer, uint32_t a, uint32_t b, uint32_t result, size_t num_bits, bool is_xor_gate); diff --git a/barretenberg/cpp/src/barretenberg/dsl/acir_format/sha256_constraint.hpp b/barretenberg/cpp/src/barretenberg/dsl/acir_format/sha256_constraint.hpp index eccb1521ab86..3c434bae26d6 100644 --- a/barretenberg/cpp/src/barretenberg/dsl/acir_format/sha256_constraint.hpp +++ b/barretenberg/cpp/src/barretenberg/dsl/acir_format/sha256_constraint.hpp @@ -10,6 +10,8 @@ struct Sha256Input { uint32_t num_bits; friend bool operator==(Sha256Input const& lhs, Sha256Input const& rhs) = default; + // for serialization, update with any new fields + MSGPACK_FIELDS(witness, num_bits); }; struct Sha256Constraint { @@ -17,6 +19,8 @@ struct Sha256Constraint { std::vector result; friend bool operator==(Sha256Constraint const& lhs, Sha256Constraint const& rhs) = default; + // for serialization, update with any new fields + MSGPACK_FIELDS(inputs, result); }; // This function does not work (properly) because the stdlib:sha256 function is not working correctly for 512 bits diff --git a/barretenberg/cpp/src/barretenberg/ecc/curves/bn254/fq.hpp b/barretenberg/cpp/src/barretenberg/ecc/curves/bn254/fq.hpp index 32885d81f16a..080184d2dba0 100644 --- a/barretenberg/cpp/src/barretenberg/ecc/curves/bn254/fq.hpp +++ b/barretenberg/cpp/src/barretenberg/ecc/curves/bn254/fq.hpp @@ -60,4 +60,10 @@ class Bn254FqParams { typedef field fq; -} // namespace barretenberg \ No newline at end of file +} // namespace barretenberg + +// define this as a named alias in msgpack schema generation +inline void msgpack_schema_pack(auto& packer, barretenberg::fq const&) +{ + packer.pack_alias("Fq", "bin32"); +} diff --git a/barretenberg/cpp/src/barretenberg/ecc/curves/bn254/fq.test.cpp b/barretenberg/cpp/src/barretenberg/ecc/curves/bn254/fq.test.cpp index 79e7bdea70ff..ce5e663236da 100644 --- a/barretenberg/cpp/src/barretenberg/ecc/curves/bn254/fq.test.cpp +++ b/barretenberg/cpp/src/barretenberg/ecc/curves/bn254/fq.test.cpp @@ -1,9 +1,16 @@ #include "fq.hpp" #include "pseudorandom.hpp" +#include "barretenberg/serialize/test_helper.hpp" #include using namespace barretenberg; +TEST(fq, msgpack) +{ + auto [actual, expected] = msgpack_roundtrip(barretenberg::fq{1ull, 2ull, 3ull, 4ull}); + EXPECT_EQ(actual, expected); +} + TEST(fq, eq) { constexpr fq a{ 0x01, 0x02, 0x03, 0x04 }; diff --git a/barretenberg/cpp/src/barretenberg/ecc/curves/bn254/fr.hpp b/barretenberg/cpp/src/barretenberg/ecc/curves/bn254/fr.hpp index 0c2f11910b94..96f445e57898 100644 --- a/barretenberg/cpp/src/barretenberg/ecc/curves/bn254/fr.hpp +++ b/barretenberg/cpp/src/barretenberg/ecc/curves/bn254/fr.hpp @@ -65,4 +65,10 @@ class Bn254FrParams { typedef field fr; -} // namespace barretenberg \ No newline at end of file +} // namespace barretenberg + +// define this as a named alias in msgpack schema generation +inline void msgpack_schema_pack(auto& packer, barretenberg::fr const&) +{ + packer.pack_alias("Fr", "bin32"); +} diff --git a/barretenberg/cpp/src/barretenberg/ecc/curves/bn254/fr.test.cpp b/barretenberg/cpp/src/barretenberg/ecc/curves/bn254/fr.test.cpp index 683b49171823..28cd331e1b1a 100644 --- a/barretenberg/cpp/src/barretenberg/ecc/curves/bn254/fr.test.cpp +++ b/barretenberg/cpp/src/barretenberg/ecc/curves/bn254/fr.test.cpp @@ -1,8 +1,15 @@ #include "fr.hpp" +#include "barretenberg/serialize/test_helper.hpp" #include using namespace barretenberg; +TEST(fr, msgpack) +{ + auto [actual, expected] = msgpack_roundtrip(barretenberg::fr{1ull, 2ull, 3ull, 4ull}); + EXPECT_EQ(actual, expected); +} + TEST(fr, eq) { fr a{ 0x01, 0x02, 0x03, 0x04 }; diff --git a/barretenberg/cpp/src/barretenberg/ecc/curves/bn254/g1.hpp b/barretenberg/cpp/src/barretenberg/ecc/curves/bn254/g1.hpp index dcabc0e3a9f0..65d0ffaa8440 100644 --- a/barretenberg/cpp/src/barretenberg/ecc/curves/bn254/g1.hpp +++ b/barretenberg/cpp/src/barretenberg/ecc/curves/bn254/g1.hpp @@ -17,4 +17,12 @@ struct Bn254G1Params { }; typedef group g1; -} // namespace barretenberg \ No newline at end of file + +} // namespace barretenberg + +// specialize the name in msgpack schema generation +// consumed by the typescript schema compiler, helps disambiguate templates +inline std::string msgpack_schema_name(barretenberg::g1::affine_element const&) +{ + return "G1AffineElement"; +} diff --git a/barretenberg/cpp/src/barretenberg/ecc/fields/field.hpp b/barretenberg/cpp/src/barretenberg/ecc/fields/field.hpp index 80a87f46aff1..05d201c4795b 100644 --- a/barretenberg/cpp/src/barretenberg/ecc/fields/field.hpp +++ b/barretenberg/cpp/src/barretenberg/ecc/fields/field.hpp @@ -406,6 +406,10 @@ template struct alignas(32) field { // BBERG_INLINE sstatic constexpr void butterfly(field& left, field& right) noexcept; + // For serialization + void msgpack_pack(auto& packer) const; + void msgpack_unpack(auto o); + private: static constexpr uint256_t twice_modulus = modulus + modulus; static constexpr uint256_t not_modulus = -modulus; @@ -543,7 +547,6 @@ template void read(B& it, field& value) read(it, result.data[0]); value = result.to_montgomery_form(); } - template void write(B& buf, field const& value) { using serialize::write; diff --git a/barretenberg/cpp/src/barretenberg/ecc/fields/field_impl.hpp b/barretenberg/cpp/src/barretenberg/ecc/fields/field_impl.hpp index 057a7e446367..a486aab45f0b 100644 --- a/barretenberg/cpp/src/barretenberg/ecc/fields/field_impl.hpp +++ b/barretenberg/cpp/src/barretenberg/ecc/fields/field_impl.hpp @@ -12,7 +12,6 @@ #endif #include "field_impl_generic.hpp" - namespace barretenberg { // template constexpr void field::butterfly(field& left, field& right) noexcept @@ -607,4 +606,45 @@ template constexpr field field::multiplicative_generator() noexc return target; } +// This function is used to serialize a field. It matches the old serialization format by first +// converting the field from Montgomery form, which is a special representation used for efficient +// modular arithmetic. +template void field::msgpack_pack(auto& packer) const +{ + // The field is first converted from Montgomery form, similar to how the old format did it. + auto adjusted = from_montgomery_form(); + + // The data is then converted to big endian format using htonll, which stands for "host to network long long". + // This is necessary because the data will be written to a raw msgpack buffer, which requires big endian format. + uint64_t bin_data[4] = { + htonll(adjusted.data[3]), htonll(adjusted.data[2]), htonll(adjusted.data[1]), htonll(adjusted.data[0]) + }; + + // The packer is then used to write the binary data to the buffer, just like in the old format. + packer.pack_bin(sizeof(bin_data)); + packer.pack_bin_body((const char*)bin_data, sizeof(bin_data)); +} + +// This function is used to deserialize a field. It also matches the old deserialization format by +// reading the binary data as big endian uint64_t's, correcting them to the host endianness, and +// then converting the field back to Montgomery form. +template void field::msgpack_unpack(auto o) +{ + // The binary data is first extracted from the msgpack object. + std::array raw_data = o; + + // The binary data is then read as big endian uint64_t's. This is done by casting the raw data to uint64_t* and then + // using ntohll ("network to host long long") to correct the endianness to the host's endianness. + uint64_t* cast_data = (uint64_t*)&raw_data[0]; + uint64_t reversed[] = {ntohll(cast_data[3]), ntohll(cast_data[2]), ntohll(cast_data[1]), ntohll(cast_data[0])}; + + // The corrected data is then copied back into the field's data array. + for (int i = 0; i < 4; i++) { + data[i] = reversed[i]; + } + + // Finally, the field is converted back to Montgomery form, just like in the old format. + *this = to_montgomery_form(); +} + } // namespace barretenberg diff --git a/barretenberg/cpp/src/barretenberg/ecc/groups/affine_element.hpp b/barretenberg/cpp/src/barretenberg/ecc/groups/affine_element.hpp index 8d85003a513e..312b01bfa3bc 100644 --- a/barretenberg/cpp/src/barretenberg/ecc/groups/affine_element.hpp +++ b/barretenberg/cpp/src/barretenberg/ecc/groups/affine_element.hpp @@ -3,6 +3,7 @@ #include #include #include "barretenberg/ecc/curves/bn254/fq2.hpp" +#include "barretenberg/serialize/msgpack.hpp" namespace barretenberg { namespace group_elements { @@ -176,6 +177,8 @@ template class alignas(64) affine_el } Fq x; Fq y; + // for serialization: update with new fields + MSGPACK_FIELDS(x, y); }; template void read(B& it, affine_element& value) diff --git a/barretenberg/cpp/src/barretenberg/ecc/groups/affine_element.test.cpp b/barretenberg/cpp/src/barretenberg/ecc/groups/affine_element.test.cpp index b7dcf57d4639..352cfe23511f 100644 --- a/barretenberg/cpp/src/barretenberg/ecc/groups/affine_element.test.cpp +++ b/barretenberg/cpp/src/barretenberg/ecc/groups/affine_element.test.cpp @@ -5,6 +5,7 @@ #include "barretenberg/common/test.hpp" #include #include "barretenberg/common/serialize.hpp" +#include "barretenberg/serialize/test_helper.hpp" namespace test_affine_element { template class test_affine_element : public testing::Test { @@ -122,4 +123,10 @@ TEST(affine_element, infinity_ordering_regression) P.self_set_infinity(); EXPECT_NE(P < Q, Q < P); } + +TEST(affine_element, msgpack) +{ + auto [actual, expected] = msgpack_roundtrip(secp256k1::g1::affine_element{ 1, 1 }); + EXPECT_EQ(actual, expected); +} } // namespace test_affine_element diff --git a/barretenberg/cpp/src/barretenberg/ecc/groups/element.hpp b/barretenberg/cpp/src/barretenberg/ecc/groups/element.hpp index 1b6096f5ceae..3f93f0f552a1 100644 --- a/barretenberg/cpp/src/barretenberg/ecc/groups/element.hpp +++ b/barretenberg/cpp/src/barretenberg/ecc/groups/element.hpp @@ -123,6 +123,8 @@ template class alignas(32) element { // } // return { x, y, Fq::one() }; // } + // for serialization: update with new fields + MSGPACK_FIELDS(x, y, z); static void conditional_negate_affine(const affine_element& in, affine_element& out, diff --git a/barretenberg/cpp/src/barretenberg/ecc/groups/element_impl.hpp b/barretenberg/cpp/src/barretenberg/ecc/groups/element_impl.hpp index c3a785953001..60f2faef2842 100644 --- a/barretenberg/cpp/src/barretenberg/ecc/groups/element_impl.hpp +++ b/barretenberg/cpp/src/barretenberg/ecc/groups/element_impl.hpp @@ -925,7 +925,6 @@ element element::random_coordinates_on_curve(numeric::rand Fq yy; Fq x; Fq y; - Fq t0; while (!found_one) { x = Fq::random_element(engine); yy = x.sqr() * x + T::b; @@ -935,8 +934,6 @@ element element::random_coordinates_on_curve(numeric::rand auto [found_root, y1] = yy.sqrt(); y = y1; found_one = found_root; - // t0 = y.sqr(); - // found_one = (yy == t0); } return { x, y, Fq::one() }; } diff --git a/barretenberg/cpp/src/barretenberg/ecc/serialize.test.cpp b/barretenberg/cpp/src/barretenberg/ecc/serialize.test.cpp new file mode 100644 index 000000000000..6a685d33054f --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/ecc/serialize.test.cpp @@ -0,0 +1,9 @@ +#include +#include "barretenberg/serialize/test_helper.hpp" +#include "barretenberg/ecc/fields/field.hpp" + +TEST(msgpack_tests, msgpack_field) +{ + auto [actual, expected] = msgpack_roundtrip(barretenberg::fr{1ull, 2ull, 3ull, 4ull}); + EXPECT_EQ(actual, expected); +} \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/honk/sumcheck/sumcheck.test.cpp b/barretenberg/cpp/src/barretenberg/honk/sumcheck/sumcheck.test.cpp index bae050619961..356ad662b9b8 100644 --- a/barretenberg/cpp/src/barretenberg/honk/sumcheck/sumcheck.test.cpp +++ b/barretenberg/cpp/src/barretenberg/honk/sumcheck/sumcheck.test.cpp @@ -96,7 +96,6 @@ TEST(Sumcheck, PolynomialNormalization) std::array id_3; std::array lagrange_first; std::array lagrange_last; - std::array pow_zeta; for (size_t i = 0; i < multivariate_n; i++) { w_l[i] = FF::random_element(); w_r[i] = FF::random_element(); diff --git a/barretenberg/cpp/src/barretenberg/plonk/proof_system/prover/c_bind.cpp b/barretenberg/cpp/src/barretenberg/plonk/proof_system/prover/c_bind.cpp index ef1d1c13923a..df6b1b03e787 100644 --- a/barretenberg/cpp/src/barretenberg/plonk/proof_system/prover/c_bind.cpp +++ b/barretenberg/cpp/src/barretenberg/plonk/proof_system/prover/c_bind.cpp @@ -1,13 +1,17 @@ #include "prover.hpp" #include "barretenberg/env/data_store.hpp" #include "barretenberg/env/crs.hpp" +#include "barretenberg/serialize/test_helper.hpp" +#include "barretenberg/serialize/raw_pointer.hpp" -#define WASM_EXPORT __attribute__((visibility("default"))) +#define WASM_EXPORT extern "C" __attribute__((visibility("default"))) using namespace barretenberg; -extern "C" { - +// TODO(AD): This __wasm__ guard is a hack, but these test functions are currently +// the only consumer of this API and it has no native definition. +// Eventually, remove this, and don't rely on asyncify with Charlie's work. +#ifdef __wasm__ /** * Called by `barretenberg_wasm.test.ts` to test the asyncify intrumentation and logic that * allows for WASM code to make calls to async code in JS. @@ -30,6 +34,8 @@ WASM_EXPORT void* test_async_func(size_t size, int val) return addr; } } +#endif + /** * @brief Simple wrapper for env_load_verifier_crs. * @return The CRS. @@ -50,6 +56,14 @@ WASM_EXPORT void* test_env_load_prover_crs(size_t num_points) using WasmProver = plonk::UltraProver; +typedef RawPointer WasmProverPtr; + +// TODO(AD): Currently just a motivating example, TODO bind rest of library +CBIND(prover_process_queue2, [](WasmProverPtr prover) { + prover->queue.process_queue(); + return 0; +}); + WASM_EXPORT void prover_process_queue(WasmProver* prover) { prover->queue.process_queue(); @@ -168,4 +182,3 @@ WASM_EXPORT void delete_evaluation_domain(void* domain) { delete reinterpret_cast(domain); } -} diff --git a/barretenberg/cpp/src/barretenberg/plonk/proof_system/types/polynomial_manifest.hpp b/barretenberg/cpp/src/barretenberg/plonk/proof_system/types/polynomial_manifest.hpp index 8a56d34168a0..c16aeaafdf0a 100644 --- a/barretenberg/cpp/src/barretenberg/plonk/proof_system/types/polynomial_manifest.hpp +++ b/barretenberg/cpp/src/barretenberg/plonk/proof_system/types/polynomial_manifest.hpp @@ -255,4 +255,4 @@ class PrecomputedPolyList { std::string operator[](size_t index) const { return precomputed_poly_ids[index]; } }; -} // namespace proof_system::plonk +} // namespace proof_system::plonk \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/plonk/proof_system/types/proof.hpp b/barretenberg/cpp/src/barretenberg/plonk/proof_system/types/proof.hpp index d0a65aa2b68d..389bf211e2e0 100644 --- a/barretenberg/cpp/src/barretenberg/plonk/proof_system/types/proof.hpp +++ b/barretenberg/cpp/src/barretenberg/plonk/proof_system/types/proof.hpp @@ -4,11 +4,14 @@ #include #include #include "barretenberg/common/serialize.hpp" +#include "barretenberg/serialize/msgpack.hpp" namespace proof_system::plonk { struct proof { std::vector proof_data; + void msgpack_pack(auto& packer) const { packer.pack(proof_data); } + void msgpack_unpack(auto object) { proof_data = (std::vector)object; } bool operator==(proof const& other) const = default; }; @@ -42,3 +45,9 @@ inline std::ostream& operator<<(std::ostream& os, proof const& data) } } // namespace proof_system::plonk + +// help our msgpack schema compiler with this typedef +inline void msgpack_schema_pack(auto& packer, proof_system::plonk::proof const&) +{ + packer.pack_alias("Proof", "bin32"); +} diff --git a/barretenberg/cpp/src/barretenberg/plonk/proof_system/verification_key/verification_key.hpp b/barretenberg/cpp/src/barretenberg/plonk/proof_system/verification_key/verification_key.hpp index 709eea6ee786..0ee17d60e690 100644 --- a/barretenberg/cpp/src/barretenberg/plonk/proof_system/verification_key/verification_key.hpp +++ b/barretenberg/cpp/src/barretenberg/plonk/proof_system/verification_key/verification_key.hpp @@ -7,6 +7,7 @@ #include "barretenberg/polynomials/evaluation_domain.hpp" #include "barretenberg/crypto/sha256/sha256.hpp" #include "barretenberg/plonk/proof_system/types/polynomial_manifest.hpp" +#include "barretenberg/serialize/msgpack.hpp" namespace proof_system::plonk { @@ -18,6 +19,13 @@ struct verification_key_data { bool contains_recursive_proof = false; std::vector recursive_proof_public_input_indices; + // for serialization: update with any new fields + MSGPACK_FIELDS(composer_type, + circuit_size, + num_public_inputs, + commitments, + contains_recursive_proof, + recursive_proof_public_input_indices); barretenberg::fr compress_native(size_t const hash_index = 0); }; @@ -50,6 +58,8 @@ inline bool operator==(verification_key_data const& lhs, verification_key_data c } struct verification_key { + // default constructor needed for msgpack unpack + verification_key() = default; verification_key(verification_key_data&& data, std::shared_ptr const& crs); verification_key(const size_t num_gates, const size_t num_inputs, @@ -83,8 +93,32 @@ struct verification_key { bool contains_recursive_proof = false; std::vector recursive_proof_public_input_indices; size_t program_width = 3; + + // for serialization: update with new fields + void msgpack_pack(auto& packer) const + { + verification_key_data data = { composer_type, + static_cast(circuit_size), + static_cast(num_public_inputs), + commitments, + contains_recursive_proof, + recursive_proof_public_input_indices }; + packer.pack(data); + } + void msgpack_unpack(auto obj) + { + verification_key_data data = obj; + auto env_crs = std::make_unique(); + *this = verification_key{ std::move(data), env_crs->get_verifier_crs() }; + } }; +// specialize schema serialization +inline void msgpack_schema(auto& packer, proof_system::plonk::verification_key const&) +{ + packer.pack_schema(proof_system::plonk::verification_key_data{}); +} + template inline void read(B& buf, verification_key& key) { auto env_crs = std::make_unique(); @@ -125,3 +159,10 @@ inline std::ostream& operator<<(std::ostream& os, verification_key const& key) }; } // namespace proof_system::plonk + +// help our msgpack schema compiler with this struct +// Alias verification_key as verification_key_data +inline void msgpack_schema_pack(auto& packer, proof_system::plonk::verification_key const&) +{ + msgpack_schema_pack(packer, proof_system::plonk::verification_key_data{}); +} \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/plonk/proof_system/widgets/transition_widgets/turbo_arithmetic_widget.hpp b/barretenberg/cpp/src/barretenberg/plonk/proof_system/widgets/transition_widgets/turbo_arithmetic_widget.hpp index 9daaba1f0c72..5b46fb8b8091 100644 --- a/barretenberg/cpp/src/barretenberg/plonk/proof_system/widgets/transition_widgets/turbo_arithmetic_widget.hpp +++ b/barretenberg/cpp/src/barretenberg/plonk/proof_system/widgets/transition_widgets/turbo_arithmetic_widget.hpp @@ -156,7 +156,6 @@ template class TurboArithme Field T3; Field T4; Field T5; - Field T6; /** * Quad extraction term. This term is only active when q_arith is set to 2. diff --git a/barretenberg/cpp/src/barretenberg/proof_system/circuit_constructors/turbo_circuit_constructor.cpp b/barretenberg/cpp/src/barretenberg/proof_system/circuit_constructors/turbo_circuit_constructor.cpp index 517587f750f8..8bc4b461e2ad 100644 --- a/barretenberg/cpp/src/barretenberg/proof_system/circuit_constructors/turbo_circuit_constructor.cpp +++ b/barretenberg/cpp/src/barretenberg/proof_system/circuit_constructors/turbo_circuit_constructor.cpp @@ -929,14 +929,9 @@ inline bool TurboCircuitConstructor::lazy_logic_gate_check(const size_t gate_ind constexpr fr three(3); constexpr fr minus_one = -fr::one(); - fr delta_sum; - fr delta_squared_sum; fr T0; fr T1; fr T2; - fr T3; - fr T4; - fr identity; // T0 = a T0 = wire_1_value + wire_1_value; @@ -1448,7 +1443,7 @@ inline fr TurboCircuitConstructor::fixed_base_gate_evaluation(const size_t gate_ * */ bool TurboCircuitConstructor::check_circuit() { -//#define LAZY_CIRCUIT_CHECKS +// #define LAZY_CIRCUIT_CHECKS #ifdef LAZY_CIRCUIT_CHECKS for (size_t i = 0; i < num_gates; i++) { if (!q_arith[i].is_zero() && !lazy_arithmetic_gate_check(i)) { diff --git a/barretenberg/cpp/src/barretenberg/serialize/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/serialize/CMakeLists.txt new file mode 100644 index 000000000000..f7c49f1111ec --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/serialize/CMakeLists.txt @@ -0,0 +1,7 @@ +# For running tests only, not to be depended on +# The non-test portion of barretenberg (the part you actually use) is header only +barretenberg_module( + serialize-tests + proof_system + transcript +) \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/serialize/cbind.hpp b/barretenberg/cpp/src/barretenberg/serialize/cbind.hpp new file mode 100644 index 000000000000..a04761527007 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/serialize/cbind.hpp @@ -0,0 +1,102 @@ +#pragma once +// Meant to be the main header included by translation units that use msgpack. +// Note: heavy header due to serialization logic, don't include if msgpack.hpp will do +// CBinding helpers that take a function or a lambda and +// - bind the input as a coded msgpack array of all the arguments (using template metamagic) +// - bind the return value to an out buffer, where the caller must free the memory + +#include "msgpack_impl/check_memory_span.hpp" +#include "msgpack_impl/concepts.hpp" +#include "msgpack_impl/msgpack_impl.hpp" +#include "msgpack_impl/name_value_pair_macro.hpp" +#include "msgpack_impl/schema_impl.hpp" +#include "msgpack_impl/schema_name.hpp" +#include "msgpack_impl/struct_map_impl.hpp" +#include "msgpack_impl/variant_impl.hpp" +#include "msgpack_impl/func_traits.hpp" + +#include +#include + +/** + * Represents this as a bbmalloc'ed object, fit for sending to e.g. TypeScript. + * @param obj The object. + * @return The buffer pointer/size pair. + */ +inline std::pair msgpack_encode_buffer(auto&& obj) +{ + // Create a buffer to store the encoded data + msgpack::sbuffer buffer; + msgpack::pack(buffer, obj); + + uint8_t* output = (uint8_t*)aligned_alloc(64, buffer.size()); + memcpy(output, buffer.data(), buffer.size()); + // Convert the buffer data to a string and return it + return { output, buffer.size() }; +} + +// This is a template function that will return the argument types +// of a given function type T as a tuple. +template constexpr auto param_tuple() +{ + // decltype is used to determine the type of an expression at compile-time. + // get_func_traits() is assumed to return a structure whose ::Args member is a tuple + // of argument types of function T. This function constructs an instance of that tuple and returns it. + return typename decltype(get_func_traits())::Args{}; +} + +// This function is intended to bind a function to a MessagePack-formatted input data, +// perform the function with the unpacked data, then pack the result back into MessagePack format. +inline void msgpack_cbind_impl( + auto func, // The function to be applied + const uint8_t* input_in, // The input data in MessagePack format + size_t input_len_in, // The length of the input data + uint8_t** output_out, // The output data in MessagePack format + size_t* output_len_out) // The length of the output data +{ + // Get the parameter types of the function as a tuple. + auto params = param_tuple(); + + // Unpack the input data into the parameter tuple. + // msgpack::unpack takes a buffer and its size, and returns an object_handle. + // Calling .get() on that handle yields an object, and calling .convert on that + // object converts it into the given type, in this case, the parameter tuple for func. + msgpack::unpack((const char*)input_in, input_len_in).get().convert(params); + + // Apply the function to the parameters, then encode the result into a MessagePack buffer. + // std::apply takes a function and a tuple, and applies the function to the tuple's elements. + // msgpack_encode_buffer is assumed to take the result of the function and return a pair + // consisting of a pointer to the output buffer and its size. + auto [output, output_len] = msgpack_encode_buffer(std::apply(func, params)); + + // Assign the output data and its length to the given output parameters. + *output_out = output; + *output_len_out = output_len; +} + +// returns a C-style string json of the schema +inline void msgpack_cbind_schema_impl(auto func, uint8_t** output_out, size_t* output_len_out) +{ + (void)func; // unused except for type + // Object representation of the cbind + auto cbind_obj = get_func_traits(); + std::string schema = msgpack_schema_to_string(cbind_obj); + *output_out = (uint8_t*)aligned_alloc(64, schema.size() + 1); + memcpy(*output_out, schema.c_str(), schema.size() + 1); + *output_len_out = schema.size(); +} + +// The CBIND macro is a convenient utility that abstracts away several steps in binding C functions with msgpack +// serialization. It creates two separate functions: +// 1. cname function: This decodes the input arguments from msgpack format, calls the target function, +// and then encodes the return value back into msgpack format. +// 2. cname##__schema function: This creates a JSON schema of the function's input arguments and return type. +#define CBIND(cname, func) \ + WASM_EXPORT void cname(const uint8_t* input_in, size_t input_len_in, uint8_t** output_out, size_t* output_len_out) \ + { \ + msgpack_cbind_impl(func, input_in, input_len_in, output_out, output_len_out); \ + } \ + WASM_EXPORT void cname##__schema(uint8_t** output_out, size_t* output_len_out) \ + { \ + msgpack_cbind_schema_impl(func, output_out, output_len_out); \ + } diff --git a/barretenberg/cpp/src/barretenberg/serialize/cbind_fwd.hpp b/barretenberg/cpp/src/barretenberg/serialize/cbind_fwd.hpp new file mode 100644 index 000000000000..ef5eee9204ad --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/serialize/cbind_fwd.hpp @@ -0,0 +1,11 @@ +#pragma once +#include +#include +// CBIND forward declarations for msgback default bind format (encode as tuple of args and return value as msgpack +// string) +#define WASM_EXPORT extern "C" __attribute__((visibility("default"))) + +#define CBIND_DECL(cname) \ + WASM_EXPORT void cname( \ + const uint8_t* input_in, size_t input_len_in, uint8_t** output_out, size_t* output_len_out); \ + WASM_EXPORT void cname##__schema(uint8_t** output_out, size_t* output_len_out); diff --git a/barretenberg/cpp/src/barretenberg/serialize/msgpack.hpp b/barretenberg/cpp/src/barretenberg/serialize/msgpack.hpp new file mode 100644 index 000000000000..767a2401e29c --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/serialize/msgpack.hpp @@ -0,0 +1,105 @@ +#pragma once +/* Minimal header for declaring msgpack fields. +This should be included as "barretenberg/serialize/msgpack.hpp" unless a translation wants +to use msgpack for bindings, then "barretenberg/serialize/cbind.hpp" should be included. + +## Overview + +The Msgpack module allows for efficient serialization and deserialization of data structures. It can be applied to +map-like objects, array-like objects, and custom serialization/deserialization logic. + +## Binding objects + +Marking structs/classes with their fields for msgpack allows you to pack and unpack the class. + +1. All objects bound should have a default constructor +2. Objects can be tightly packed as binary (see field_impl.hpp), array-like, or map-like. See below +3. You should list all fields of a class in the below methods, or use the custom method. + +### Typical Objects + +To make objects serializable as a map-like format, define the `msgpack` method in your class as follows: + +```cpp +void msgpack(auto ar) { + ar(NVP(composer_type, circuit_size, num_public_inputs, commitments, contains_recursive_proof, +recursive_proof_public_input_indices)); +} +or +MSGPACK_FIELDS(composer_type, circuit_size, num_public_inputs, commitments, contains_recursive_proof, +recursive_proof_public_input_indices); +``` + +This approach assumes 1. all members are default constructible 2. you give it all members 3. all members are writable +references + +This method maps the object's properties (e.g., `composer_type`, `circuit_size`, etc.) to their respective keys in the +serialized data. + + +### Custom Serialization and Deserialization + +For custom serialization and deserialization, define `msgpack_pack` and `msgpack_unpack` methods in your class: + +```cpp +// For serialization +template void field::msgpack_pack(auto& packer) const +{ + auto adjusted = from_montgomery_form(); + uint64_t bin_data[4] = { + htonll(adjusted.data[3]), htonll(adjusted.data[2]), htonll(adjusted.data[1]), htonll(adjusted.data[0]) + }; + packer.pack_bin(sizeof(bin_data)); + packer.pack_bin_body((const char*)bin_data, sizeof(bin_data)); +} + +// For deserialization +template void field::msgpack_unpack(auto o) +{ + msgpack::read_bin64(o, data, 4); + uint64_t reversed[] = {data[3], data[2], data[1], data[0]}; + for (int i = 0; i < 4; i++) { + data[i] = reversed[i]; + } + *this = to_montgomery_form(); +} +``` + +These methods allow you to implement custom logic for the serialization and deserialization processes. + + +## Packing/Unpacking + +Only when actually using msgpack to write or read data, include "barretenberg/serialize/cbind.hpp". +You can then use msgpack library features to serialize and deserialize C++ objects. + +e.g. packing +``` + // Create a buffer to store the encoded data + msgpack::sbuffer buffer; + msgpack::pack(buffer, obj); + + uint8_t* output = (uint8_t*)aligned_alloc(64, buffer.size()); + memcpy(output, buffer.data(), buffer.size()); + // Convert the buffer data to a string and return it + return { output, buffer.size() }; +``` + +e.g. unpacking + +``` + msgpack::unpack((const char*)encoded_data, encoded_data_size).get().convert(*value); +``` +*/ +#include +#include "msgpack_impl/name_value_pair_macro.hpp" +#include "msgpack_impl/concepts.hpp" + +// Helper for above documented syntax +// Define a macro that takes any amount of parameters and expands to a msgpack method definition +// __VA__ARGS__ expands to the parmeters, comma separated. +#define MSGPACK_FIELDS(...) \ + void msgpack(auto pack_fn) \ + { \ + pack_fn(NVP(__VA_ARGS__)); \ + } diff --git a/barretenberg/cpp/src/barretenberg/serialize/msgpack_impl/check_memory_span.hpp b/barretenberg/cpp/src/barretenberg/serialize/msgpack_impl/check_memory_span.hpp new file mode 100644 index 000000000000..7a310f7cf3d5 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/serialize/msgpack_impl/check_memory_span.hpp @@ -0,0 +1,76 @@ +#pragma once +// Note: heavy header due to serialization logic, don't include outside of tests +#include + +#include +#include +#include +#include +#include +#include "barretenberg/common/throw_or_abort.hpp" +#include "schema_name.hpp" + +namespace msgpack { +template uintptr_t __aligned_for(uintptr_t ptr) +{ + // Round to next alignment, (ptr % alignof(T)) == 0 after + return ptr + (alignof(T) - (ptr % alignof(T))) % alignof(T); +} +template std::string check_memory_span(T* obj, Args*... args) +{ + // We need to handle alignment. Thankfully, we have a tool here. + // Convert the variadic template arguments to a vector of pairs. + // Each pair contains a pointer (as uintptr_t) and its size. + std::vector> pointers{ { (uintptr_t)(args), sizeof(Args) }... }; + // Sort the vector based on the pointer values. + std::sort(pointers.begin(), pointers.end(), [](const auto& a, const auto& b) { return a.first < b.first; }); + + for (size_t i = 1; i < pointers.size(); ++i) { + // Check if any of the Args* pointers overlap. + auto last_end = pointers[i - 1].first + pointers[i - 1].second; + if (last_end > pointers[i].first) { + return "Overlap in " + msgpack_schema_name(*obj) + " MSGPACK_FIELDS() params detected!"; + } + // Check if gap is too large. + // Give some fuzzy room in case of 64 byte alignment restrictions. + if (__aligned_for(last_end) < pointers[i].first) { + return "Gap in " + msgpack_schema_name(*obj) + " MSGPACK_FIELDS() params detected before member #" + + std::to_string(i) + " !"; + } + } + + // Check if all Args* pointers exist in T* memory. + uintptr_t t_start = reinterpret_cast(obj); + uintptr_t t_end = t_start + sizeof(T); + if (pointers.front().first < t_start || pointers.back().first + pointers.back().second > t_end) { + return "Some " + msgpack_schema_name(*obj) + " MSGPACK_FIELDS() params don't exist in object!"; + } + + // Check if all of T* memory is used by the Args* pointers. + size_t start = (size_t)obj; + size_t end = (size_t)obj; + for (auto [ptr, size] : pointers) { + end = std::max(end, ptr + size); + } + size_t total_size = end - start; + if (__aligned_for(total_size) < sizeof(T)) { + return "Incomplete " + msgpack_schema_name(*obj) + " MSGPACK_FIELDS() params! Not all of object specified."; + } + return {}; +} + +template std::string check_msgpack_method(T& object) +{ + std::string result; + auto checker = [&](auto&... values) { result = check_memory_span(&object, &values...); }; + object.msgpack([&](auto&... keys_and_values) { std::apply(checker, drop_keys(std::tie(keys_and_values...))); }); + return result; +} +void check_msgpack_usage(auto object) +{ + std::string result = check_msgpack_method(object); + if (!result.empty()) { + throw_or_abort(result); + } +} +} // namespace msgpack \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/serialize/msgpack_impl/concepts.hpp b/barretenberg/cpp/src/barretenberg/serialize/msgpack_impl/concepts.hpp new file mode 100644 index 000000000000..07579bdf24ba --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/serialize/msgpack_impl/concepts.hpp @@ -0,0 +1,11 @@ +#pragma once + +struct DoNothing { + void operator()(auto...) {} +}; +namespace msgpack_concepts { +template +concept HasMsgPack = requires(T t, DoNothing nop) { t.msgpack(nop); }; +template +concept HasMsgPackPack = requires(T t, DoNothing nop) { t.msgpack_pack(nop); }; +} // namespace msgpack_concepts \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/serialize/msgpack_impl/func_traits.hpp b/barretenberg/cpp/src/barretenberg/serialize/msgpack_impl/func_traits.hpp new file mode 100644 index 000000000000..a514ebbba1f4 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/serialize/msgpack_impl/func_traits.hpp @@ -0,0 +1,58 @@ +#pragma once +#include +#include "../msgpack.hpp" + +// Define a template struct to deduce function traits for different function types +template +struct func_traits; + +// Specialization for function pointers +template +struct func_traits { + typedef std::tuple Args; // Define a tuple type that holds all argument types + Args args; // Args instance + R ret; // Holds return type + MSGPACK_FIELDS(args, ret); // Macro from msgpack library to serialize/deserialize fields +}; + +// Specialization for function references +template +struct func_traits { + typedef std::tuple Args; + Args args; + R ret; + MSGPACK_FIELDS(args, ret); +}; + +// Specialization for member function pointers. This also includes lambda types, +// as they are functors (objects with operator()) and hence have a member function pointer +template +struct func_traits { + typedef std::tuple Args; + Args args; + R ret; + MSGPACK_FIELDS(args, ret); +}; + +// Define a concept that checks if the type is a lambda (or functor) type +// This is done by checking if T::operator() exists +template +concept LambdaType = requires() { + typename std::enable_if_t, void>; +}; + +// Overload for lambda (or functor) types +template +constexpr auto get_func_traits() +{ + // If T is a lambda type (i.e. it has operator()), deduce its traits using func_traits + return func_traits(); +} + +// Overload for non-lambda types +template +constexpr auto get_func_traits() +{ + // If T is not a lambda, just deduce its traits using func_traits + return func_traits(); +} diff --git a/barretenberg/cpp/src/barretenberg/serialize/msgpack_impl/msgpack_impl.hpp b/barretenberg/cpp/src/barretenberg/serialize/msgpack_impl/msgpack_impl.hpp new file mode 100644 index 000000000000..1db14a898adc --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/serialize/msgpack_impl/msgpack_impl.hpp @@ -0,0 +1,4 @@ +#pragma once +// Note: Meant to only be included in compilation units that need msgpack +#include "struct_map_impl.hpp" +#include "variant_impl.hpp" \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/serialize/msgpack_impl/name_value_pair_macro.hpp b/barretenberg/cpp/src/barretenberg/serialize/msgpack_impl/name_value_pair_macro.hpp new file mode 100644 index 000000000000..1f3ccc347423 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/serialize/msgpack_impl/name_value_pair_macro.hpp @@ -0,0 +1,40 @@ +#pragma once + +/* start of #define NVP: + * expands to name-value pairs (NVPs), e.g. NVP(x,y,z) -> "x", x, "y", y", "z", z + * used in msgpack serialization. */ + +// hacky counting of variadic macro params: +#define VA_NARGS_IMPL(_1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, _15, _16, _17, _18, _19, _20, N, ...) N +// AD: support for 20 fields!? one may ask. Well, after 15 not being enough... +#define VA_NARGS(...) VA_NARGS_IMPL(__VA_ARGS__, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1) + +// name-value pair expansion for variables +// used in msgpack map expansion +// n<=3 case +#define _NVP1(x) #x, x +#define _NVP2(x, y) #x, x, #y, y +#define _NVP3(x, y, z) #x, x, #y, y, #z, z +// n>3 cases +#define _NVP4(x, ...) _NVP1(x), _NVP3(__VA_ARGS__) +#define _NVP5(x, ...) _NVP1(x), _NVP4(__VA_ARGS__) +#define _NVP6(x, ...) _NVP1(x), _NVP5(__VA_ARGS__) +#define _NVP7(x, ...) _NVP1(x), _NVP6(__VA_ARGS__) +#define _NVP8(x, ...) _NVP1(x), _NVP7(__VA_ARGS__) +#define _NVP9(x, ...) _NVP1(x), _NVP8(__VA_ARGS__) +#define _NVP10(x, ...) _NVP1(x), _NVP9(__VA_ARGS__) +#define _NVP11(x, ...) _NVP1(x), _NVP10(__VA_ARGS__) +#define _NVP12(x, ...) _NVP1(x), _NVP11(__VA_ARGS__) +#define _NVP13(x, ...) _NVP1(x), _NVP12(__VA_ARGS__) +#define _NVP14(x, ...) _NVP1(x), _NVP13(__VA_ARGS__) +#define _NVP15(x, ...) _NVP1(x), _NVP14(__VA_ARGS__) +#define _NVP16(x, ...) _NVP1(x), _NVP15(__VA_ARGS__) +#define _NVP17(x, ...) _NVP1(x), _NVP16(__VA_ARGS__) +#define _NVP18(x, ...) _NVP1(x), _NVP17(__VA_ARGS__) +#define _NVP19(x, ...) _NVP1(x), _NVP18(__VA_ARGS__) +#define _NVP20(x, ...) _NVP1(x), _NVP19(__VA_ARGS__) + +#define CONCAT(a, b) a ## b +#define _NVP_N(n) CONCAT(_NVP, n) +#define NVP(...) _NVP_N(VA_NARGS(__VA_ARGS__))(__VA_ARGS__) +// end of #define NVP \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/serialize/msgpack_impl/schema_impl.hpp b/barretenberg/cpp/src/barretenberg/serialize/msgpack_impl/schema_impl.hpp new file mode 100644 index 000000000000..a99038a1b3b8 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/serialize/msgpack_impl/schema_impl.hpp @@ -0,0 +1,190 @@ +#pragma once + +#include +#include +#include +#include +#include "schema_name.hpp" + +/** + * Define a serialization schema based on compile-time information about a type being serialized. + * This is then consumed by typescript to make bindings. + */ +struct MsgpackSchemaPacker : msgpack::packer { + MsgpackSchemaPacker(msgpack::sbuffer& stream) + : packer(stream) + {} + // For tracking emitted types + std::set emitted_types; + // Returns if already was emitted + bool set_emitted(const std::string& type) + { + if (emitted_types.find(type) == emitted_types.end()) { + emitted_types.insert(type); + return false; + } + return true; + } + + /** + * Pack a type indicating it is an alias of a certain msgpack type + * Packs in the form ["alias", [schema_name, msgpack_name]] + * @param schema_name The CPP type. + * @param msgpack_name The msgpack type. + */ + void pack_alias(const std::string& schema_name, const std::string& msgpack_name) + { + // We will pack a size 2 tuple + pack_array(2); + pack("alias"); + // That has a size 2 tuple as its 2nd arg + pack_array(2); + pack(schema_name); + pack(msgpack_name); + } + + /** + * Pack the schema of a given object. + * @tparam T the object's type. + * @param obj the object. + */ + template void pack_schema(const T& obj) { msgpack_schema_pack(*this, obj); } +}; + +// Helper for packing (key, value, key, value, ...) arguments +inline void _schema_pack_map_content(MsgpackSchemaPacker&) +{ + // base case +} + +namespace msgpack_concepts { +template +concept SchemaPackable = requires(T value, MsgpackSchemaPacker packer) { msgpack_schema_pack(packer, value); }; +} + +// Helper for packing (key, value, key, value, ...) arguments +template +inline void _schema_pack_map_content(MsgpackSchemaPacker& packer, std::string key, Value value, Rest... rest) +{ + static_assert( + msgpack_concepts::SchemaPackable, + "see the first type argument in the error trace, it might require a specialization of msgpack_schema_pack"); + packer.pack(key); + msgpack_schema_pack(packer, value); + _schema_pack_map_content(packer, rest...); +} + +/** + * Schema pack base case for types with no special msgpack method. + * @tparam T the type. + * @param packer the schema packer. + */ +template + requires(!msgpack_concepts::HasMsgPack && !msgpack_concepts::HasMsgPackPack) +inline void msgpack_schema_pack(MsgpackSchemaPacker& packer, T const&) +{ + packer.pack(msgpack_schema_name(T{})); +} + +/** + * @brief Encode a type that defines msgpack based on its key value pairs. + * + * @tparam T the msgpack()'able type + * @param packer Our special packer. + * @param object The object in question. + */ +template inline void msgpack_schema_pack(MsgpackSchemaPacker& packer, T const& object) +{ + std::string type = msgpack_schema_name(object); + if (packer.set_emitted(type)) { + packer.pack(type); + return; // already emitted + } + msgpack::check_msgpack_usage(object); + // Encode as map + const_cast(object).msgpack([&](auto&... args) { + size_t kv_size = sizeof...(args); + // Calculate the number of entries in our map (half the size of keys + values, plus the typename) + packer.pack_map(uint32_t(1 + kv_size / 2)); + packer.pack("__typename"); + packer.pack(type); + // Pack the map content based on the args to msgpack + _schema_pack_map_content(packer, args...); + }); +} + +// Recurse over any templated containers +// Outputs e.g. ['vector', ['sub-type']] +template +inline void _msgpack_schema_pack(MsgpackSchemaPacker& packer, const std::string& schema_name) +{ + // We will pack a size 2 tuple + packer.pack_array(2); + packer.pack(schema_name); + packer.pack_array(sizeof...(Args)); + // helper for better errors + auto pack = [&](auto arg) { + static_assert(msgpack_concepts::SchemaPackable, + "see the type argument of this lambda in the error trace, it might require a specialization of " + "msgpack_schema_pack"); + msgpack_schema_pack(packer, arg); + }; + + // Note: if this fails to compile, check first in list of template Arg's + // it may need a msgpack_schema_pack specialization (particularly if it doesn't define MSGPACK_FIELDS). + (pack(Args{}), ...); /* pack schemas of all template Args */ +} +template inline void msgpack_schema_pack(MsgpackSchemaPacker& packer, std::tuple const&) +{ + _msgpack_schema_pack(packer, "tuple"); +} +template inline void msgpack_schema_pack(MsgpackSchemaPacker& packer, std::map const&) +{ + _msgpack_schema_pack(packer, "map"); +} +template inline void msgpack_schema_pack(MsgpackSchemaPacker& packer, std::optional const&) +{ + _msgpack_schema_pack(packer, "optional"); +} +template inline void msgpack_schema_pack(MsgpackSchemaPacker& packer, std::vector const&) +{ + _msgpack_schema_pack(packer, "vector"); +} +template inline void msgpack_schema_pack(MsgpackSchemaPacker& packer, std::variant const&) +{ + _msgpack_schema_pack(packer, "variant"); +} +template inline void msgpack_schema_pack(MsgpackSchemaPacker& packer, std::shared_ptr const&) +{ + _msgpack_schema_pack(packer, "shared_ptr"); +} + +// Outputs e.g. ['array', ['array-type', 'N']] +template +inline void msgpack_schema_pack(MsgpackSchemaPacker& packer, std::array const&) +{ + // We will pack a size 2 tuple + packer.pack_array(2); + packer.pack("array"); + // That has a size 2 tuple as its 2nd arg + packer.pack_array(2); /* param list format for consistency*/ + msgpack_schema_pack(packer, T{}); + packer.pack(N); +} + +/** + * @brief Print's an object's derived msgpack schema as a string. + * + * @param obj The object to print schema of. + * @return std::string The schema as a string. + */ +inline std::string msgpack_schema_to_string(auto obj) +{ + msgpack::sbuffer output; + MsgpackSchemaPacker printer{ output }; + msgpack_schema_pack(printer, obj); + msgpack::object_handle oh = msgpack::unpack(output.data(), output.size()); + std::stringstream pretty_output; + pretty_output << oh.get() << std::endl; + return pretty_output.str(); +} diff --git a/barretenberg/cpp/src/barretenberg/serialize/msgpack_impl/schema_name.hpp b/barretenberg/cpp/src/barretenberg/serialize/msgpack_impl/schema_name.hpp new file mode 100644 index 000000000000..c70c8f2a5bce --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/serialize/msgpack_impl/schema_name.hpp @@ -0,0 +1,31 @@ +#pragma once +#include "barretenberg/ecc/curves/bn254/g1.hpp" +#include +#include + +/** + * Converts C++ type information into a human-readable format. + * This function leverages __cxa_demangle to demangle the name generated by typeid. + * Special cases are handled for "basic_string" (returns "string") and "i" (returns "int"). + * Template specializations are truncated and only the base type name is returned + * @tparam T the type. + * @return the readable schema name. + */ +template std::string msgpack_schema_name(T const&) +{ + std::string result = abi::__cxa_demangle(typeid(T).name(), NULL, NULL, NULL); + if (result.find("basic_string") != std::string::npos) { + return "string"; + } + if (result == "i") { + return "int"; + } + + if (result.find('<') != size_t(-1)) { + result = result.substr(0, result.find('<')); + } + if (result.rfind(':') != size_t(-1)) { + result = result.substr(result.rfind(':') + 1, result.size()); + } + return result; +} diff --git a/barretenberg/cpp/src/barretenberg/serialize/msgpack_impl/struct_map_impl.hpp b/barretenberg/cpp/src/barretenberg/serialize/msgpack_impl/struct_map_impl.hpp new file mode 100644 index 000000000000..4bdf4909c9c5 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/serialize/msgpack_impl/struct_map_impl.hpp @@ -0,0 +1,76 @@ +#pragma once +// Note: heavy header due to serialization logic, don't include if msgpack.hpp will do +#include +#include +#include +#include +#include +#define MSGPACK_NO_BOOST +#include +#include "concepts.hpp" + +namespace msgpack { +template auto drop_keys_impl(Tuple&& tuple, std::index_sequence) +{ + // Expand 0 to n/2 to 1 to n+1 (increments of 2) + // Use it to filter the tuple + return std::tie(std::get(std::forward(tuple))...); +} + +template auto drop_keys(std::tuple&& tuple) +{ + static_assert(sizeof...(Args) % 2 == 0, "Tuple must contain an even number of elements"); + // Compile time sequence of integers from 0 to n/2 + auto compile_time_0_to_n_div_2 = std::make_index_sequence{}; + return drop_keys_impl(tuple, compile_time_0_to_n_div_2); +} +} // namespace msgpack + +namespace msgpack { +template +concept MsgpackConstructible = requires(T object, Args... args) { T{ args... }; }; +} + +namespace msgpack::adaptor { +// reads structs with msgpack() method from a JSON-like dictionary +template struct convert { + msgpack::object const& operator()(msgpack::object const& o, T& v) const + { + static_assert(std::is_default_constructible_v, + "MSGPACK_FIELDS requires default-constructible types (used during unpacking)"); + v.msgpack([&](auto&... args) { + auto static_checker = [&](auto&... value_args) { + static_assert(msgpack::MsgpackConstructible, + "MSGPACK_FIELDS requires a constructor that can take the types listed in MSGPACK_FIELDS. " + "Type or arg count mismatch, or member initializer constructor not available."); + }; + std::apply(static_checker, drop_keys(std::tie(args...))); + msgpack::type::define_map{ args... }.msgpack_unpack(o); + }); + return o; + } +}; + +// converts structs with msgpack() method from a JSON-like dictionary +template struct pack { + template packer& operator()(msgpack::packer& o, T const& v) const + { + static_assert(std::is_default_constructible_v, + "MSGPACK_FIELDS requires default-constructible types (used during unpacking)"); + const_cast(v).msgpack([&](auto&... args) { + auto static_checker = [&](auto&... value_args) { + static_assert(msgpack::MsgpackConstructible, + "T requires a constructor that can take the fields listed in MSGPACK_FIELDS (T will be " + "in template parameters in the compiler stack trace)" + "Check the MSGPACK_FIELDS macro usage in T for incompleteness or wrong order." + "Alternatively, a matching member initializer constructor might not be available for T " + "and should be defined."); + }; + std::apply(static_checker, drop_keys(std::tie(args...))); + msgpack::type::define_map{ args... }.msgpack_pack(o); + }); + return o; + } +}; + +} // namespace msgpack::adaptor diff --git a/barretenberg/cpp/src/barretenberg/serialize/msgpack_impl/variant_impl.hpp b/barretenberg/cpp/src/barretenberg/serialize/msgpack_impl/variant_impl.hpp new file mode 100644 index 000000000000..8241ba47abb8 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/serialize/msgpack_impl/variant_impl.hpp @@ -0,0 +1,16 @@ +#pragma once +// Note: Meant to only be included in compilation units that need msgpack +#define MSGPACK_NO_BOOST +#include +#include + +namespace msgpack::adaptor { +// writes std::variant to msgpack format (TODO should we read std::variant?) +template +struct pack> { + auto &operator()(auto& o, std::variant const &variant) const { + std::visit([&o](auto &&arg) { msgpack::pack(o, arg); }, variant); + return o; + } +}; +} \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/serialize/msgpack_schema.test.cpp b/barretenberg/cpp/src/barretenberg/serialize/msgpack_schema.test.cpp new file mode 100644 index 000000000000..516caac5f524 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/serialize/msgpack_schema.test.cpp @@ -0,0 +1,86 @@ +#include "barretenberg/serialize/msgpack.hpp" +#include "barretenberg/serialize/cbind.hpp" + +#include + +// Sanity checking for msgpack +// TODO eventually move to barretenberg + +struct GoodExample { + barretenberg::fr a; + barretenberg::fr b; + MSGPACK_FIELDS(a, b); +} good_example; + +struct BadExampleOverlap { + barretenberg::fr a; + barretenberg::fr b; + MSGPACK_FIELDS(a, a); +} bad_example_overlap; + +struct BadExampleIncomplete { + barretenberg::fr a; + barretenberg::fr b; + MSGPACK_FIELDS(a); +} bad_example_incomplete; + +struct BadExampleCompileTimeError { + std::vector a; + barretenberg::fr b; + + MSGPACK_FIELDS(b); // Type mismatch, expect 'a', will catch at compile-time +} bad_example_compile_time_error; + +struct BadExampleOutOfObject { + barretenberg::fr a; + barretenberg::fr b; + void msgpack(auto ar) + { + BadExampleOutOfObject other_object; + ar("a", other_object.a, "b", other_object.b); + } +} bad_example_out_of_object; + +// TODO eventually move to barretenberg +TEST(msgpack_tests, msgpack_sanity_sanity) +{ + EXPECT_EQ(msgpack::check_msgpack_method(good_example), ""); + EXPECT_EQ(msgpack::check_msgpack_method(bad_example_overlap), + "Overlap in BadExampleOverlap MSGPACK_FIELDS() params detected!"); + EXPECT_EQ(msgpack::check_msgpack_method(bad_example_incomplete), + "Incomplete BadExampleIncomplete MSGPACK_FIELDS() params! Not all of object specified."); + + // If we actually try to msgpack BadExampleCompileTimeError we will statically error + // This is great, but we need to check the underlying facility *somehow* + auto checker = [&](auto&... values) { + std::string incomplete_msgpack_status = "error"; + if constexpr (msgpack::MsgpackConstructible) { + incomplete_msgpack_status = ""; + } + EXPECT_EQ(incomplete_msgpack_status, "error"); + }; + bad_example_compile_time_error.msgpack(checker); + + EXPECT_EQ(msgpack::check_msgpack_method(bad_example_out_of_object), + "Some BadExampleOutOfObject MSGPACK_FIELDS() params don't exist in object!"); +} + +struct ComplicatedSchema { + std::vector> array; + std::optional good_or_not; + barretenberg::fr bare; + std::variant huh; + MSGPACK_FIELDS(array, good_or_not, bare, huh); +} complicated_schema; + +TEST(msgpack_tests, msgpack_schema_sanity) +{ + EXPECT_EQ( + msgpack_schema_to_string(good_example), + "{\"__typename\":\"GoodExample\",\"a\":[\"alias\",[\"Fr\",\"bin32\"]],\"b\":[\"alias\",[\"Fr\",\"bin32\"]]}\n"); + EXPECT_EQ(msgpack_schema_to_string(complicated_schema), + "{\"__typename\":\"ComplicatedSchema\",\"array\":[\"vector\",[[\"array\",[[\"alias\",[\"Fr\",\"bin32\"]]," + "20]]]],\"good_or_not\":[\"optional\",[{\"__typename\":\"GoodExample\",\"a\":[\"alias\",[\"Fr\"," + "\"bin32\"]],\"b\":[\"alias\",[\"Fr\",\"bin32\"]]}]],\"bare\":[\"alias\",[\"Fr\",\"bin32\"]],\"huh\":[" + "\"variant\",[[\"alias\",[\"Fr\",\"bin32\"]],\"GoodExample\"]]}\n"); +} diff --git a/barretenberg/cpp/src/barretenberg/serialize/raw_pointer.hpp b/barretenberg/cpp/src/barretenberg/serialize/raw_pointer.hpp new file mode 100644 index 000000000000..4b6e0b16b0e1 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/serialize/raw_pointer.hpp @@ -0,0 +1,37 @@ +#pragma once +#include +#include "msgpack_impl/schema_name.hpp" + +// Holds a raw pointer to an object of type T. +// It provides methods for packing and unpacking the pointer using MessagePack, +// a binary serialization format. +template +struct RawPointer { + // Raw pointer to an object of type T + T* ptr = nullptr; + + // Pack the raw pointer into a MessagePack packer. + // The pointer is first cast to an integer type (uintptr_t) which can hold a pointer, + // and then packed into the packer. + void msgpack_pack(auto& packer) const { + packer.pack(reinterpret_cast(ptr)); + } + + // Unpack the raw pointer from a MessagePack object. + // The object is first cast to an integer type (uintptr_t), and then to a pointer of type T. + void msgpack_unpack(auto object) { + ptr = reinterpret_cast((uintptr_t)object); + } + + // Overload the arrow operator to return the raw pointer. + // This allows users to directly access the object pointed to by the raw pointer. + T* operator->() { + return ptr; + } +}; + +// help our msgpack schema compiler with this struct +template inline void msgpack_schema_pack(auto& packer, RawPointer const&) +{ + packer.pack_alias(msgpack_schema_name(T{}) + "Ptr", "int"); +} diff --git a/barretenberg/cpp/src/barretenberg/serialize/test_helper.hpp b/barretenberg/cpp/src/barretenberg/serialize/test_helper.hpp new file mode 100644 index 000000000000..903ea3c4317f --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/serialize/test_helper.hpp @@ -0,0 +1,38 @@ +#pragma once +#include +#include "cbind.hpp" + +/*** + * Do a roundtrip test encode/decode of an object. + * @tparam T The object type. + * @param object The object. Can be a default-initialized object. + */ +template std::pair msgpack_roundtrip(const T& object) +{ + T result; + msgpack::sbuffer buffer; + msgpack::pack(buffer, object); + msgpack::unpack(buffer.data(), buffer.size()).get().convert(result); + return { object, result }; +} + +template inline T call_msgpack_cbind(auto cbind_func, auto... test_args) +{ + auto [input, input_len] = msgpack_encode_buffer(std::make_tuple(test_args...)); + uint8_t* output; + size_t output_len; + cbind_func(input, input_len, &output, &output_len); + T actual_ret; + msgpack::unpack((const char*)output, output_len).get().convert(actual_ret); + aligned_free(output); + return actual_ret; +} + +// Running the end-to-end tests that msgpack bind creates +// This should suffice in testing the binding interface, function tests can be separate +inline auto call_func_and_wrapper(auto func, auto cbind_func, auto... test_args) +{ + auto expected_ret = func(test_args...); + auto actual_ret = call_msgpack_cbind(cbind_func, test_args...); + return std::make_pair(actual_ret, expected_ret); +} diff --git a/barretenberg/cpp/src/barretenberg/stdlib/primitives/address/address.hpp b/barretenberg/cpp/src/barretenberg/stdlib/primitives/address/address.hpp index 0b27a443c658..aaed93c92df4 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib/primitives/address/address.hpp +++ b/barretenberg/cpp/src/barretenberg/stdlib/primitives/address/address.hpp @@ -47,6 +47,10 @@ class address { friend std::ostream& operator<<(std::ostream& os, address const& v) { return os << v.address_; } fr to_field() const { return address_; } + + // delegate serialization to field + void msgpack_pack(auto& packer) const { address_.msgpack_pack(packer); } + void msgpack_unpack(auto const& o) { address_.msgpack_unpack(o); } }; template void read(B& it, address& addr) @@ -139,4 +143,10 @@ template class address_t { }; } // namespace stdlib -} // namespace proof_system::plonk \ No newline at end of file +} // namespace proof_system::plonk + +// help our msgpack schema compiler with this buffer alias (as far as wire representation is concerned) class +inline void msgpack_schema_pack(auto& packer, proof_system::plonk::stdlib::address const&) +{ + packer.pack_alias("Address", "bin32"); +} diff --git a/barretenberg/cpp/src/barretenberg/stdlib/recursion/aggregation_state/native_aggregation_state.hpp b/barretenberg/cpp/src/barretenberg/stdlib/recursion/aggregation_state/native_aggregation_state.hpp index 47b82b3f5741..1e97da3f77ec 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib/recursion/aggregation_state/native_aggregation_state.hpp +++ b/barretenberg/cpp/src/barretenberg/stdlib/recursion/aggregation_state/native_aggregation_state.hpp @@ -21,6 +21,7 @@ struct native_aggregation_state { std::vector proof_witness_indices; bool has_data = false; + MSGPACK_FIELDS(P0, P1, public_inputs, proof_witness_indices, has_data); bool operator==(native_aggregation_state const& other) const { return P0 == other.P0 && P1 == other.P1 && public_inputs == other.public_inputs && diff --git a/barretenberg/cpp/src/msgpack-c b/barretenberg/cpp/src/msgpack-c new file mode 160000 index 000000000000..1c90fb3a7c08 --- /dev/null +++ b/barretenberg/cpp/src/msgpack-c @@ -0,0 +1 @@ +Subproject commit 1c90fb3a7c08466d67b15062713a02da2e073fef