From 9f68072d203ca83487cff9c01063b211291e2d07 Mon Sep 17 00:00:00 2001 From: Ypatia Tsavliri Date: Wed, 6 Nov 2024 14:14:56 +0200 Subject: [PATCH] Set traversal limit in enumeration serialization --- test/src/unit-enumerations.cc | 2 +- test/src/unit-request-handlers.cc | 2 +- tiledb/sm/c_api/tiledb.cc | 1 + tiledb/sm/rest/rest_client_remote.cc | 4 ++-- tiledb/sm/serialization/array_schema.cc | 17 ++++++++++++++--- tiledb/sm/serialization/array_schema.h | 1 + .../sm/serialization/array_schema_evolution.cc | 17 ++++++++++++++--- .../sm/serialization/array_schema_evolution.h | 2 ++ tiledb/sm/serialization/enumeration.cc | 17 ++++++++++++++--- tiledb/sm/serialization/enumeration.h | 1 + 10 files changed, 51 insertions(+), 13 deletions(-) diff --git a/test/src/unit-enumerations.cc b/test/src/unit-enumerations.cc index 2f4392ff44d..2a5669fa0a8 100644 --- a/test/src/unit-enumerations.cc +++ b/test/src/unit-enumerations.cc @@ -2940,7 +2940,7 @@ shared_ptr EnumerationFx::ser_des_array_schema_evolution( ArraySchemaEvolution* ret; throw_if_not_ok(serialization::array_schema_evolution_deserialize( - &ret, stype, buf, memory_tracker_)); + &ret, cfg_, stype, buf, memory_tracker_)); return shared_ptr(ret); } diff --git a/test/src/unit-request-handlers.cc b/test/src/unit-request-handlers.cc index f1182124933..12bc6468aca 100644 --- a/test/src/unit-request-handlers.cc +++ b/test/src/unit-request-handlers.cc @@ -546,7 +546,7 @@ HandleLoadArraySchemaRequestFx::call_handler( REQUIRE(rval == TILEDB_OK); return serialization::deserialize_load_array_schema_response( - uri_, stype, resp_buf->buffer(), memory_tracker_); + uri_, cfg_, stype, resp_buf->buffer(), memory_tracker_); } shared_ptr HandleQueryPlanRequestFx::create_schema() { diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index b23150f54af..ea967c4fd0a 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -1572,6 +1572,7 @@ int32_t tiledb_deserialize_array_schema_evolution( ctx, tiledb::sm::serialization::array_schema_evolution_deserialize( &((*array_schema_evolution)->array_schema_evolution_), + ctx->config(), (tiledb::sm::SerializationType)serialize_type, buffer->buffer(), memory_tracker))) { diff --git a/tiledb/sm/rest/rest_client_remote.cc b/tiledb/sm/rest/rest_client_remote.cc index fb97579ba5f..463b1cc59ca 100644 --- a/tiledb/sm/rest/rest_client_remote.cc +++ b/tiledb/sm/rest/rest_client_remote.cc @@ -274,7 +274,7 @@ RestClientRemote::post_array_schema_from_rest( // Ensure data has a null delimiter for cap'n proto if using JSON throw_if_not_ok(ensure_json_null_delimited_string(&returned_data)); return serialization::deserialize_load_array_schema_response( - uri, serialization_type_, returned_data, memory_tracker_); + uri, config, serialization_type_, returned_data, memory_tracker_); } Status RestClientRemote::post_array_schema_to_rest( @@ -616,7 +616,7 @@ RestClientRemote::post_enumerations_from_rest( // Ensure data has a null delimiter for cap'n proto if using JSON throw_if_not_ok(ensure_json_null_delimited_string(&returned_data)); return serialization::deserialize_load_enumerations_response( - array_schema, serialization_type_, returned_data, memory_tracker); + array_schema, config, serialization_type_, returned_data, memory_tracker); } void RestClientRemote::post_query_plan_from_rest( diff --git a/tiledb/sm/serialization/array_schema.cc b/tiledb/sm/serialization/array_schema.cc index e8fe0f962ca..4496097ea8e 100644 --- a/tiledb/sm/serialization/array_schema.cc +++ b/tiledb/sm/serialization/array_schema.cc @@ -1824,6 +1824,7 @@ std::tuple< std::unordered_map>> deserialize_load_array_schema_response( const URI& uri, + const Config& config, SerializationType serialization_type, span data, shared_ptr memory_tracker) { @@ -1840,10 +1841,19 @@ deserialize_load_array_schema_response( uri, reader, memory_tracker); } case SerializationType::CAPNP: { + // Set traversal limit from config + uint64_t limit = + config.get("rest.capnp_traversal_limit").value(); + ::capnp::ReaderOptions readerOptions; + // capnp uses the limit in words + readerOptions.traversalLimitInWords = limit / sizeof(::capnp::word); + const auto mBytes = reinterpret_cast(data.data()); - ::capnp::FlatArrayMessageReader array_reader(kj::arrayPtr( - reinterpret_cast(mBytes), - data.size() / sizeof(::capnp::word))); + ::capnp::FlatArrayMessageReader array_reader( + kj::arrayPtr( + reinterpret_cast(mBytes), + data.size() / sizeof(::capnp::word)), + readerOptions); auto reader = array_reader.getRoot(); return load_array_schema_response_from_capnp( uri, reader, memory_tracker); @@ -1926,6 +1936,7 @@ std::tuple< std::unordered_map>> deserialize_load_array_schema_response( const URI&, + const Config&, SerializationType, span, shared_ptr) { diff --git a/tiledb/sm/serialization/array_schema.h b/tiledb/sm/serialization/array_schema.h index 03ce35664fc..54406042b70 100644 --- a/tiledb/sm/serialization/array_schema.h +++ b/tiledb/sm/serialization/array_schema.h @@ -208,6 +208,7 @@ std::tuple< std::unordered_map>> deserialize_load_array_schema_response( const URI& uri, + const Config& config, SerializationType serialization_type, span data, shared_ptr memory_tracker); diff --git a/tiledb/sm/serialization/array_schema_evolution.cc b/tiledb/sm/serialization/array_schema_evolution.cc index ab6fb82bbff..efb82e24fcd 100644 --- a/tiledb/sm/serialization/array_schema_evolution.cc +++ b/tiledb/sm/serialization/array_schema_evolution.cc @@ -289,6 +289,7 @@ Status array_schema_evolution_serialize( Status array_schema_evolution_deserialize( ArraySchemaEvolution** array_schema_evolution, + const Config& config, SerializationType serialize_type, span serialized_buffer, shared_ptr memory_tracker) { @@ -312,11 +313,20 @@ Status array_schema_evolution_deserialize( break; } case SerializationType::CAPNP: { + // Set traversal limit from config + uint64_t limit = + config.get("rest.capnp_traversal_limit").value(); + ::capnp::ReaderOptions readerOptions; + // capnp uses the limit in words + readerOptions.traversalLimitInWords = limit / sizeof(::capnp::word); + const auto mBytes = reinterpret_cast(serialized_buffer.data()); - ::capnp::FlatArrayMessageReader reader(kj::arrayPtr( - reinterpret_cast(mBytes), - serialized_buffer.size() / sizeof(::capnp::word))); + ::capnp::FlatArrayMessageReader reader( + kj::arrayPtr( + reinterpret_cast(mBytes), + serialized_buffer.size() / sizeof(::capnp::word)), + readerOptions); capnp::ArraySchemaEvolution::Reader array_schema_evolution_reader = reader.getRoot(); decoded_array_schema_evolution = array_schema_evolution_from_capnp( @@ -363,6 +373,7 @@ Status array_schema_evolution_serialize( Status array_schema_evolution_deserialize( ArraySchemaEvolution**, + const Config&, SerializationType, span, shared_ptr) { diff --git a/tiledb/sm/serialization/array_schema_evolution.h b/tiledb/sm/serialization/array_schema_evolution.h index 11e3de8fa5b..ae4bd86109a 100644 --- a/tiledb/sm/serialization/array_schema_evolution.h +++ b/tiledb/sm/serialization/array_schema_evolution.h @@ -70,6 +70,7 @@ Status array_schema_evolution_serialize( /** * Deserialize an array schema evolution via Cap'n Proto * @param array_schema_evolution pointer to store evolution object in + * @param config associated config object * @param serialize_type format to serialize into Cap'n Proto or JSON * @param serialized_buffer buffer where serialized bytes are stored * @param memory_tracker memory tracker associated with the evolution object @@ -77,6 +78,7 @@ Status array_schema_evolution_serialize( */ Status array_schema_evolution_deserialize( ArraySchemaEvolution** array_schema_evolution, + const Config& config, SerializationType serialize_type, span serialized_buffer, shared_ptr memory_tracker); diff --git a/tiledb/sm/serialization/enumeration.cc b/tiledb/sm/serialization/enumeration.cc index c21251ab40a..507c8224166 100644 --- a/tiledb/sm/serialization/enumeration.cc +++ b/tiledb/sm/serialization/enumeration.cc @@ -346,6 +346,7 @@ void serialize_load_enumerations_response( std::unordered_map>> deserialize_load_enumerations_response( const ArraySchema& array_schema, + const Config& config, SerializationType serialize_type, span response, shared_ptr memory_tracker) { @@ -362,10 +363,19 @@ deserialize_load_enumerations_response( reader, array_schema, memory_tracker); } case SerializationType::CAPNP: { + // Set traversal limit from config + uint64_t limit = + config.get("rest.capnp_traversal_limit").value(); + ::capnp::ReaderOptions readerOptions; + // capnp uses the limit in words + readerOptions.traversalLimitInWords = limit / sizeof(::capnp::word); + const auto mBytes = reinterpret_cast(response.data()); - ::capnp::FlatArrayMessageReader array_reader(kj::arrayPtr( - reinterpret_cast(mBytes), - response.size() / sizeof(::capnp::word))); + ::capnp::FlatArrayMessageReader array_reader( + kj::arrayPtr( + reinterpret_cast(mBytes), + response.size() / sizeof(::capnp::word)), + readerOptions); capnp::LoadEnumerationsResponse::Reader reader = array_reader.getRoot(); return load_enumerations_response_from_capnp( @@ -414,6 +424,7 @@ void serialize_load_enumerations_response( std::unordered_map>> deserialize_load_enumerations_response( const Array&, + const Config&, SerializationType, span, shared_ptr) { diff --git a/tiledb/sm/serialization/enumeration.h b/tiledb/sm/serialization/enumeration.h index 210d91f9d23..a015b97b41c 100644 --- a/tiledb/sm/serialization/enumeration.h +++ b/tiledb/sm/serialization/enumeration.h @@ -93,6 +93,7 @@ void serialize_load_enumerations_response( std::unordered_map>> deserialize_load_enumerations_response( const ArraySchema& array_schema, + const Config& config, SerializationType serialization_type, span response, shared_ptr memory_tracker);