From 9b4e5ea0c8b117716bc96dac71338ffc0d98a2db Mon Sep 17 00:00:00 2001 From: Shaun M Reed Date: Tue, 27 Aug 2024 15:32:37 -0400 Subject: [PATCH] Add serialization and API changes for `post_array_schema_from_rest`. (#5237) This factors out serialization and API changes in #5181 that are required for the HandleGetArraySchema route. These changes will need to be available on REST before we can enable the new route for loading the array schema. There is a quick summary of the changes required in [SC-52877](https://app.shortcut.com/tiledb-inc/story/52877/core-serialization-changes-for-loadarrayschema-models). --- TYPE: IMPROVEMENT DESC: Add serialization and API changes for post_array_schema_from_rest. --- test/src/unit-capi-config.cc | 2 +- test/src/unit-enumerations.cc | 63 ++++++++++ test/src/unit-request-handlers.cc | 114 +++++++++++++++--- test/support/src/helpers.cc | 20 +++ test/support/src/helpers.h | 9 ++ tiledb/api/c_api/array/array_api.cc | 72 +++++------ tiledb/api/c_api/array/array_api_external.h | 37 +++++- tiledb/api/c_api/config/config_api_external.h | 6 +- tiledb/sm/array/array.cc | 22 ++-- tiledb/sm/array/array.h | 4 +- tiledb/sm/array/array_directory.cc | 2 +- tiledb/sm/array_schema/CMakeLists.txt | 2 +- tiledb/sm/array_schema/array_schema.cc | 2 +- .../array_schema/array_schema_operations.cc | 61 ++++++++++ .../sm/array_schema/array_schema_operations.h | 13 ++ tiledb/sm/c_api/tiledb.cc | 15 ++- tiledb/sm/config/config.cc | 2 +- tiledb/sm/cpp_api/config.h | 4 + tiledb/sm/rest/rest_client.h | 5 +- tiledb/sm/rest/rest_client_remote.cc | 9 +- tiledb/sm/rest/rest_client_remote.h | 13 +- tiledb/sm/serialization/array_schema.cc | 83 ++++++++++--- tiledb/sm/serialization/array_schema.h | 16 ++- tiledb/sm/serialization/tiledb-rest.capnp | 5 + tiledb/sm/serialization/tiledb-rest.capnp.c++ | 60 +++++++-- tiledb/sm/serialization/tiledb-rest.capnp.h | 108 ++++++++++++++++- 26 files changed, 633 insertions(+), 116 deletions(-) diff --git a/test/src/unit-capi-config.cc b/test/src/unit-capi-config.cc index 94336bf1003..8875898ce62 100644 --- a/test/src/unit-capi-config.cc +++ b/test/src/unit-capi-config.cc @@ -230,7 +230,7 @@ void check_save_to_file() { ss << "rest.curl.buffer_size 524288\n"; ss << "rest.curl.verbose false\n"; ss << "rest.http_compressor any\n"; - ss << "rest.load_enumerations_on_array_open true\n"; + ss << "rest.load_enumerations_on_array_open false\n"; ss << "rest.load_metadata_on_array_open true\n"; ss << "rest.load_non_empty_domain_on_array_open true\n"; ss << "rest.retry_count 25\n"; diff --git a/test/src/unit-enumerations.cc b/test/src/unit-enumerations.cc index 91036639838..482496bd76a 100644 --- a/test/src/unit-enumerations.cc +++ b/test/src/unit-enumerations.cc @@ -1120,6 +1120,69 @@ TEST_CASE_METHOD( REQUIRE(schema->is_enumeration_loaded("test_enmr") == true); } +TEST_CASE_METHOD( + EnumerationFx, + "Array - Load All Enumerations - All Schemas", + "[enumeration][array][load-all-enumerations][all-schemas]") { + create_array(); + auto array = get_array(QueryType::READ); + auto schema = array->array_schema_latest_ptr(); + REQUIRE(schema->is_enumeration_loaded("test_enmr") == false); + std::string schema_name_1 = schema->name(); + + // Evolve once to add an enumeration. + auto ase = make_shared(HERE(), memory_tracker_); + std::vector var_values{"one", "two", "three"}; + auto var_enmr = create_enumeration( + var_values, false, Datatype::STRING_ASCII, "ase_var_enmr"); + ase->add_enumeration(var_enmr); + auto attr4 = make_shared(HERE(), "attr4", Datatype::UINT16); + attr4->set_enumeration_name("ase_var_enmr"); + CHECK_NOTHROW(ase->evolve_schema(schema)); + // Apply evolution to the array and reopen. + CHECK_NOTHROW(Array::evolve_array_schema( + ctx_.resources(), uri_, ase.get(), array->get_encryption_key())); + CHECK(array->reopen().ok()); + CHECK_NOTHROW(array->load_all_enumerations()); + auto all_schemas = array->array_schemas_all(); + schema = array->array_schema_latest_ptr(); + std::string schema_name_2 = schema->name(); + + // Check all schemas. + CHECK(all_schemas[schema_name_1]->is_enumeration_loaded("test_enmr") == true); + CHECK(all_schemas[schema_name_2]->is_enumeration_loaded("test_enmr") == true); + CHECK( + all_schemas[schema_name_2]->is_enumeration_loaded("ase_var_enmr") == + true); + + // Evolve a second time to drop an enumeration. + ase = make_shared(HERE(), memory_tracker_); + ase->drop_enumeration("test_enmr"); + ase->drop_attribute("attr1"); + CHECK_NOTHROW(ase->evolve_schema(schema)); + // Apply evolution to the array and reopen. + CHECK_NOTHROW(Array::evolve_array_schema( + ctx_.resources(), uri_, ase.get(), array->get_encryption_key())); + CHECK(array->reopen().ok()); + CHECK_NOTHROW(array->load_all_enumerations()); + all_schemas = array->array_schemas_all(); + schema = array->array_schema_latest_ptr(); + std::string schema_name_3 = schema->name(); + + // Check all schemas. + CHECK(all_schemas[schema_name_1]->is_enumeration_loaded("test_enmr") == true); + CHECK(all_schemas[schema_name_2]->is_enumeration_loaded("test_enmr") == true); + CHECK( + all_schemas[schema_name_2]->is_enumeration_loaded("ase_var_enmr") == + true); + CHECK_THROWS_WITH( + all_schemas[schema_name_3]->is_enumeration_loaded("test_enmr"), + Catch::Matchers::ContainsSubstring("No enumeration named")); + CHECK( + all_schemas[schema_name_3]->is_enumeration_loaded("ase_var_enmr") == + true); +} + TEST_CASE_METHOD( EnumerationFx, "Array - Load All Enumerations - Repeated", diff --git a/test/src/unit-request-handlers.cc b/test/src/unit-request-handlers.cc index b7368840d7a..5480e86b0f4 100644 --- a/test/src/unit-request-handlers.cc +++ b/test/src/unit-request-handlers.cc @@ -32,6 +32,7 @@ #ifdef TILEDB_SERIALIZATION +#include "test/support/src/helpers.h" #include "test/support/src/mem_helpers.h" #include "test/support/tdb_catch.h" #include "tiledb/api/c_api/buffer/buffer_api_internal.h" @@ -41,6 +42,7 @@ #include "tiledb/sm/c_api/tiledb_serialization.h" #include "tiledb/sm/c_api/tiledb_struct_def.h" #include "tiledb/sm/cpp_api/tiledb" +#include "tiledb/sm/cpp_api/tiledb_experimental" #include "tiledb/sm/crypto/encryption_key.h" #include "tiledb/sm/enums/array_type.h" #include "tiledb/sm/enums/encryption_type.h" @@ -67,6 +69,7 @@ struct RequestHandlerFx { Config cfg_; Context ctx_; EncryptionKey enc_key_; + shared_ptr schema_; }; struct HandleLoadArraySchemaRequestFx : RequestHandlerFx { @@ -75,11 +78,17 @@ struct HandleLoadArraySchemaRequestFx : RequestHandlerFx { } virtual shared_ptr create_schema() override; - shared_ptr call_handler( + + std::tuple< + shared_ptr, + std::unordered_map>> + call_handler( serialization::LoadArraySchemaRequest req, SerializationType stype); shared_ptr create_string_enumeration( std::string name, std::vector& values); + + shared_ptr schema_add_attribute(const std::string& attr_name); }; struct HandleQueryPlanRequestFx : RequestHandlerFx { @@ -116,15 +125,23 @@ struct HandleConsolidationPlanRequestFx : RequestHandlerFx { TEST_CASE_METHOD( HandleLoadArraySchemaRequestFx, - "tiledb_handle_load_array_schema_request - default request", + "tiledb_handle_load_array_schema_request - no enumerations", "[request_handler][load_array_schema][default]") { auto stype = GENERATE(SerializationType::JSON, SerializationType::CAPNP); create_array(); - auto schema = - call_handler(serialization::LoadArraySchemaRequest(false), stype); + auto schema_response = + call_handler(serialization::LoadArraySchemaRequest(cfg_), stype); + auto schema = std::get<0>(schema_response); REQUIRE(schema->has_enumeration("enmr")); REQUIRE(schema->get_loaded_enumeration_names().size() == 0); + tiledb::test::schema_equiv(*schema, *schema_); + + // We did not evolve the schema so there should only be one. + auto all_schemas = std::get<1>(schema_response); + REQUIRE(all_schemas.size() == 1); + tiledb::test::schema_equiv( + *all_schemas.find(schema->name())->second, *schema_); } TEST_CASE_METHOD( @@ -134,12 +151,57 @@ TEST_CASE_METHOD( auto stype = GENERATE(SerializationType::JSON, SerializationType::CAPNP); create_array(); - auto schema = - call_handler(serialization::LoadArraySchemaRequest(true), stype); + REQUIRE(cfg_.set("rest.load_enumerations_on_array_open", "true").ok()); + auto schema_response = + call_handler(serialization::LoadArraySchemaRequest(cfg_), stype); + auto schema = std::get<0>(schema_response); REQUIRE(schema->has_enumeration("enmr")); REQUIRE(schema->get_loaded_enumeration_names().size() == 1); REQUIRE(schema->get_loaded_enumeration_names()[0] == "enmr"); REQUIRE(schema->get_enumeration("enmr") != nullptr); + tiledb::test::schema_equiv(*schema, *schema_); + + // We did not evolve the schema so there should only be one. + auto all_schemas = std::get<1>(schema_response); + REQUIRE(all_schemas.size() == 1); + tiledb::test::schema_equiv( + *all_schemas.find(schema->name())->second, *schema_); +} + +TEST_CASE_METHOD( + HandleLoadArraySchemaRequestFx, + "tiledb_handle_load_array_schema_request - multiple schemas", + "[request_handler][load_array_schema][schema-evolution]") { + auto stype = GENERATE(SerializationType::JSON, SerializationType::CAPNP); + std::string load_enums = GENERATE("true", "false"); + + create_array(); + + std::vector> all_schemas{schema_}; + all_schemas.push_back(schema_add_attribute("b")); + all_schemas.push_back(schema_add_attribute("c")); + all_schemas.push_back(schema_add_attribute("d")); + + REQUIRE(cfg_.set("rest.load_enumerations_on_array_open", load_enums).ok()); + auto schema_response = + call_handler(serialization::LoadArraySchemaRequest(cfg_), stype); + auto schema = std::get<0>(schema_response); + if (load_enums == "true") { + REQUIRE(schema->has_enumeration("enmr")); + REQUIRE(schema->get_loaded_enumeration_names().size() == 1); + REQUIRE(schema->get_loaded_enumeration_names()[0] == "enmr"); + REQUIRE(schema->get_enumeration("enmr") != nullptr); + } + // The latest schema should be equal to the last applied evolution. + tiledb::test::schema_equiv(*schema, *all_schemas.back()); + + // Validate schemas returned from the request in the order they were created. + auto r_all_schemas = std::get<1>(schema_response); + std::map> resp( + r_all_schemas.begin(), r_all_schemas.end()); + for (int i = 0; const auto& s : resp) { + tiledb::test::schema_equiv(*s.second, *all_schemas[i++]); + } } TEST_CASE_METHOD( @@ -346,7 +408,9 @@ TEST_CASE_METHOD( RequestHandlerFx::RequestHandlerFx(const std::string uri) : memory_tracker_(tiledb::test::create_test_memory_tracker()) , uri_(uri) - , ctx_(cfg_) { + , ctx_(cfg_) + , schema_(make_shared( + ArrayType::DENSE, ctx_.resources().ephemeral_memory_tracker())) { delete_array(); throw_if_not_ok(enc_key_.set_key(EncryptionType::NO_ENCRYPTION, nullptr, 0)); } @@ -405,9 +469,28 @@ HandleLoadArraySchemaRequestFx::create_string_enumeration( tiledb::test::create_test_memory_tracker()); } +shared_ptr HandleLoadArraySchemaRequestFx::schema_add_attribute( + const std::string& attr_name) { + tiledb::Context ctx; + tiledb::ArraySchemaEvolution ase(ctx); + auto attr = tiledb::Attribute::create(ctx, attr_name); + ase.add_attribute(attr); + // Evolve and update the original schema member variable. + schema_ = ase.ptr()->array_schema_evolution_->evolve_schema(schema_); + // Apply the schema evolution. + Array::evolve_array_schema( + this->ctx_.resources(), + this->uri_, + ase.ptr()->array_schema_evolution_, + this->enc_key_); + + // Return the new evolved schema for validation. + return schema_; +} + shared_ptr HandleLoadArraySchemaRequestFx::create_schema() { // Create a schema to serialize - auto schema = + schema_ = make_shared(HERE(), ArrayType::SPARSE, memory_tracker_); auto dim = make_shared(HERE(), "dim1", Datatype::INT32, memory_tracker_); @@ -416,20 +499,23 @@ shared_ptr HandleLoadArraySchemaRequestFx::create_schema() { auto dom = make_shared(HERE(), memory_tracker_); throw_if_not_ok(dom->add_dimension(dim)); - throw_if_not_ok(schema->set_domain(dom)); + throw_if_not_ok(schema_->set_domain(dom)); std::vector values = {"pig", "cow", "chicken", "dog", "cat"}; auto enmr = create_string_enumeration("enmr", values); - schema->add_enumeration(enmr); + schema_->add_enumeration(enmr); auto attr = make_shared(HERE(), "attr", Datatype::INT32); attr->set_enumeration_name("enmr"); - throw_if_not_ok(schema->add_attribute(attr)); + throw_if_not_ok(schema_->add_attribute(attr)); - return schema; + return schema_; } -shared_ptr HandleLoadArraySchemaRequestFx::call_handler( +std::tuple< + shared_ptr, + std::unordered_map>> +HandleLoadArraySchemaRequestFx::call_handler( serialization::LoadArraySchemaRequest req, SerializationType stype) { // If this looks weird, its because we're using the public C++ API to create // these objets instead of the internal APIs elsewhere in this test suite. @@ -451,7 +537,7 @@ shared_ptr HandleLoadArraySchemaRequestFx::call_handler( REQUIRE(rval == TILEDB_OK); return serialization::deserialize_load_array_schema_response( - stype, resp_buf->buffer(), memory_tracker_); + uri_, stype, resp_buf->buffer(), memory_tracker_); } shared_ptr HandleQueryPlanRequestFx::create_schema() { diff --git a/test/support/src/helpers.cc b/test/support/src/helpers.cc index dd497972016..c8259245bb2 100644 --- a/test/support/src/helpers.cc +++ b/test/support/src/helpers.cc @@ -1621,6 +1621,26 @@ void read_sparse_v11( tiledb_query_free(&query); } +void schema_equiv( + const sm::ArraySchema& schema1, const sm::ArraySchema& schema2) { + CHECK(schema1.array_type() == schema2.array_type()); + CHECK(schema1.attributes().size() == schema2.attributes().size()); + for (unsigned int i = 0; i < schema2.attribute_num(); i++) { + auto a = schema1.attribute(i); + auto b = schema2.attribute(i); + CHECK(a->cell_val_num() == b->cell_val_num()); + CHECK(a->name() == b->name()); + CHECK(a->type() == b->type()); + CHECK(a->nullable() == b->nullable()); + CHECK(a->get_enumeration_name() == b->get_enumeration_name()); + } + CHECK(schema1.capacity() == schema2.capacity()); + CHECK(schema1.cell_order() == schema2.cell_order()); + CHECK(schema1.tile_order() == schema2.tile_order()); + CHECK(schema1.allows_dups() == schema2.allows_dups()); + CHECK(schema1.array_uri().to_string() == schema2.array_uri().to_string()); +} + template void check_subarray( tiledb::sm::Subarray& subarray, const SubarrayRanges& ranges); template void check_subarray( diff --git a/test/support/src/helpers.h b/test/support/src/helpers.h index dff048bc1a9..201821c09c6 100644 --- a/test/support/src/helpers.h +++ b/test/support/src/helpers.h @@ -957,6 +957,15 @@ void write_sparse_v11( */ void read_sparse_v11( tiledb_ctx_t* ctx, const std::string& array_name, uint64_t timestamp); + +/** + * Helper function to test two array schemas are equivalent. + * + * @param schema1 Expected array schema. + * @param schema2 Actual array schema. + */ +void schema_equiv( + const sm::ArraySchema& schema1, const sm::ArraySchema& schema2); } // namespace tiledb::test #endif diff --git a/tiledb/api/c_api/array/array_api.cc b/tiledb/api/c_api/array/array_api.cc index 64604685c02..91601b9e503 100644 --- a/tiledb/api/c_api/array/array_api.cc +++ b/tiledb/api/c_api/array/array_api.cc @@ -37,6 +37,7 @@ #include "tiledb/api/c_api/context/context_api_internal.h" #include "tiledb/api/c_api_support/c_api_support.h" #include "tiledb/sm/array/array_directory.h" +#include "tiledb/sm/array_schema/array_schema_operations.h" #include "tiledb/sm/enums/encryption_type.h" #include "tiledb/sm/rest/rest_client.h" @@ -46,50 +47,32 @@ capi_return_t tiledb_array_schema_load( tiledb_ctx_t* ctx, const char* array_uri, tiledb_array_schema_t** array_schema) { + // Create array schema + ensure_context_is_valid(ctx); ensure_output_pointer_is_valid(array_schema); - auto uri = tiledb::sm::URI(array_uri); - if (uri.is_invalid()) { - throw CAPIException("Invalid input uri."); - } + // Use a default constructed config to load the schema with default options. + *array_schema = tiledb_array_schema_t::make_handle( + load_array_schema(ctx->context(), sm::URI(array_uri), sm::Config())); - if (uri.is_tiledb()) { - auto& rest_client = ctx->context().rest_client(); - auto&& [st, array_schema_rest] = - rest_client.get_array_schema_from_rest(uri); - if (!st.ok()) { - throw CAPIException("Failed to load array schema; " + st.message()); - } - *array_schema = - tiledb_array_schema_t::make_handle(*(array_schema_rest->get())); - } else { - // Create key - tiledb::sm::EncryptionKey key; - throw_if_not_ok( - key.set_key(tiledb::sm::EncryptionType::NO_ENCRYPTION, nullptr, 0)); + return TILEDB_OK; +} + +capi_return_t tiledb_array_schema_load_with_config( + tiledb_ctx_t* ctx, + tiledb_config_t* config, + const char* array_uri, + tiledb_array_schema_t** array_schema) { + ensure_context_is_valid(ctx); + ensure_config_is_valid(config); + ensure_output_pointer_is_valid(array_schema); - // Load URIs from the array directory - optional array_dir; - try { - array_dir.emplace( - ctx->resources(), - uri, - 0, - UINT64_MAX, - tiledb::sm::ArrayDirectoryMode::SCHEMA_ONLY); - } catch (const std::logic_error& le) { - throw CAPIException( - "Failed to load array schema; " + - Status_ArrayDirectoryError(le.what()).message()); - } + // Use passed config or context config to load the schema with set options. + *array_schema = tiledb_array_schema_t::make_handle(load_array_schema( + ctx->context(), + sm::URI(array_uri), + config ? config->config() : ctx->config())); - // Load latest array schema - auto tracker = ctx->resources().ephemeral_memory_tracker(); - auto&& array_schema_latest = - array_dir->load_array_schema_latest(key, tracker); - *array_schema = - tiledb_array_schema_t::make_handle(*(array_schema_latest.get())); - } return TILEDB_OK; } @@ -105,3 +88,14 @@ CAPI_INTERFACE( return api_entry_with_context( ctx, array_uri, array_schema); } + +CAPI_INTERFACE( + array_schema_load_with_config, + tiledb_ctx_t* ctx, + tiledb_config_t* config, + const char* array_uri, + tiledb_array_schema_t** array_schema) { + return api_entry_with_context< + tiledb::api::tiledb_array_schema_load_with_config>( + ctx, config, array_uri, array_schema); +} diff --git a/tiledb/api/c_api/array/array_api_external.h b/tiledb/api/c_api/array/array_api_external.h index 5b7b0d70b9a..e273e07fbfe 100644 --- a/tiledb/api/c_api/array/array_api_external.h +++ b/tiledb/api/c_api/array/array_api_external.h @@ -42,8 +42,8 @@ extern "C" { #endif /** - * Retrieves the schema of an array from the disk, creating an array schema - * struct. + * Retrieves the latest schema of an array from the disk, creating an array + * schema struct. * * **Example:** * @@ -64,6 +64,39 @@ TILEDB_EXPORT capi_return_t tiledb_array_schema_load( const char* array_uri, tiledb_array_schema_t** array_schema) TILEDB_NOEXCEPT; +/** + * Retrieves the latest schema of an array, creating an array schema struct. + * Options to load additional features are read from the provided + * tiledb_config_t* instance. If the provided config is nullptr, the config from + * `ctx` is used instead. + * + * Currently supported options to be read from the config: + * - rest.load_enumerations_on_array_open - boolean + * + * **Example:** + * + * @code{.c} + * tiledb_array_schema_t* array_schema; + * tiledb_array_schema_load_with_config( + * ctx, + * config, + * "s3://tiledb_bucket/my_array", + * &array_schema); + * // Make sure to free the array schema in the end + * @endcode + * + * @param ctx The TileDB context. + * @param config The TileDB config. + * @param array_uri The array whose schema will be retrieved. + * @param array_schema The array schema to be retrieved, or `NULL` upon error. + * @return `TILEDB_OK` for success and `TILEDB_OOM` or `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_schema_load_with_config( + tiledb_ctx_t* ctx, + tiledb_config_t* config, + const char* array_uri, + tiledb_array_schema_t** array_schema) TILEDB_NOEXCEPT; + #ifdef __cplusplus } #endif diff --git a/tiledb/api/c_api/config/config_api_external.h b/tiledb/api/c_api/config/config_api_external.h index b57b58a3c7b..73893c7c86b 100644 --- a/tiledb/api/c_api/config/config_api_external.h +++ b/tiledb/api/c_api/config/config_api_external.h @@ -740,13 +740,17 @@ TILEDB_EXPORT void tiledb_config_free(tiledb_config_t** config) TILEDB_NOEXCEPT; * If true, array non empty domain will be loaded and sent to server together * with the open array
* **Default**: true + * - `rest.load_enumerations_on_array_open`
+ * If true, enumerations will be loaded and sent to server together with + * the open array. + * **Default**: false * - `rest.use_refactored_array_open`
* If true, the new REST routes and APIs for opening an array * will be used
* **Default**: true * - `rest.use_refactored_array_open_and_query_submit`
* If true, the new REST routes and APIs for opening an array and submitting - * a query will be used
+ * a query will be used
* **Default**: true * - `rest.curl.buffer_size`
* Set curl buffer size for REST requests
diff --git a/tiledb/sm/array/array.cc b/tiledb/sm/array/array.cc index c37682971d3..23144019733 100644 --- a/tiledb/sm/array/array.cc +++ b/tiledb/sm/array/array.cc @@ -804,11 +804,17 @@ void Array::encryption_type( shared_ptr Array::get_enumeration( const std::string& enumeration_name) { - return get_enumerations({enumeration_name})[0]; + if (!is_open_) { + throw ArrayException("Unable to load enumerations; Array is not open."); + } + + return get_enumerations( + {enumeration_name}, opened_array_->array_schema_latest_ptr())[0]; } std::vector> Array::get_enumerations( - const std::vector& enumeration_names) { + const std::vector& enumeration_names, + shared_ptr schema) { if (!is_open_) { throw ArrayException("Unable to load enumerations; Array is not open."); } @@ -816,7 +822,7 @@ std::vector> Array::get_enumerations( // Dedupe requested names and filter out anything already loaded. std::unordered_set enmrs_to_load; for (auto& enmr_name : enumeration_names) { - if (array_schema_latest().is_enumeration_loaded(enmr_name)) { + if (schema->is_enumeration_loaded(enmr_name)) { continue; } enmrs_to_load.insert(enmr_name); @@ -850,7 +856,7 @@ std::vector> Array::get_enumerations( // Create a vector of paths to be loaded. std::vector paths_to_load; for (auto& enmr_name : enmrs_to_load) { - auto path = array_schema_latest().get_enumeration_path_name(enmr_name); + auto path = schema->get_enumeration_path_name(enmr_name); paths_to_load.push_back(path); } @@ -861,14 +867,14 @@ std::vector> Array::get_enumerations( // Store the loaded enumerations in the schema for (auto& enmr : loaded) { - opened_array_->array_schema_latest_ptr()->store_enumeration(enmr); + schema->store_enumeration(enmr); } } // Return the requested list of enumerations std::vector> ret(enumeration_names.size()); for (size_t i = 0; i < enumeration_names.size(); i++) { - ret[i] = array_schema_latest().get_enumeration(enumeration_names[i]); + ret[i] = schema->get_enumeration(enumeration_names[i]); } return ret; } @@ -878,7 +884,9 @@ void Array::load_all_enumerations() { throw ArrayException("Unable to load all enumerations; Array is not open."); } // Load all enumerations, discarding the returned list of loaded enumerations. - get_enumerations(array_schema_latest().get_enumeration_names()); + for (const auto& schema : array_schemas_all()) { + get_enumerations(schema.second->get_enumeration_names(), schema.second); + } } bool Array::is_empty() const { diff --git a/tiledb/sm/array/array.h b/tiledb/sm/array/array.h index bf6b3292daa..c2ece7e384f 100644 --- a/tiledb/sm/array/array.h +++ b/tiledb/sm/array/array.h @@ -589,10 +589,12 @@ class Array { * loaded before this function returns. * * @param enumeration_names The names of the enumerations. + * @param schema The ArraySchema to store loaded enumerations in. * @return std::vector> The loaded enumerations. */ std::vector> get_enumerations( - const std::vector& enumeration_names); + const std::vector& enumeration_names, + shared_ptr schema); /** Load all enumerations for the array. */ void load_all_enumerations(); diff --git a/tiledb/sm/array/array_directory.cc b/tiledb/sm/array/array_directory.cc index 32f966b4606..444036d9f2e 100644 --- a/tiledb/sm/array/array_directory.cc +++ b/tiledb/sm/array/array_directory.cc @@ -123,7 +123,7 @@ shared_ptr ArrayDirectory::load_array_schema_latest( auto&& array_schema = load_array_schema_from_uri( resources_.get(), schema_uri, encryption_key, memory_tracker); - array_schema->set_array_uri(uri_); + array_schema->set_array_uri(uri_.remove_trailing_slash()); return std::move(array_schema); } diff --git a/tiledb/sm/array_schema/CMakeLists.txt b/tiledb/sm/array_schema/CMakeLists.txt index 831d51d21f4..9a398c660c3 100644 --- a/tiledb/sm/array_schema/CMakeLists.txt +++ b/tiledb/sm/array_schema/CMakeLists.txt @@ -98,7 +98,7 @@ conclude(object_library) # commence(object_library array_schema_operations) this_target_sources(array_schema_operations.cc) - this_target_object_libraries(array_schema generic_tile_io tiledb_crypto) + this_target_object_libraries(array_directory array_schema generic_tile_io tiledb_crypto) conclude(object_library) add_test_subdirectory() diff --git a/tiledb/sm/array_schema/array_schema.cc b/tiledb/sm/array_schema/array_schema.cc index 24df5692c67..1de4e2982f8 100644 --- a/tiledb/sm/array_schema/array_schema.cc +++ b/tiledb/sm/array_schema/array_schema.cc @@ -1150,7 +1150,7 @@ void ArraySchema::drop_enumeration(const std::string& enmr_name) { } if (attr_enmr_name.value() == enmr_name) { throw ArraySchemaException( - "Unable to drop enumeration '" + enmr_name + "' as it is used by " + + "Unable to drop enumeration '" + enmr_name + "' as it is used by" + " attribute '" + attr->name() + "'."); } } diff --git a/tiledb/sm/array_schema/array_schema_operations.cc b/tiledb/sm/array_schema/array_schema_operations.cc index 6fdf2384af9..d17388b48d6 100644 --- a/tiledb/sm/array_schema/array_schema_operations.cc +++ b/tiledb/sm/array_schema/array_schema_operations.cc @@ -36,8 +36,12 @@ #include "tiledb/sm/array_schema/dimension_label.h" #include "tiledb/sm/array_schema/domain.h" #include "tiledb/sm/array_schema/enumeration.h" +#include "tiledb/sm/config/config.h" +#include "tiledb/sm/crypto/encryption_key.h" #include "tiledb/sm/filesystem/uri.h" #include "tiledb/sm/misc/integral_type_casts.h" +#include "tiledb/sm/rest/rest_client.h" +#include "tiledb/sm/storage_manager/context.h" #include "tiledb/sm/storage_manager/context_resources.h" #include "tiledb/sm/tile/generic_tile_io.h" #include "tiledb/sm/tile/tile.h" @@ -227,4 +231,61 @@ void store_array_schema( } } +shared_ptr load_array_schema( + const Context& ctx, const URI& uri, const Config& config) { + // Check array name + if (uri.is_invalid()) { + throw std::runtime_error("Failed to load array schema; Invalid array URI"); + } + + if (uri.is_tiledb()) { + auto& rest_client = ctx.rest_client(); + auto&& [st, array_schema_response] = + rest_client.get_array_schema_from_rest(uri); + throw_if_not_ok(st); + return std::move(array_schema_response).value(); + } else { + // Create key + tiledb::sm::EncryptionKey key; + throw_if_not_ok( + key.set_key(tiledb::sm::EncryptionType::NO_ENCRYPTION, nullptr, 0)); + + // Load URIs from the array directory + optional array_dir; + array_dir.emplace( + ctx.resources(), + uri, + 0, + UINT64_MAX, + tiledb::sm::ArrayDirectoryMode::SCHEMA_ONLY); + + auto tracker = ctx.resources().ephemeral_memory_tracker(); + // Load latest array schema + auto&& array_schema_latest = + array_dir->load_array_schema_latest(key, tracker); + + // Load enumerations if config option is set. + bool incl_enums = config.get( + "rest.load_enumerations_on_array_open", Config::must_find); + if (incl_enums) { + std::vector enmr_paths_to_load; + auto enmr_names = array_schema_latest->get_enumeration_names(); + for (auto& name : enmr_names) { + if (!array_schema_latest->is_enumeration_loaded(name)) { + auto& path = array_schema_latest->get_enumeration_path_name(name); + enmr_paths_to_load.emplace_back(path); + } + } + + auto enmrs_loaded = array_dir->load_enumerations_from_paths( + enmr_paths_to_load, key, tracker); + for (auto& enmr : enmrs_loaded) { + array_schema_latest->store_enumeration(enmr); + } + } + + return std::move(array_schema_latest); + } +} + } // namespace tiledb::sm diff --git a/tiledb/sm/array_schema/array_schema_operations.h b/tiledb/sm/array_schema/array_schema_operations.h index 19a230bc366..64c1adbd0b5 100644 --- a/tiledb/sm/array_schema/array_schema_operations.h +++ b/tiledb/sm/array_schema/array_schema_operations.h @@ -41,8 +41,11 @@ using namespace tiledb::common; namespace tiledb::sm { class ArraySchema; +class Config; +class Context; class ContextResources; class EncryptionKey; +class URI; /* ********************************* */ /* API */ @@ -76,6 +79,16 @@ void store_array_schema( const shared_ptr& array_schema, const EncryptionKey& encryption_key); +/** + * Handles loading an array schema from a URI. + * + * @param ctx TileDB Context. + * @param uri The URI of the array to load schema. + * @param config TileDB Config. If null, the context config will be used. + */ +shared_ptr load_array_schema( + const Context& ctx, const URI& uri, const Config& config); + } // namespace tiledb::sm #endif // TILEDB_ARRAY_SCHEMA_OPERATIONS_H diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 0dee22e1ba2..4cd80bf5cc0 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -3314,12 +3314,13 @@ capi_return_t tiledb_handle_load_array_schema_request( tiledb_serialization_type_t serialization_type, const tiledb_buffer_t* request, tiledb_buffer_t* response) { + ensure_context_is_valid(ctx); if (sanity_check(ctx, array) == TILEDB_ERR) { - throw std::invalid_argument("Array paramter must be valid."); + throw CAPIStatusException("Array paramter must be valid."); } - - api::ensure_buffer_is_valid(request); - api::ensure_buffer_is_valid(response); + ensure_array_is_valid(array); + ensure_buffer_is_valid(request); + ensure_buffer_is_valid(response); auto load_schema_req = tiledb::sm::serialization::deserialize_load_array_schema_request( @@ -3331,7 +3332,7 @@ capi_return_t tiledb_handle_load_array_schema_request( } tiledb::sm::serialization::serialize_load_array_schema_response( - array->array_->array_schema_latest(), + *array->array_, static_cast(serialization_type), response->buffer()); @@ -3355,7 +3356,9 @@ capi_return_t tiledb_handle_load_enumerations_request( tiledb::sm::serialization::deserialize_load_enumerations_request( static_cast(serialization_type), request->buffer()); - auto enumerations = array->array_->get_enumerations(enumeration_names); + auto enumerations = array->array_->get_enumerations( + enumeration_names, + array->array_->opened_array()->array_schema_latest_ptr()); tiledb::sm::serialization::serialize_load_enumerations_response( enumerations, diff --git a/tiledb/sm/config/config.cc b/tiledb/sm/config/config.cc index 66f22432b03..107770cc239 100644 --- a/tiledb/sm/config/config.cc +++ b/tiledb/sm/config/config.cc @@ -91,7 +91,7 @@ const std::string Config::REST_RETRY_DELAY_FACTOR = "1.25"; const std::string Config::REST_CURL_BUFFER_SIZE = "524288"; const std::string Config::REST_CAPNP_TRAVERSAL_LIMIT = "2147483648"; const std::string Config::REST_CURL_VERBOSE = "false"; -const std::string Config::REST_LOAD_ENUMERATIONS_ON_ARRAY_OPEN = "true"; +const std::string Config::REST_LOAD_ENUMERATIONS_ON_ARRAY_OPEN = "false"; const std::string Config::REST_LOAD_METADATA_ON_ARRAY_OPEN = "true"; const std::string Config::REST_LOAD_NON_EMPTY_DOMAIN_ON_ARRAY_OPEN = "true"; const std::string Config::REST_USE_REFACTORED_ARRAY_OPEN = "true"; diff --git a/tiledb/sm/cpp_api/config.h b/tiledb/sm/cpp_api/config.h index c72af74df97..dd50be155d5 100644 --- a/tiledb/sm/cpp_api/config.h +++ b/tiledb/sm/cpp_api/config.h @@ -915,6 +915,10 @@ class Config { * If true, array non empty domain will be loaded and sent to server * together with the open array
* **Default**: true + * - `rest.load_enumerations_on_array_open`
+ * If true, enumerations will be loaded and sent to server together with + * the open array. + * **Default**: false * - `rest.use_refactored_array_open`
* If true, the new REST routes and APIs for opening an array will be used *
diff --git a/tiledb/sm/rest/rest_client.h b/tiledb/sm/rest/rest_client.h index c7237954309..80d33d8a841 100644 --- a/tiledb/sm/rest/rest_client.h +++ b/tiledb/sm/rest/rest_client.h @@ -317,7 +317,10 @@ class RestClient { } /// Operation disabled in base class. - inline virtual shared_ptr post_array_schema_from_rest( + inline virtual std::tuple< + shared_ptr, + std::unordered_map>> + post_array_schema_from_rest( const Config&, const URI&, uint64_t, uint64_t, bool) { throw RestClientDisabledException(); } diff --git a/tiledb/sm/rest/rest_client_remote.cc b/tiledb/sm/rest/rest_client_remote.cc index 5f5cf9daf27..afe69ea87a3 100644 --- a/tiledb/sm/rest/rest_client_remote.cc +++ b/tiledb/sm/rest/rest_client_remote.cc @@ -226,13 +226,16 @@ RestClientRemote::get_array_schema_from_rest(const URI& uri) { serialization_type_, returned_data, memory_tracker_)}; } -shared_ptr RestClientRemote::post_array_schema_from_rest( +std::tuple< + shared_ptr, + std::unordered_map>> +RestClientRemote::post_array_schema_from_rest( const Config& config, const URI& uri, uint64_t timestamp_start, uint64_t timestamp_end, bool include_enumerations) { - serialization::LoadArraySchemaRequest req(include_enumerations); + serialization::LoadArraySchemaRequest req(config); Buffer buf; serialization::serialize_load_array_schema_request( @@ -271,7 +274,7 @@ shared_ptr RestClientRemote::post_array_schema_from_rest( // Ensure data has a null delimiter for cap'n proto if using JSON throw_if_not_ok(ensure_json_null_delimited_string(&returned_data)); return serialization::deserialize_load_array_schema_response( - serialization_type_, returned_data, memory_tracker_); + uri, serialization_type_, returned_data, memory_tracker_); } Status RestClientRemote::post_array_schema_to_rest( diff --git a/tiledb/sm/rest/rest_client_remote.h b/tiledb/sm/rest/rest_client_remote.h index 4ff2dc91a02..d40cdda2121 100644 --- a/tiledb/sm/rest/rest_client_remote.h +++ b/tiledb/sm/rest/rest_client_remote.h @@ -151,10 +151,17 @@ class RestClientRemote : public RestClient { * get_array_schema_from_rest after TileDB-Cloud-REST merges support for the * POST endpoint. * + * @param config The TileDB config. * @param uri The Array URI to load the schema from. - * @return shared_ptr The loaded array schema. - */ - shared_ptr post_array_schema_from_rest( + * @param timestamp_start The starting timestamp used to open the array. + * @param timestamp_end The ending timestamp used to open the array. + * @return Tuple containing the latest array schema, and all array schemas for + * the array opened with provided timestamps. + */ + std::tuple< + shared_ptr, + std::unordered_map>> + post_array_schema_from_rest( const Config& config, const URI& uri, uint64_t timestamp_start, diff --git a/tiledb/sm/serialization/array_schema.cc b/tiledb/sm/serialization/array_schema.cc index b9d64cc8430..74c6c0e131b 100644 --- a/tiledb/sm/serialization/array_schema.cc +++ b/tiledb/sm/serialization/array_schema.cc @@ -1850,6 +1850,8 @@ void load_array_schema_request_to_capnp( const LoadArraySchemaRequest& req) { auto config_builder = builder.initConfig(); throw_if_not_ok(config_to_capnp(config, &config_builder)); + // This boolean is only serialized to support clients using TileDB < 2.26. + // Future options should only be serialized within the Config object above. builder.setIncludeEnumerations(req.include_enumerations()); } @@ -1906,7 +1908,15 @@ void serialize_load_array_schema_request( LoadArraySchemaRequest load_array_schema_request_from_capnp( capnp::LoadArraySchemaRequest::Reader& reader) { - return LoadArraySchemaRequest(reader.getIncludeEnumerations()); + tdb_unique_ptr decoded_config = nullptr; + if (reader.hasConfig()) { + throw_if_not_ok(config_from_capnp(reader.getConfig(), &decoded_config)); + } else { + decoded_config.reset(tdb_new(Config)); + } + // We intentionally do not use the includeEnumerations field, as it is stored + // in the Config and set using the LoadArraySchemaRequest constructor. + return LoadArraySchemaRequest(*decoded_config); } LoadArraySchemaRequest deserialize_load_array_schema_request( @@ -1949,20 +1959,31 @@ LoadArraySchemaRequest deserialize_load_array_schema_request( } void load_array_schema_response_to_capnp( - capnp::LoadArraySchemaResponse::Builder& builder, - const ArraySchema& schema) { + capnp::LoadArraySchemaResponse::Builder& builder, const Array& array) { auto schema_builder = builder.initSchema(); - throw_if_not_ok(array_schema_to_capnp(schema, &schema_builder, false)); + throw_if_not_ok(array_schema_to_capnp( + array.array_schema_latest(), &schema_builder, false)); + + const auto& array_schemas_all = array.array_schemas_all(); + auto array_schemas_all_builder = builder.initArraySchemasAll(); + auto entries_builder = + array_schemas_all_builder.initEntries(array_schemas_all.size()); + uint64_t i = 0; + for (const auto& schema : array_schemas_all) { + auto entry = entries_builder[i++]; + entry.setKey(schema.first); + auto schema_entry_builder = entry.initValue(); + throw_if_not_ok( + array_schema_to_capnp(*(schema.second), &schema_entry_builder, false)); + } } void serialize_load_array_schema_response( - const ArraySchema& schema, - SerializationType serialization_type, - Buffer& data) { + const Array& array, SerializationType serialization_type, Buffer& data) { try { ::capnp::MallocMessageBuilder message; auto builder = message.initRoot(); - load_array_schema_response_to_capnp(builder, schema); + load_array_schema_response_to_capnp(builder, array); data.reset_size(); data.reset_offset(); @@ -2005,14 +2026,39 @@ void serialize_load_array_schema_response( } } -shared_ptr load_array_schema_response_from_capnp( +std::tuple< + shared_ptr, + std::unordered_map>> +load_array_schema_response_from_capnp( + const URI& uri, capnp::LoadArraySchemaResponse::Reader& reader, shared_ptr memory_tracker) { auto schema_reader = reader.getSchema(); - return array_schema_from_capnp(schema_reader, URI(), memory_tracker); + auto schema = array_schema_from_capnp(schema_reader, URI(), memory_tracker); + schema->set_array_uri(uri); + + std::unordered_map> all_schemas; + if (reader.hasArraySchemasAll()) { + auto all_schemas_reader = reader.getArraySchemasAll(); + + if (all_schemas_reader.hasEntries()) { + auto entries = all_schemas_reader.getEntries(); + for (auto array_schema_build : entries) { + auto schema_entry = array_schema_from_capnp( + array_schema_build.getValue(), schema->array_uri(), memory_tracker); + schema_entry->set_array_uri(schema->array_uri()); + all_schemas[array_schema_build.getKey()] = schema_entry; + } + } + } + return {schema, all_schemas}; } -shared_ptr deserialize_load_array_schema_response( +std::tuple< + shared_ptr, + std::unordered_map>> +deserialize_load_array_schema_response( + const URI& uri, SerializationType serialization_type, const Buffer& data, shared_ptr memory_tracker) { @@ -2026,7 +2072,8 @@ shared_ptr deserialize_load_array_schema_response( json.decode( kj::StringPtr(static_cast(data.data())), builder); auto reader = builder.asReader(); - return load_array_schema_response_from_capnp(reader, memory_tracker); + return load_array_schema_response_from_capnp( + uri, reader, memory_tracker); } case SerializationType::CAPNP: { const auto mBytes = reinterpret_cast(data.data()); @@ -2034,7 +2081,8 @@ shared_ptr deserialize_load_array_schema_response( reinterpret_cast(mBytes), data.size() / sizeof(::capnp::word))); auto reader = array_reader.getRoot(); - return load_array_schema_response_from_capnp(reader, memory_tracker); + return load_array_schema_response_from_capnp( + uri, reader, memory_tracker); } default: { throw ArraySchemaSerializationException( @@ -2115,12 +2163,15 @@ LoadArraySchemaRequest deserialize_load_array_schema_request( } void serialize_load_array_schema_response( - const ArraySchema&, SerializationType, Buffer&) { + const Array&, SerializationType, Buffer&) { throw ArraySchemaSerializationDisabledException(); } -shared_ptr deserialize_load_array_schema_response( - SerializationType, const Buffer&, shared_ptr) { +std::tuple< + shared_ptr, + std::unordered_map>> +deserialize_load_array_schema_response( + const URI&, SerializationType, const Buffer&, shared_ptr) { throw ArraySchemaSerializationDisabledException(); } diff --git a/tiledb/sm/serialization/array_schema.h b/tiledb/sm/serialization/array_schema.h index 14b51d7b620..2fa3dff698b 100644 --- a/tiledb/sm/serialization/array_schema.h +++ b/tiledb/sm/serialization/array_schema.h @@ -52,14 +52,16 @@ class Buffer; class ArraySchema; class Dimension; class MemoryTracker; +class URI; enum class SerializationType : uint8_t; namespace serialization { class LoadArraySchemaRequest { public: - LoadArraySchemaRequest(bool include_enumerations = false) - : include_enumerations_(include_enumerations) { + explicit LoadArraySchemaRequest(const Config& config) + : include_enumerations_(config.get( + "rest.load_enumerations_on_array_open", Config::must_find)) { } inline bool include_enumerations() const { @@ -208,11 +210,13 @@ LoadArraySchemaRequest deserialize_load_array_schema_request( SerializationType serialization_type, const Buffer& data); void serialize_load_array_schema_response( - const ArraySchema& schema, - SerializationType serialization_type, - Buffer& data); + const Array& array, SerializationType serialization_type, Buffer& data); -shared_ptr deserialize_load_array_schema_response( +std::tuple< + shared_ptr, + std::unordered_map>> +deserialize_load_array_schema_response( + const URI& uri, SerializationType serialization_type, const Buffer& data, shared_ptr memory_tracker); diff --git a/tiledb/sm/serialization/tiledb-rest.capnp b/tiledb/sm/serialization/tiledb-rest.capnp index a8a7fc1255e..168cba4cdf9 100644 --- a/tiledb/sm/serialization/tiledb-rest.capnp +++ b/tiledb/sm/serialization/tiledb-rest.capnp @@ -1267,11 +1267,16 @@ struct LoadArraySchemaRequest { includeEnumerations @1 :Bool; # When true, include all enumeration data in the returned ArraySchema + # This field is only serialized for backwards compatibility. Future options + # that modify array schema load behavior should be handled within the Config. } struct LoadArraySchemaResponse { schema @0 :ArraySchema; # The loaded ArraySchema + + arraySchemasAll @1 :Map(Text, ArraySchema); + # map of all Array Schemas } struct QueryPlanRequest { diff --git a/tiledb/sm/serialization/tiledb-rest.capnp.c++ b/tiledb/sm/serialization/tiledb-rest.capnp.c++ index f04316412c5..bc5d31a2b56 100644 --- a/tiledb/sm/serialization/tiledb-rest.capnp.c++ +++ b/tiledb/sm/serialization/tiledb-rest.capnp.c++ @@ -9797,17 +9797,17 @@ const ::capnp::_::RawSchema s_83f094010132ff21 = { 1, 2, i_83f094010132ff21, nullptr, nullptr, { &s_83f094010132ff21, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<35> b_ebe17f59ac9a1df1 = { +static const ::capnp::_::AlignedData<69> b_ebe17f59ac9a1df1 = { { 0, 0, 0, 0, 5, 0, 6, 0, 241, 29, 154, 172, 89, 127, 225, 235, 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, - 1, 0, 7, 0, 0, 0, 0, 0, + 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 82, 1, 0, 0, 41, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 37, 0, 0, 0, 63, 0, 0, 0, + 37, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, @@ -9817,16 +9817,50 @@ static const ::capnp::_::AlignedData<35> b_ebe17f59ac9a1df1 = { 97, 82, 101, 115, 112, 111, 110, 115, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, - 4, 0, 0, 0, 3, 0, 4, 0, + 8, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 13, 0, 0, 0, 58, 0, 0, 0, + 41, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 8, 0, 0, 0, 3, 0, 1, 0, - 20, 0, 0, 0, 2, 0, 1, 0, + 36, 0, 0, 0, 3, 0, 1, 0, + 48, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 45, 0, 0, 0, 130, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 44, 0, 0, 0, 3, 0, 1, 0, + 128, 0, 0, 0, 2, 0, 1, 0, 115, 99, 104, 101, 109, 97, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, + 254, 150, 226, 152, 47, 227, 29, 215, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 97, 114, 114, 97, 121, 83, 99, 104, + 101, 109, 97, 115, 65, 108, 108, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 140, 113, 113, 174, 148, 193, 121, 241, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1, 0, + 1, 0, 0, 0, 31, 0, 0, 0, + 4, 0, 0, 0, 2, 0, 1, 0, + 140, 113, 113, 174, 148, 193, 121, 241, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 39, 0, 0, 0, + 8, 0, 0, 0, 1, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 3, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 3, 0, 1, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, 254, 150, 226, 152, 47, 227, 29, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -9838,12 +9872,16 @@ static const ::capnp::_::AlignedData<35> b_ebe17f59ac9a1df1 = { #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_ebe17f59ac9a1df1[] = { &s_d71de32f98e296fe, + &s_f179c194ae71718c, +}; +static const uint16_t m_ebe17f59ac9a1df1[] = {1, 0}; +static const uint16_t i_ebe17f59ac9a1df1[] = {0, 1}; +KJ_CONSTEXPR(const) ::capnp::_::RawBrandedSchema::Dependency bd_ebe17f59ac9a1df1[] = { + { 16777217, ::tiledb::sm::serialization::capnp::Map< ::capnp::Text, ::tiledb::sm::serialization::capnp::ArraySchema>::_capnpPrivate::brand() }, }; -static const uint16_t m_ebe17f59ac9a1df1[] = {0}; -static const uint16_t i_ebe17f59ac9a1df1[] = {0}; const ::capnp::_::RawSchema s_ebe17f59ac9a1df1 = { - 0xebe17f59ac9a1df1, b_ebe17f59ac9a1df1.words, 35, d_ebe17f59ac9a1df1, m_ebe17f59ac9a1df1, - 1, 1, i_ebe17f59ac9a1df1, nullptr, nullptr, { &s_ebe17f59ac9a1df1, nullptr, nullptr, 0, 0, nullptr }, false + 0xebe17f59ac9a1df1, b_ebe17f59ac9a1df1.words, 69, d_ebe17f59ac9a1df1, m_ebe17f59ac9a1df1, + 2, 2, i_ebe17f59ac9a1df1, nullptr, nullptr, { &s_ebe17f59ac9a1df1, nullptr, bd_ebe17f59ac9a1df1, 0, sizeof(bd_ebe17f59ac9a1df1) / sizeof(bd_ebe17f59ac9a1df1[0]), nullptr }, true }; #endif // !CAPNP_LITE static const ::capnp::_::AlignedData<49> b_e06f571aa93eb314 = { diff --git a/tiledb/sm/serialization/tiledb-rest.capnp.h b/tiledb/sm/serialization/tiledb-rest.capnp.h index 2035123aeab..518bab1543f 100644 --- a/tiledb/sm/serialization/tiledb-rest.capnp.h +++ b/tiledb/sm/serialization/tiledb-rest.capnp.h @@ -1684,7 +1684,7 @@ struct LoadArraySchemaResponse { class Pipeline; struct _capnpPrivate { - CAPNP_DECLARE_STRUCT_HEADER(ebe17f59ac9a1df1, 0, 1) + CAPNP_DECLARE_STRUCT_HEADER(ebe17f59ac9a1df1, 0, 2) #if !CAPNP_LITE static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; @@ -15033,6 +15033,12 @@ class LoadArraySchemaResponse::Reader { inline ::tiledb::sm::serialization::capnp::ArraySchema::Reader getSchema() const; + inline bool hasArraySchemasAll() const; + inline ::tiledb::sm::serialization::capnp::Map< + ::capnp::Text, + ::tiledb::sm::serialization::capnp::ArraySchema>::Reader + getArraySchemasAll() const; + private: ::capnp::_::StructReader _reader; template @@ -15082,6 +15088,28 @@ class LoadArraySchemaResponse::Builder { inline ::capnp::Orphan<::tiledb::sm::serialization::capnp::ArraySchema> disownSchema(); + inline bool hasArraySchemasAll(); + inline ::tiledb::sm::serialization::capnp::Map< + ::capnp::Text, + ::tiledb::sm::serialization::capnp::ArraySchema>::Builder + getArraySchemasAll(); + inline void setArraySchemasAll( + ::tiledb::sm::serialization::capnp::Map< + ::capnp::Text, + ::tiledb::sm::serialization::capnp::ArraySchema>::Reader value); + inline ::tiledb::sm::serialization::capnp::Map< + ::capnp::Text, + ::tiledb::sm::serialization::capnp::ArraySchema>::Builder + initArraySchemasAll(); + inline void adoptArraySchemasAll( + ::capnp::Orphan<::tiledb::sm::serialization::capnp::Map< + ::capnp::Text, + ::tiledb::sm::serialization::capnp::ArraySchema>>&& value); + inline ::capnp::Orphan<::tiledb::sm::serialization::capnp::Map< + ::capnp::Text, + ::tiledb::sm::serialization::capnp::ArraySchema>> + disownArraySchemasAll(); + private: ::capnp::_::StructBuilder _builder; template @@ -15104,6 +15132,10 @@ class LoadArraySchemaResponse::Pipeline { } inline ::tiledb::sm::serialization::capnp::ArraySchema::Pipeline getSchema(); + inline ::tiledb::sm::serialization::capnp::Map< + ::capnp::Text, + ::tiledb::sm::serialization::capnp::ArraySchema>::Pipeline + getArraySchemasAll(); private: ::capnp::AnyPointer::Pipeline _typeless; @@ -33186,6 +33218,80 @@ LoadArraySchemaResponse::Builder::disownSchema() { _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); } +inline bool LoadArraySchemaResponse::Reader::hasArraySchemasAll() const { + return !_reader.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS) + .isNull(); +} +inline bool LoadArraySchemaResponse::Builder::hasArraySchemasAll() { + return !_builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS) + .isNull(); +} +inline ::tiledb::sm::serialization::capnp:: + Map<::capnp::Text, ::tiledb::sm::serialization::capnp::ArraySchema>::Reader + LoadArraySchemaResponse::Reader::getArraySchemasAll() const { + return ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Map< + ::capnp::Text, + ::tiledb::sm::serialization::capnp::ArraySchema>>:: + get(_reader.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); +} +inline ::tiledb::sm::serialization::capnp:: + Map<::capnp::Text, ::tiledb::sm::serialization::capnp::ArraySchema>::Builder + LoadArraySchemaResponse::Builder::getArraySchemasAll() { + return ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Map< + ::capnp::Text, + ::tiledb::sm::serialization::capnp::ArraySchema>>:: + get(_builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); +} +#if !CAPNP_LITE +inline ::tiledb::sm::serialization::capnp::Map< + ::capnp::Text, + ::tiledb::sm::serialization::capnp::ArraySchema>::Pipeline +LoadArraySchemaResponse::Pipeline::getArraySchemasAll() { + return ::tiledb::sm::serialization::capnp:: + Map<::capnp::Text, ::tiledb::sm::serialization::capnp::ArraySchema>:: + Pipeline(_typeless.getPointerField(1)); +} +#endif // !CAPNP_LITE +inline void LoadArraySchemaResponse::Builder::setArraySchemasAll( + ::tiledb::sm::serialization::capnp::Map< + ::capnp::Text, + ::tiledb::sm::serialization::capnp::ArraySchema>::Reader value) { + ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Map< + ::capnp::Text, + ::tiledb::sm::serialization::capnp::ArraySchema>>:: + set(_builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), + value); +} +inline ::tiledb::sm::serialization::capnp:: + Map<::capnp::Text, ::tiledb::sm::serialization::capnp::ArraySchema>::Builder + LoadArraySchemaResponse::Builder::initArraySchemasAll() { + return ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Map< + ::capnp::Text, + ::tiledb::sm::serialization::capnp::ArraySchema>>:: + init(_builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); +} +inline void LoadArraySchemaResponse::Builder::adoptArraySchemasAll( + ::capnp::Orphan<::tiledb::sm::serialization::capnp::Map< + ::capnp::Text, + ::tiledb::sm::serialization::capnp::ArraySchema>>&& value) { + ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Map< + ::capnp::Text, + ::tiledb::sm::serialization::capnp::ArraySchema>>:: + adopt( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::tiledb::sm::serialization::capnp::Map< + ::capnp::Text, + ::tiledb::sm::serialization::capnp::ArraySchema>> +LoadArraySchemaResponse::Builder::disownArraySchemasAll() { + return ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Map< + ::capnp::Text, + ::tiledb::sm::serialization::capnp::ArraySchema>>:: + disown( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); +} + inline bool QueryPlanRequest::Reader::hasConfig() const { return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) .isNull();