diff --git a/CMakeLists.txt b/CMakeLists.txt index 635ae3690ceb..38455849f61d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -227,7 +227,10 @@ else() "$<$:-O3>" "$<$:-g3;-ggdb3;-gdwarf-3>") if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") - add_compile_options("$<$:-fstandalone-debug>") + add_compile_options( + "$<$:-fstandalone-debug>" + "$<$:-fprofile-instr-generate;-fcoverage-mapping>") + add_link_options("$<$:--coverage;-fprofile-instr-generate;-fcoverage-mapping>") endif() # Disable newer Apple Clang warnings about unqualified calls to std::move @@ -344,6 +347,7 @@ list(APPEND TILEDB_C_API_RELATIVE_HEADERS "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/datatype/datatype_api_external.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/dimension/dimension_api_external.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/dimension_label/dimension_label_api_external.h" + "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/enumeration/enumeration_api_experimental.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/error/error_api_external.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/filesystem/filesystem_api_enum.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/filesystem/filesystem_api_external.h" diff --git a/PJD_TODO.md b/PJD_TODO.md new file mode 100644 index 000000000000..d67f3852a2c2 --- /dev/null +++ b/PJD_TODO.md @@ -0,0 +1,9 @@ +Chores Left +=== + +* NEED HELP + - New REST API endpoint for loading one or more enumerations + - Update get_array_schema_from_rest API endpoint to have a "?include_enumerations=true|false" QS parameter. + +* Miscellany + * Document enumeration format changes in the storage format docs diff --git a/examples/cpp_api/enumerations.cc b/examples/cpp_api/enumerations.cc new file mode 100644 index 000000000000..ea439d181c81 --- /dev/null +++ b/examples/cpp_api/enumerations.cc @@ -0,0 +1,160 @@ +/** + * @file enumerations.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * When run, this program will create a simple 2D sparse array with an + * enumeration and then use a query condition to select data based on the + * enumerations values. + */ + +#include +#include +#include + +using namespace tiledb; + +// Name of array. +std::string array_name("enumerations_example_array"); + +void create_array() { + Context ctx; + + // First some standard boiler plate for creating an array. Nothing here + // is important or required for Enumerations. + ArraySchema schema(ctx, TILEDB_SPARSE); + schema.set_order({{TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR}}); + + auto dim1 = Dimension::create(ctx, "rows", {{1, 4}}, 4); + auto dim2 = Dimension::create(ctx, "cols", {{1, 4}}, 4); + + Domain dom(ctx); + dom.add_dimensions(dim1, dim2); + schema.set_domain(dom); + + // The most basic enumeration only requires a name and a vector of values + // to use as lookups. Enumeration values can be any supported TileDB type + // although they are most commonly strings. + std::vector values = {"red", "green", "blue"}; + auto enmr = Enumeration::create(ctx, "colors", values); + + // To use an enumeration with an attribute, we just set the enumeration + // name on the attribute before adding it to the schema. Attributes that + // use an enumeration are required to have an integral type that is wide + // enough to index the entire enumeration. For instance, an enumeration with + // 256 values can fit in a uint8_t type, but at 257 values, the attribute + // would require a type of int16_t at a minimum. + auto attr = Attribute::create(ctx, "attr"); + AttributeExperimental::set_enumeration_name(ctx, attr, "colors"); + + // The enumeration must be added to the schema before any attribute that + // references the enumeration so that the requirements of the attribute + // can be accurately checked. + ArraySchemaExperimental::add_enumeration(ctx, schema, enmr); + + // Finally, we add the attribute as per normal. + schema.add_attribute(attr); + + Array::create(array_name, schema); +} + +void write_array() { + Context ctx; + + std::vector row_data = {1, 2, 2}; + std::vector col_data = {1, 4, 3}; + + // Attribute data for an enumeration is just numeric indices into the + // list of enumeration values. + std::vector attr_data = {2, 1, 1}; + + // Open the array for writing and create the query. + Array array(ctx, array_name, TILEDB_WRITE); + Query query(ctx, array); + query.set_layout(TILEDB_UNORDERED) + .set_data_buffer("rows", row_data) + .set_data_buffer("cols", col_data) + .set_data_buffer("attr", attr_data); + + // Write and close the array + query.submit(); + array.close(); +} + +void read_array() { + Context ctx; + + // This is all standard boiler plate for reading from an array. The + // section below will demonstrate using a QueryCondition to select + // rows based on the enumeration. + Array array(ctx, array_name, TILEDB_READ); + Subarray subarray(ctx, array); + subarray.add_range(0, 1, 4).add_range(1, 1, 4); + + std::vector row_data(16); + std::vector col_data(16); + std::vector attr_data(16); + + Query query(ctx, array, TILEDB_READ); + query.set_subarray(subarray) + .set_layout(TILEDB_ROW_MAJOR) + .set_data_buffer("rows", row_data) + .set_data_buffer("cols", col_data) + .set_data_buffer("attr", attr_data); + + // Query conditions apply against the enumeration's values instead of the + // integral data. Thus, we can select values here using color names instead + // of the integer indices. + QueryCondition qc(ctx); + qc.init("attr", "green", 5, TILEDB_EQ); + query.set_condition(qc); + + // Finally, submit and display the query results + query.submit(); + array.close(); + + // Print out the results. + auto result_num = (int)query.result_buffer_elements()["attr"].second; + for (int i = 0; i < result_num; i++) { + int r = row_data[i]; + int c = col_data[i]; + int a = attr_data[i]; + std::cout << "Cell (" << r << ", " << c << ") has attr " << a << "\n"; + } +} + +int main() { + Context ctx; + + if (Object::object(ctx, array_name).type() != Object::Type::Array) { + create_array(); + write_array(); + } + + read_array(); + return 0; +} diff --git a/format_spec/FORMAT_SPEC.md b/format_spec/FORMAT_SPEC.md index 42e16b15a986..ee0db16d584e 100644 --- a/format_spec/FORMAT_SPEC.md +++ b/format_spec/FORMAT_SPEC.md @@ -4,7 +4,7 @@ title: Format Specification **Notes:** -* The current TileDB format version number is **18** (`uint32_t`). +* The current TileDB format version number is **20** (`uint32_t`). * Data written by TileDB and referenced in this document is **little-endian** with the following exceptions: diff --git a/format_spec/enumeration.md b/format_spec/enumeration.md new file mode 100644 index 000000000000..a334a6632340 --- /dev/null +++ b/format_spec/enumeration.md @@ -0,0 +1,27 @@ +--- +title: Enumerations +--- + +## Main Structure + +``` +my_array # array folder + | ... + |_ schema # ArraySchema directory named `__schema` + |_ enumerations # Enumeration directory named `__enumerations` + |_ enumeration # enumeration data with names `__uuid_v` + + +Enumeration data is stored in a subdirectory of the [array schema][./array_schema.md] +directory. Enumerations are stored using [Generic Tiles][./generic_tile.md]. + +| **Field** | **Type** | **Description** | +| :--- | :--- | :--- | +| Version number | `uint32_t` | Enumerations version number | +| Datatype | `uint8_t` | The datatype of the enumeration values | +| Cell Val Num | `uint32_t` | The cell val num of the enumeration values | +| Ordered | `bool` | Whether the enumeration values should be considered ordered | +| Data Size | `uint64_t` | The number of bytes used to store the values | +| Data | `uint8_t` * Data Size | The data for the enumeration values | +| Offsets Size | `uint64_t` | The number of bytes used to store offsets if cell_var_num is TILEDB_VAR_NUM | +| Offsets | `uint8_t` * Offsets Size | The offsets data for the enumeration if cell_var_num is TILEDB_VAR_NUM | diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index c8fe3287492b..b58b15a78694 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -134,6 +134,7 @@ set(TILEDB_UNIT_TEST_SOURCES src/unit-capi-dense_neg.cc src/unit-capi-dense_vector.cc src/unit-capi-enum_values.cc + src/unit-capi-enumerations.cc src/unit-capi-error.cc src/unit-capi-filestore.cc src/unit-capi-fill_values.cc @@ -172,6 +173,8 @@ set(TILEDB_UNIT_TEST_SOURCES src/unit-dimension.cc src/unit-duplicates.cc src/unit-empty-var-length.cc + src/unit-enumerations.cc + src/unit-enum-helpers.cc src/unit-filter-buffer.cc src/unit-filter-pipeline.cc src/unit-global-order.cc @@ -213,6 +216,7 @@ if (TILEDB_CPP_API) src/unit-cppapi-deletes.cc src/unit-cppapi-dense-qc-coords-mode.cc src/unit-cppapi-time.cc + src/unit-cppapi-enumerations.cc src/unit-cppapi-fill_values.cc src/unit-cppapi-filter.cc src/unit-cppapi-float-scaling-filter.cc diff --git a/test/src/unit-capi-array_schema.cc b/test/src/unit-capi-array_schema.cc index 7a16a681a0ef..043e8b8bc471 100644 --- a/test/src/unit-capi-array_schema.cc +++ b/test/src/unit-capi-array_schema.cc @@ -959,7 +959,10 @@ int ArraySchemaFx::get_schema_file_struct(const char* path, void* data) { int rc = tiledb_vfs_is_dir(ctx, vfs, path, &is_dir); CHECK(rc == TILEDB_OK); - data_struct->path = path; + if (!is_dir) { + data_struct->path = path; + } + return 1; } diff --git a/test/src/unit-capi-enumerations.cc b/test/src/unit-capi-enumerations.cc new file mode 100644 index 000000000000..58add24ea2f0 --- /dev/null +++ b/test/src/unit-capi-enumerations.cc @@ -0,0 +1,119 @@ +/** + * @file unit-capi-enumerations.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * Tests for the various Enumerations C API errors. + */ + +#include +#include "tiledb/sm/c_api/tiledb.h" +#include "tiledb/sm/c_api/tiledb_experimental.h" + +#include + +TEST_CASE( + "C API: Test invalid attribute for tiledb_attribute_set_enumeration_name", + "[enumeration][capi][error]") { + tiledb_ctx_t* ctx; + + auto rc = tiledb_ctx_alloc(nullptr, &ctx); + CHECK(rc == TILEDB_OK); + + rc = tiledb_attribute_set_enumeration_name(ctx, nullptr, "enmr_name"); + REQUIRE(rc == TILEDB_ERR); +} + +TEST_CASE( + "C API: Test invalid attribute for tiledb_attribute_get_enumeration_name", + "[enumeration][capi][error]") { + tiledb_ctx_t* ctx; + + auto rc = tiledb_ctx_alloc(nullptr, &ctx); + CHECK(rc == TILEDB_OK); + + tiledb_string_t* name; + rc = tiledb_attribute_get_enumeration_name(ctx, nullptr, &name); + REQUIRE(rc == TILEDB_ERR); +} + +TEST_CASE( + "C API: Test invalid array schema for tiledb_array_schema_add_enumeration", + "[enumeration][capi][error]") { + tiledb_ctx_t* ctx; + + auto rc = tiledb_ctx_alloc(nullptr, &ctx); + CHECK(rc == TILEDB_OK); + + tiledb_enumeration_t* enmr; + uint32_t values[5] = {1, 2, 3, 4, 5}; + rc = tiledb_enumeration_alloc( + ctx, + "an_enumeration", + TILEDB_UINT32, + 1, + 0, + values, + sizeof(uint32_t) * 5, + nullptr, + 0, + &enmr); + REQUIRE(rc == TILEDB_OK); + + rc = tiledb_array_schema_add_enumeration(ctx, nullptr, enmr); + REQUIRE(rc == TILEDB_ERR); +} + +TEST_CASE( + "C API: Test invalid array for tiledb_array_get_enumeration", + "[enumeration][capi][error]") { + tiledb_ctx_t* ctx; + + auto rc = tiledb_ctx_alloc(nullptr, &ctx); + CHECK(rc == TILEDB_OK); + + tiledb_enumeration_t* enmr; + rc = tiledb_array_get_enumeration(ctx, nullptr, "an_enumeration", &enmr); + REQUIRE(rc == TILEDB_ERR); +} + +TEST_CASE( + "C API: Test invalid enumeration name for tiledb_array_get_enumeration", + "[enumeration][capi][error]") { + tiledb_ctx_t* ctx; + + auto rc = tiledb_ctx_alloc(nullptr, &ctx); + CHECK(rc == TILEDB_OK); + + tiledb_array_t* array; + rc = tiledb_array_alloc(ctx, "array_uri", &array); + REQUIRE(rc == TILEDB_OK); + + tiledb_enumeration_t* enmr; + rc = tiledb_array_get_enumeration(ctx, array, nullptr, &enmr); + REQUIRE(rc == TILEDB_ERR); +} diff --git a/test/src/unit-cppapi-deletes.cc b/test/src/unit-cppapi-deletes.cc index a27b83a7c418..b61292dfc0d2 100644 --- a/test/src/unit-cppapi-deletes.cc +++ b/test/src/unit-cppapi-deletes.cc @@ -111,6 +111,7 @@ struct DeletesFx { bool is_array(const std::string& array_name); void validate_array_dir_after_delete(const std::string& path); void validate_group_dir_after_delete(const std::string& path); + std::vector list_schemas(const std::string& array_name); }; DeletesFx::DeletesFx() @@ -507,6 +508,29 @@ void DeletesFx::validate_group_dir_after_delete(const std::string& path) { REQUIRE(!vfs_.is_dir(path + tiledb::sm::constants::group_metadata_dir_name)); } +std::vector DeletesFx::list_schemas( + const std::string& array_name) { + auto& enum_dir = tiledb::sm::constants::array_enumerations_dir_name; + auto schemas = + vfs_.ls(array_name + tiledb::sm::constants::array_schema_dir_name); + + auto it = schemas.begin(); + while (it != schemas.end()) { + if ((*it).size() < enum_dir.size()) { + continue; + } + if ((*it).substr((*it).size() - enum_dir.size()) == enum_dir) { + break; + } + ++it; + } + if (it != schemas.end()) { + schemas.erase(it); + } + + return schemas; +} + TEST_CASE_METHOD( DeletesFx, "CPP API: Test writing delete condition", @@ -1968,8 +1992,7 @@ TEST_CASE_METHOD( // Check write CHECK(tiledb::test::num_commits(SPARSE_ARRAY_NAME) == 4); CHECK(tiledb::test::num_fragments(SPARSE_ARRAY_NAME) == 4); - auto schemas = - vfs_.ls(array_name + tiledb::sm::constants::array_schema_dir_name); + auto schemas = list_schemas(array_name); CHECK(schemas.size() == 1); auto meta = vfs_.ls(array_name + tiledb::sm::constants::array_metadata_dir_name); @@ -2040,8 +2063,7 @@ TEST_CASE_METHOD( vfs_.touch(extraneous_file_path); // Check write - auto schemas = - vfs_.ls(array_name + tiledb::sm::constants::array_schema_dir_name); + auto schemas = list_schemas(array_name); CHECK(schemas.size() == 1); auto uris = vfs_.ls(array_name); bool ok_exists = false; @@ -2259,11 +2281,9 @@ TEST_CASE_METHOD( auto group_meta_dir = vfs_.ls(GROUP_NAME + tiledb::sm::constants::group_metadata_dir_name); CHECK(group_meta_dir.size() == 1); - auto array_schema = - vfs_.ls(array_path + tiledb::sm::constants::array_schema_dir_name); + auto array_schema = list_schemas(array_path); CHECK(array_schema.size() == 1); - auto array2_schema = - vfs_.ls(array2_path + tiledb::sm::constants::array_schema_dir_name); + auto array2_schema = list_schemas(array2_path); CHECK(array2_schema.size() == 1); // Recursively delete group in modify exclusive mode diff --git a/test/src/unit-cppapi-enumerations.cc b/test/src/unit-cppapi-enumerations.cc new file mode 100644 index 000000000000..677e3715f72c --- /dev/null +++ b/test/src/unit-cppapi-enumerations.cc @@ -0,0 +1,501 @@ +/** + * @file unit-cppapi-enumeration.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * Tests the C++ API for enumeration related functions. + */ + +#include + +#include +#include "tiledb/sm/array_schema/array_schema.h" +#include "tiledb/sm/c_api/tiledb_struct_def.h" +#include "tiledb/sm/cpp_api/tiledb" +#include "tiledb/sm/cpp_api/tiledb_experimental" + +using namespace tiledb; + +struct CPPEnumerationFx { + CPPEnumerationFx(); + ~CPPEnumerationFx(); + + template + void check_dump(const T& val); + + void create_array(); + void rm_array(); + + std::string uri_; + Context ctx_; + VFS vfs_; +}; + +const static std::string enmr_name = "an_enumeration"; +const static std::string dump_name = "enumeration_dump_test.txt"; + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Enumeration API - Create Fixed Size", + "[enumeration][create][fixed-size]") { + std::vector values = {1, 2, 3, 4, 5}; + auto enmr = Enumeration::create(ctx_, enmr_name, values); + REQUIRE(enmr.ptr() != nullptr); + REQUIRE(enmr.name() == enmr_name); + REQUIRE(enmr.type() == TILEDB_INT32); + REQUIRE(enmr.cell_val_num() == 1); + REQUIRE(enmr.ordered() == false); + + auto data = enmr.as_vector(); + REQUIRE(data == values); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Enumeration API - Create Variable Size", + "[enumeration][create][var-size]") { + std::vector values = {"fee", "fi", "fo", "fum"}; + auto enmr = Enumeration::create(ctx_, enmr_name, values); + REQUIRE(enmr.ptr() != nullptr); + REQUIRE(enmr.name() == enmr_name); + REQUIRE(enmr.type() == TILEDB_STRING_ASCII); + REQUIRE(enmr.cell_val_num() == TILEDB_VAR_NUM); + REQUIRE(enmr.ordered() == false); + + auto data = enmr.as_vector(); + REQUIRE(data == values); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Enumeration API - Create Ordered", + "[enumeration][create][var-size]") { + std::vector values = {1, 2, 3, 4, 5}; + auto enmr = Enumeration::create(ctx_, enmr_name, values, true); + REQUIRE(enmr.ptr() != nullptr); + REQUIRE(enmr.name() == enmr_name); + REQUIRE(enmr.type() == TILEDB_INT32); + REQUIRE(enmr.cell_val_num() == 1); + REQUIRE(enmr.ordered() == true); + + auto data = enmr.as_vector(); + REQUIRE(data == values); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Enumeration API - Dump Basic", + "[enumeration][dump][basic]") { + std::vector values = {1, 2, 3, 4, 5}; + auto enmr = Enumeration::create(ctx_, enmr_name, values, true); + check_dump(enmr); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Add Enumeration to Schema", + "[enumeration][add-attribute]") { + ArraySchema schema(ctx_, TILEDB_DENSE); + + auto dim = Dimension::create(ctx_, "dim", {{-100, 100}}); + auto dom = Domain(ctx_); + dom.add_dimension(dim); + schema.set_domain(dom); + + std::vector values = {"fred", "wilma", "barney", "pebbles"}; + auto enmr = Enumeration::create(ctx_, enmr_name, values); + ArraySchemaExperimental::add_enumeration(ctx_, schema, enmr); + + auto attr = Attribute::create(ctx_, "attr"); + AttributeExperimental::set_enumeration_name(ctx_, attr, enmr_name); + schema.add_attribute(attr); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Load Schema from URI", + "[enumeration][add-attribute]") { + create_array(); + auto schema = ArraySchemaExperimental::load_with_enumerations(ctx_, uri_); + auto enmr_names = + schema.ptr().get()->array_schema_->get_loaded_enumeration_names(); + REQUIRE(enmr_names.size() > 0); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Load Schema from URI - REMOTE NOT SUPPORTED YET", + "[enumeration][add-attribute][fixme]") { + std::string uri = "tiledb://namespace/array_name"; + REQUIRE_THROWS(ArraySchemaExperimental::load_with_enumerations(ctx_, uri)); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Schema Dump With Enumeration", + "[enumeration][array-schema][dump]") { + ArraySchema schema(ctx_, TILEDB_DENSE); + + auto dim = Dimension::create(ctx_, "dim", {{-100, 100}}); + auto dom = Domain(ctx_); + dom.add_dimension(dim); + schema.set_domain(dom); + + std::vector values = {"fred", "wilma", "barney", "pebbles"}; + auto enmr = Enumeration::create(ctx_, enmr_name, values); + ArraySchemaExperimental::add_enumeration(ctx_, schema, enmr); + + auto attr = Attribute::create(ctx_, "attr"); + AttributeExperimental::set_enumeration_name(ctx_, attr, enmr_name); + schema.add_attribute(attr); + + check_dump(schema); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Enumerations From Disk - Array::get_enumeration", + "[enumeration][array-get-enumeration]") { + create_array(); + auto array = Array(ctx_, uri_, TILEDB_READ); + auto enmr = ArrayExperimental::get_enumeration(ctx_, array, "an_enumeration"); + REQUIRE(enmr.ptr() != nullptr); + REQUIRE(enmr.name() == "an_enumeration"); + REQUIRE(enmr.type() == TILEDB_STRING_ASCII); + REQUIRE(enmr.cell_val_num() == TILEDB_VAR_NUM); + REQUIRE(enmr.ordered() == false); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Enumerations From Disk - Attribute::get_enumeration_name", + "[enumeration][attr-get-enumeration-name]") { + create_array(); + auto schema = Array::load_schema(ctx_, uri_); + + auto attr1 = schema.attribute("attr1"); + auto enmr_name1 = AttributeExperimental::get_enumeration_name(ctx_, attr1); + REQUIRE(enmr_name1.has_value() == true); + + auto attr2 = schema.attribute("attr2"); + auto enmr_name2 = AttributeExperimental::get_enumeration_name(ctx_, attr2); + REQUIRE(enmr_name2.has_value() == false); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Array::load_all_enumerations", + "[enumeration][array-load-all-enumerations]") { + create_array(); + auto array = Array(ctx_, uri_, TILEDB_READ); + REQUIRE_NOTHROW(ArrayExperimental::load_all_enumerations(ctx_, array)); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "C API: Array load_all_enumerations - Check nullptr", + "[enumeration][array-load-all-enumerations]") { + auto rc = tiledb_array_load_all_enumerations(ctx_.ptr().get(), nullptr, 0); + REQUIRE(rc != TILEDB_OK); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: ArraySchemaEvolution - Add Enumeration", + "[enumeration][array-schema-evolution][add-enumeration]") { + ArraySchemaEvolution ase(ctx_); + std::vector values = {"fred", "wilma", "barney", "pebbles"}; + auto enmr = Enumeration::create(ctx_, enmr_name, values); + CHECK_NOTHROW(ase.add_enumeration(enmr)); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "C API: ArraySchemaEvolution - Add Enumeration - Check nullptr", + "[enumeration][array-schema-evolution][error]") { + auto rc = tiledb_array_schema_evolution_add_enumeration( + ctx_.ptr().get(), nullptr, nullptr); + REQUIRE(rc != TILEDB_OK); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: ArraySchemaEvolution - Drop Enumeration", + "[enumeration][array-schema-evolution][drop-enumeration]") { + ArraySchemaEvolution ase(ctx_); + CHECK_NOTHROW(ase.drop_enumeration("an_enumeration_name")); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "C API: ArraySchemaEvolution - Drop Enumeration - Check nullptr", + "[enumeration][array-schema-evolution][drop-enumeration]") { + auto rc = tiledb_array_schema_evolution_drop_enumeration( + ctx_.ptr().get(), nullptr, "foo"); + REQUIRE(rc != TILEDB_OK); + + ArraySchemaEvolution ase(ctx_); + rc = tiledb_array_schema_evolution_drop_enumeration( + ctx_.ptr().get(), ase.ptr().get(), nullptr); + REQUIRE(rc != TILEDB_OK); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Enumeration Query - Basic", + "[enumeration][query][basic]") { + // Basic smoke test. Check that a simple query condition applied against + // an array returns sane results. + create_array(); + + // Check attr1 == "fred" when attr1 is an enumeration. + QueryCondition qc(ctx_); + qc.init("attr1", "fred", 4, TILEDB_EQ); + + // Execute the query condition against the array + std::vector dim(5); + std::vector attr1(5); + + auto array = Array(ctx_, uri_, TILEDB_READ); + Query query(ctx_, array); + query.add_range("dim", 1, 5) + .set_layout(TILEDB_ROW_MAJOR) + .set_data_buffer("dim", dim) + .set_data_buffer("attr1", attr1) + .set_condition(qc); + REQUIRE(query.submit() == Query::Status::COMPLETE); + query.finalize(); + + // attr1 == "fred" in position 0 and position 4. + std::vector dim_expect = {1, 2, 3, 4, 5}; + std::vector attr1_expect = {0, INT32_MIN, INT32_MIN, INT32_MIN, 0}; + + REQUIRE(dim == dim_expect); + REQUIRE(attr1 == attr1_expect); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Enumeration Query - Negation", + "[enumeration][query][negation]") { + // Another basic query test, the only twist here is that we're checking + // that query condition negation works as expected. + create_array(); + + // Create a query condition attr1 == "fred" and then negate it so that + // we can verify rewriting a negated query works. + QueryCondition qc1(ctx_); + qc1.init("attr1", "fred", 4, TILEDB_EQ); + auto qc2 = qc1.negate(); + + // Execute a read query against the created array + std::vector dim(5); + std::vector attr1(5); + + auto array = Array(ctx_, uri_, TILEDB_READ); + Query query(ctx_, array); + query.add_range("dim", 1, 5) + .set_layout(TILEDB_ROW_MAJOR) + .set_data_buffer("dim", dim) + .set_data_buffer("attr1", attr1) + .set_condition(qc2); + REQUIRE(query.submit() == Query::Status::COMPLETE); + query.finalize(); + + // attr1 == "fred" in positions 0 and 4 so those values should not match + // and return the default fill values. + std::vector dim_expect = {1, 2, 3, 4, 5}; + std::vector attr1_expect = {INT32_MIN, 1, 2, 1, INT32_MIN}; + + REQUIRE(dim == dim_expect); + REQUIRE(attr1 == attr1_expect); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Enumeration Query - Combination", + "[enumeration][query][combination]") { + // Same test as before except using multi-condition query condtions + create_array(); + + // Create query condition: attr1 == "fred" OR attr2 = 3.0f + QueryCondition qc1(ctx_); + qc1.init("attr1", "fred", 4, TILEDB_EQ); + + QueryCondition qc2(ctx_); + float val = 3.0f; + qc2.init("attr2", &val, sizeof(float), TILEDB_EQ); + + QueryCondition qc3 = qc1.combine(qc2, TILEDB_OR); + + // Execute a query with the query condition + std::vector dim(5); + std::vector attr1(5); + std::vector attr2(5); + + auto array = Array(ctx_, uri_, TILEDB_READ); + Query query(ctx_, array); + query.add_range("dim", 1, 5) + .set_layout(TILEDB_ROW_MAJOR) + .set_data_buffer("dim", dim) + .set_data_buffer("attr1", attr1) + .set_data_buffer("attr2", attr2) + .set_condition(qc3); + REQUIRE(query.submit() == Query::Status::COMPLETE); + query.finalize(); + + // Check the results match the expected results. attr1 == "fred" in + // positions 0 and 4, while attr2 == 3.0f in position 2. + std::vector dim_expect = {1, 2, 3, 4, 5}; + std::vector attr1_expect = {0, INT32_MIN, 2, INT32_MIN, 0}; + std::vector attr2_expect = { + 1.0f, std::nanf(""), 3.0f, std::nanf(""), 5.0f}; + + REQUIRE(dim == dim_expect); + REQUIRE(attr1 == attr1_expect); + + // NaN != NaN so we have to loop over the attr2 results to special case + // `isnan(v1) == isnan(v2)` when NaN is encountered the expect vector. + for (size_t i = 0; i < attr2_expect.size(); i++) { + if (std::isnan(attr2_expect[i])) { + REQUIRE(std::isnan(attr2[i])); + } else { + REQUIRE(attr2[i] == attr2_expect[i]); + } + } +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Enumeration Query - Set Use Enumeration", + "[enumeration][query][set-use-enumeration]") { + QueryCondition qc(ctx_); + qc.init("attr1", "fred", 4, TILEDB_EQ); + REQUIRE_NOTHROW( + QueryConditionExperimental::set_use_enumeration(ctx_, qc, true)); + REQUIRE_NOTHROW( + QueryConditionExperimental::set_use_enumeration(ctx_, qc, false)); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "C API: Enumeration Query - Check nullptr", + "[enumeration][query][check-nullptr]") { + auto rc = + tiledb_query_condition_set_use_enumeration(ctx_.ptr().get(), nullptr, 0); + REQUIRE(rc != TILEDB_OK); +} + +CPPEnumerationFx::CPPEnumerationFx() + : uri_("enumeration_test_array") + , vfs_(ctx_) { + rm_array(); +} + +CPPEnumerationFx::~CPPEnumerationFx() { + rm_array(); +} + +template +void CPPEnumerationFx::check_dump(const T& val) { + FILE* handle = fopen(dump_name.c_str(), "w"); + REQUIRE(handle != nullptr); + val.dump(handle); + fclose(handle); + + std::stringstream ss; + + // Scoped in an anonymous block to ensure that rstream closes before + // we attempt to delete the file for cleanup. + { + std::ifstream rstream(dump_name); + if (rstream.is_open()) { + ss << rstream.rdbuf(); + } + } + + auto data = ss.str(); + auto iter = data.find("Enumeration"); + REQUIRE(iter != std::string::npos); + + vfs_.remove_file(dump_name); +} + +void CPPEnumerationFx::create_array() { + // Create a simple array for testing. This ends up with just five elements in + // the array. dim is an int32_t dimension, attr1 is an enumeration with string + // values and int32_t attribute values. attr2 is a float attribute. + // + // The array data is summarized as below, however, pay attention to the fact + // that attr1 is storing integral index values instead of the raw string data. + // + // dim = {1, 2, 3, 4, 5} + // attr1 = {"fred", "wilma", "barney", "wilma", "fred"} + // attr2 = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f} + ArraySchema schema(ctx_, TILEDB_DENSE); + + auto dim = Dimension::create(ctx_, "dim", {{-100, 100}}); + auto dom = Domain(ctx_); + dom.add_dimension(dim); + schema.set_domain(dom); + + // The list of string values in the attr1 enumeration + std::vector values = {"fred", "wilma", "barney", "pebbles"}; + auto enmr = Enumeration::create(ctx_, "an_enumeration", values); + ArraySchemaExperimental::add_enumeration(ctx_, schema, enmr); + + auto attr1 = Attribute::create(ctx_, "attr1"); + AttributeExperimental::set_enumeration_name(ctx_, attr1, "an_enumeration"); + schema.add_attribute(attr1); + + auto attr2 = Attribute::create(ctx_, "attr2"); + schema.add_attribute(attr2); + + Array::create(uri_, schema); + + // Attribute data + std::vector attr1_values = {0, 1, 2, 1, 0}; + std::vector attr2_values = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f}; + + Array array(ctx_, uri_, TILEDB_WRITE); + Subarray subarray(ctx_, array); + subarray.set_subarray({1, 5}); + Query query(ctx_, array); + query.set_subarray(subarray) + .set_layout(TILEDB_ROW_MAJOR) + .set_data_buffer("attr1", attr1_values) + .set_data_buffer("attr2", attr2_values); + CHECK_NOTHROW(query.submit()); + query.finalize(); + array.close(); +} + +void CPPEnumerationFx::rm_array() { + if (vfs_.is_dir(uri_)) { + vfs_.remove_dir(uri_); + } +} diff --git a/test/src/unit-cppapi-schema-evolution.cc b/test/src/unit-cppapi-schema-evolution.cc index 73d847725c6a..a291bfc488e5 100644 --- a/test/src/unit-cppapi-schema-evolution.cc +++ b/test/src/unit-cppapi-schema-evolution.cc @@ -31,8 +31,15 @@ */ #include +#include "tiledb/sm/array_schema/array_schema.h" +#include "tiledb/sm/array_schema/array_schema_evolution.h" +#include "tiledb/sm/array_schema/attribute.h" +#include "tiledb/sm/array_schema/dimension.h" +#include "tiledb/sm/array_schema/domain.h" #include "tiledb/sm/cpp_api/tiledb" #include "tiledb/sm/cpp_api/tiledb_experimental" +#include "tiledb/sm/enums/array_type.h" +#include "tiledb/sm/enums/datatype.h" #include "tiledb/sm/misc/constants.h" #include @@ -776,3 +783,31 @@ TEST_CASE( vfs.remove_dir(array_uri); } } + +TEST_CASE( + "SchemaEvolution Error Handling Tests", + "[cppapi][schema][evolution][errors]") { + auto ase = make_shared(HERE()); + REQUIRE_THROWS(ase->evolve_schema(nullptr)); + REQUIRE_THROWS(ase->add_attribute(nullptr)); + + auto attr = make_shared( + HERE(), "attr", tiledb::sm::Datatype::STRING_ASCII); + ase->add_attribute(attr.get()); + REQUIRE_THROWS(ase->add_attribute(attr.get())); + + ase->set_timestamp_range(std::make_pair(1, 1)); + + auto schema = make_shared( + HERE(), tiledb::sm::ArrayType::SPARSE); + auto dim = make_shared( + HERE(), "dim1", tiledb::sm::Datatype::INT32); + int range[2] = {0, 1000}; + throw_if_not_ok(dim->set_domain(range)); + + auto dom = make_shared(HERE()); + throw_if_not_ok(dom->add_dimension(dim)); + throw_if_not_ok(schema->set_domain(dom)); + + CHECK_NOTHROW(ase->evolve_schema(schema)); +} diff --git a/test/src/unit-enum-helpers.cc b/test/src/unit-enum-helpers.cc new file mode 100644 index 000000000000..555b8f993707 --- /dev/null +++ b/test/src/unit-enum-helpers.cc @@ -0,0 +1,63 @@ +/** + * @file unit-enum-helpers.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * Tests for enum conversions. + */ + +#include + +#include "test/support/tdb_catch.h" +#include "tiledb/sm/enums/datatype.h" + +using namespace tiledb::sm; + +TEST_CASE( + "Test datatype_max_integral_value", + "[enums][datatype][max-integral-value]") { + std::vector> test_data = { + {Datatype::BOOL, 255}, + {Datatype::INT8, 127}, + {Datatype::INT16, 32767}, + {Datatype::INT32, 2147483647}, + {Datatype::INT64, 9223372036854775807}, + + {Datatype::UINT8, 255}, + {Datatype::UINT16, 65535}, + {Datatype::UINT32, 4294967295}, + {Datatype::UINT64, 18446744073709551615u}}; + + for (auto& [dtype, max_size] : test_data) { + std::cerr << "DTYPE: " << datatype_str(dtype) << std::endl; + REQUIRE(datatype_max_integral_value(dtype) == max_size); + } + + REQUIRE_THROWS(datatype_max_integral_value(Datatype::BLOB)); + REQUIRE_THROWS(datatype_max_integral_value(Datatype::FLOAT64)); + REQUIRE_THROWS(datatype_max_integral_value(Datatype::STRING_ASCII)); +} diff --git a/test/src/unit-enumerations.cc b/test/src/unit-enumerations.cc new file mode 100644 index 000000000000..062e321efb50 --- /dev/null +++ b/test/src/unit-enumerations.cc @@ -0,0 +1,1995 @@ +/** + * @file unit-enumerations.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * Tests the C++ API for enumeration related functions. + */ + +#include + +#include "test/support/tdb_catch.h" +#include "tiledb/sm/array/array.h" +#include "tiledb/sm/array/array_directory.h" +#include "tiledb/sm/array_schema/array_schema.h" +#include "tiledb/sm/array_schema/array_schema_evolution.h" +#include "tiledb/sm/array_schema/attribute.h" +#include "tiledb/sm/array_schema/dimension.h" +#include "tiledb/sm/array_schema/domain.h" +#include "tiledb/sm/array_schema/enumeration.h" +#include "tiledb/sm/buffer/buffer.h" +#include "tiledb/sm/buffer/buffer_list.h" +#include "tiledb/sm/config/config.h" +#include "tiledb/sm/enums/array_type.h" +#include "tiledb/sm/enums/encryption_type.h" +#include "tiledb/sm/enums/layout.h" +#include "tiledb/sm/query/query.h" +#include "tiledb/sm/query/query_condition.h" +#include "tiledb/sm/storage_manager/context.h" + +#ifdef TILEDB_SERIALIZATION +#include "tiledb/sm/enums/serialization_type.h" +#include "tiledb/sm/serialization/array_schema.h" +#include "tiledb/sm/serialization/array_schema_evolution.h" +#include "tiledb/sm/serialization/query.h" +#endif + +using namespace tiledb::sm; + +#define ENUM_NAME_PTR "an_enumeration" +const static std::string default_enmr_name = ENUM_NAME_PTR; + +struct EnumerationFx { + EnumerationFx(); + ~EnumerationFx(); + + template + shared_ptr create_enumeration( + const std::vector& values, + bool ordered = false, + Datatype type = static_cast(255), + std::string enmr_name = default_enmr_name); + + template + void check_enumeration( + shared_ptr enmr, + const std::string& name, + const std::vector& values, + Datatype data_type, + uint32_t cell_val_num, + bool ordered); + + template + void check_storage_serialization(const std::vector& values); + + template + void check_storage_deserialization(const std::vector& values); + + storage_size_t calculate_serialized_size(shared_ptr enmr); + WriterTile serialize_to_tile(shared_ptr enmr); + + template + std::vector as_vector(shared_ptr enmr); + + shared_ptr create_schema(); + + void create_array(); + shared_ptr get_array(QueryType type); + shared_ptr get_array_directory(); + shared_ptr get_array_schema_latest(bool load_enmrs = false); + + // Serialization helpers + ArraySchema ser_des_array_schema( + shared_ptr schema, + bool client_side, + SerializationType stype); + + shared_ptr ser_des_array_schema_evolution( + ArraySchemaEvolution* ase, bool client_side, SerializationType stype); + + void ser_des_query( + Query* q_in, Query* q_out, bool client_side, SerializationType stype); + + template + bool vec_cmp(std::vector v1, std::vector v2); + + void flatten_buffer_list(BufferList& blist, Buffer& buf); + + void rm_array(); + + URI uri_; + Config cfg_; + Context ctx_; + EncryptionKey enc_key_; +}; + +template +QueryCondition create_qc( + const char* field_name, T condition_value, const QueryConditionOp& op); + +/* ********************************* */ +/* Testing Enumeration */ +/* ********************************* */ + +TEST_CASE_METHOD( + EnumerationFx, + "Basic Fixed Size Enumeration Creation", + "[enumeration][basic][fixed]") { + std::vector values = {1, 2, 3, 4, 5}; + auto enmr = create_enumeration(values); + check_enumeration( + enmr, default_enmr_name, values, Datatype::UINT32, 1, false); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Basic Variable Size Enumeration Creation", + "[enumeration][basic][fixed]") { + std::vector values = {"foo", "bar", "baz", "bingo", "bango"}; + auto enmr = create_enumeration(values); + check_enumeration( + enmr, + default_enmr_name, + values, + Datatype::STRING_ASCII, + constants::var_num, + false); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Basic Variable Size With Empty Value Enumeration Creation", + "[enumeration][basic][fixed]") { + std::vector values = {"foo", "bar", "", "bingo", "bango"}; + auto enmr = create_enumeration(values); + check_enumeration( + enmr, + default_enmr_name, + values, + Datatype::STRING_ASCII, + constants::var_num, + false); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation with Ordered", + "[enumeration][basic][var-size][ordered]") { + std::vector values = {"foo", "bar", "baz", "bingo", "bango"}; + auto enmr = create_enumeration(values, true); + check_enumeration( + enmr, + default_enmr_name, + values, + Datatype::STRING_ASCII, + constants::var_num, + true); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation with Datatype", + "[enumeration][basic][var-size][custom-datatype]") { + std::vector values = {"foo", "bar", "baz", "bingo", "bango"}; + auto enmr = create_enumeration(values, false, Datatype::STRING_UTF8); + check_enumeration( + enmr, + default_enmr_name, + values, + Datatype::STRING_UTF8, + constants::var_num, + false); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation with Multi-Cell Val Num", + "[enumeration][basic][fixed][multi-cell-val-num]") { + std::vector values = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + auto enmr = Enumeration::create( + default_enmr_name, + Datatype::INT32, + 2, + false, + values.data(), + values.size() * sizeof(int), + nullptr, + 0); + check_enumeration(enmr, default_enmr_name, values, Datatype::INT32, 2, false); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Invalid empty name - std::string", + "[enumeration][error][invalid-name]") { + std::vector values = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + REQUIRE_THROWS(Enumeration::create( + std::string(), + Datatype::INT32, + 2, + false, + values.data(), + values.size() * sizeof(int), + nullptr, + 0)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Invalid empty name - char*", + "[enumeration][error][invalid-name]") { + std::vector values = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + REQUIRE_THROWS(Enumeration::create( + "", + Datatype::INT32, + 2, + false, + values.data(), + values.size() * sizeof(int), + nullptr, + 0)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Invalid name with slash", + "[enumeration][error][invalid-name]") { + std::vector values = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + REQUIRE_THROWS(Enumeration::create( + "an/enumeration", + Datatype::INT32, + 2, + false, + values.data(), + values.size() * sizeof(int), + nullptr, + 0)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Invalid cell val num", + "[enumeration][error][invalid-cell-val-num]") { + std::vector values = {1, 2, 3}; + REQUIRE_THROWS(Enumeration::create( + default_enmr_name, + Datatype::INT32, + 0, + false, + values.data(), + values.size() * sizeof(int), + nullptr, + 0)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - No data pointer", + "[enumeration][error][data-nullptr]") { + std::vector values = {1, 2, 3}; + REQUIRE_THROWS(Enumeration::create( + default_enmr_name, + Datatype::INT32, + 1, + false, + nullptr, + values.size() * sizeof(int), + nullptr, + 0)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Zero data size", + "[enumeration][error][data-zero-size]") { + std::vector values = {1, 2, 3}; + REQUIRE_THROWS(Enumeration::create( + default_enmr_name, + Datatype::INT32, + 1, + false, + values.data(), + 0, + nullptr, + 0)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - No offsets pointer", + "[enumeration][error][offsets-nullptr]") { + auto data = "foobarbazbam"; + std::vector offsets = {0, 3, 6, 9}; + REQUIRE_THROWS(Enumeration::create( + default_enmr_name, + Datatype::STRING_ASCII, + constants::var_num, + false, + data, + strlen(data), + nullptr, + offsets.size() * sizeof(uint64_t))); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - No offsets size", + "[enumeration][error][offsets-zero-size]") { + auto data = "foobarbazbam"; + std::vector offsets = {0, 3, 6, 9}; + REQUIRE_THROWS(Enumeration::create( + default_enmr_name, + Datatype::STRING_ASCII, + constants::var_num, + false, + data, + strlen(data), + offsets.data(), + 0)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Offsets not required, pointer provided", + "[enumeration][error][offsets-not-required]") { + std::vector values = {0, 1, 2, 3, 4}; + std::vector offsets = {0, 3, 6, 9}; + REQUIRE_THROWS(Enumeration::create( + default_enmr_name, + Datatype::INT32, + 1, + false, + values.data(), + values.size() * sizeof(int), + offsets.data(), + 0)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Offsets not required, size provided", + "[enumeration][error][offsets-not-required]") { + std::vector values = {0, 1, 2, 3, 4}; + REQUIRE_THROWS(Enumeration::create( + default_enmr_name, + Datatype::INT32, + 1, + false, + values.data(), + values.size() * sizeof(int), + nullptr, + 100)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Invalid offsests size provided", + "[enumeration][error][offsets-invalid-size]") { + auto data = "foobarbazbam"; + std::vector offsets = {0, 3, 6, 9}; + // Passing 3 for the offsets size is incorrect because the offsets size has + // to be a multiple of `sizeof(uint64_t)` + REQUIRE_THROWS(Enumeration::create( + default_enmr_name, + Datatype::STRING_ASCII, + constants::var_num, + false, + data, + strlen(data), + offsets.data(), + 3)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Offsets to data beyond provided data size", + "[enumeration][error][invalid-offset-data]") { + auto data = "foobarbazbam"; + std::vector offsets = {0, 3, 6, 100}; + // The last offset is larger than data_size + REQUIRE_THROWS(Enumeration::create( + default_enmr_name, + Datatype::STRING_ASCII, + constants::var_num, + false, + data, + strlen(data), + offsets.data(), + offsets.size() * sizeof(uint64_t))); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Invalid data size", + "[enumeration][error][invalid-data-size]") { + std::vector values = {1, 2, 3, 4, 5}; + // Passing 3 for the data size is invalid as its not a multiple of + // sizeof(int) + REQUIRE_THROWS(Enumeration::create( + default_enmr_name, + Datatype::INT32, + 1, + false, + values.data(), + 3, + nullptr, + 0)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Repeated fixed sized values", + "[enumeration][error][repeated-values]") { + std::vector values = {1, 2, 3, 3, 4, 5}; + REQUIRE_THROWS(create_enumeration(values)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Repeated variable sized values", + "[enumeration][error][repeated-values]") { + std::vector values = {"foo", "bar", "bang", "bar"}; + REQUIRE_THROWS(create_enumeration(values)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Repeated empty variable sized values", + "[enumeration][error][repeated-values]") { + std::vector values = {"foo", "", "bang", ""}; + REQUIRE_THROWS(create_enumeration(values)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Serialization - Fixed Size", + "[enumeration][serialization][fixed-size]") { + std::vector values = {1, 2, 3, 4, 5}; + check_storage_serialization(values); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Serialization - Variable Size", + "[enumeration][serialization][var-size]") { + std::vector values = {"foo", "bar", "baz", "bam", "cap"}; + check_storage_serialization(values); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Deserialization - Fixed Size", + "[enumeration][deserialization][fixed-size]") { + std::vector values = {1, 2, 3, 4, 5}; + check_storage_deserialization(values); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Deserialization - Variable Size", + "[enumeration][deserialization][var-size]") { + std::vector values = {"foo", "bar", "baz", "bam", "cap"}; + check_storage_deserialization(values); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Deserialization Error - Invalid version", + "[enumeration][deserialization][error][invalid-version]") { + std::vector values = {"foo", "bar", "baz", "bam", "cap"}; + auto enmr = create_enumeration(values); + auto tile = serialize_to_tile(enmr); + + REQUIRE(tile.size() > 4); + auto data = tile.data(); + memset(data, 1, 4); + + Deserializer deserializer(tile.data(), tile.size()); + REQUIRE_THROWS(Enumeration::deserialize(deserializer)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration index_of - Fixed Size", + "[enumeration][index-of][fixed-size]") { + std::vector values = {1, 2, 3, 4, 5}; + auto enmr = create_enumeration(values); + + for (uint64_t i = 0; i < values.size(); i++) { + int tmp = values[i]; + UntypedDatumView udv(&tmp, sizeof(int)); + REQUIRE(enmr->index_of(udv) == i); + } +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration index_of - Fixed Size Missing", + "[enumeration][index-of][fixed-size]") { + std::vector values = {1, 2, 3, 4, 5}; + auto enmr = create_enumeration(values); + + int zero = 0; + UntypedDatumView udv_zero(&zero, sizeof(int)); + REQUIRE(enmr->index_of(udv_zero) == constants::enumeration_missing_value); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration index_of - Variable Size", + "[enumeration][index-of][var-size]") { + std::vector values = {"foo", "bar", "baz", "bang", "ohai"}; + auto enmr = create_enumeration(values); + + for (uint64_t i = 0; i < values.size(); i++) { + UntypedDatumView udv(values[i].data(), values[i].size()); + REQUIRE(enmr->index_of(udv) == i); + } +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration index_of - Variable Size Missing", + "[enumeration][index-of][var-size]") { + std::vector values = {"foo", "bar", "baz", "bang", "ohai"}; + auto enmr = create_enumeration(values); + + UntypedDatumView udv_empty("", 0); + REQUIRE(enmr->index_of(udv_empty) == constants::enumeration_missing_value); +} + +/* ********************************* */ +/* Testing Attribute */ +/* ********************************* */ + +TEST_CASE_METHOD( + EnumerationFx, + "Attribute set_enumeration_name - Error - Empty Name", + "[enumeration][attribute][error]") { + auto attr = make_shared(HERE(), "foo", Datatype::INT8); + REQUIRE_THROWS(attr->set_enumeration_name("")); +} + +/* ********************************* */ +/* Testing Array */ +/* ********************************* */ + +TEST_CASE_METHOD( + EnumerationFx, + "Array - Get Enumeration", + "[enumeration][array][get-enumeration]") { + create_array(); + auto array = get_array(QueryType::READ); + auto enmr = array->get_enumeration("test_enmr"); + REQUIRE(enmr != nullptr); + + std::vector values = {"ant", "bat", "cat", "dog", "emu"}; + check_enumeration( + enmr, + "test_enmr", + values, + Datatype::STRING_ASCII, + constants::var_num, + false); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Array - Get Enumeration Repeated", + "[enumeration][array][get-enumeration]") { + create_array(); + auto array = get_array(QueryType::READ); + auto enmr1 = array->get_enumeration("test_enmr"); + auto enmr2 = array->get_enumeration("test_enmr"); + REQUIRE(enmr1 == enmr2); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Array - Get Enumeration Error - REMOTE NOT YET SUPPORTED", + "[enumeration][array][error][get-remote]") { + std::string uri_str = "tiledb://namespace/array_name"; + auto array = make_shared(HERE(), URI(uri_str), ctx_.storage_manager()); + REQUIRE_THROWS(array->get_enumeration("something_here")); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Array - Get Enumeration Error - Not Open", + "[enumeration][array][error][not-open]") { + auto array = make_shared(HERE(), uri_, ctx_.storage_manager()); + REQUIRE_THROWS(array->get_enumeration("foo")); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Array - Get Non-existent Enumeration", + "[enumeration][array][unknown-enumeration]") { + create_array(); + auto array = get_array(QueryType::READ); + REQUIRE_THROWS(array->get_enumeration("foo")); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Array - Load All Enumerations - Latest Only", + "[enumeration][array][load-all-enumerations][latest-only]") { + create_array(); + auto array = get_array(QueryType::READ); + auto schema = array->array_schema_latest_ptr(); + REQUIRE(schema->is_enumeration_loaded("test_enmr") == false); + CHECK_NOTHROW(array->load_all_enumerations()); + REQUIRE(schema->is_enumeration_loaded("test_enmr") == true); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Array - Load All Enumerations - Repeated", + "[enumeration][array][load-all-enumerations][repeated]") { + create_array(); + auto array = get_array(QueryType::READ); + auto schema = array->array_schema_latest_ptr(); + + REQUIRE(schema->is_enumeration_loaded("test_enmr") == false); + + CHECK_NOTHROW(array->load_all_enumerations()); + REQUIRE(schema->is_enumeration_loaded("test_enmr") == true); + + CHECK_NOTHROW(array->load_all_enumerations()); + REQUIRE(schema->is_enumeration_loaded("test_enmr") == true); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Array - Load All Enumerations", + "[enumeration][array][load-all-enumerations]") { + create_array(); + auto array = get_array(QueryType::READ); + auto schema = array->array_schema_latest_ptr(); + REQUIRE(schema->is_enumeration_loaded("test_enmr") == false); + CHECK_NOTHROW(array->load_all_enumerations(false)); + REQUIRE(schema->is_enumeration_loaded("test_enmr") == true); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Array - Load All Enumerations Error - REMOTE NOT YET SUPPORTED", + "[enumeration][array][error][get-remote]") { + std::string uri_str = "tiledb://namespace/array_name"; + auto array = make_shared(HERE(), URI(uri_str), ctx_.storage_manager()); + REQUIRE_THROWS(array->load_all_enumerations()); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Array - Load All Enumerations Error - Not Open", + "[enumeration][array][error][not-open]") { + auto array = make_shared(HERE(), uri_, ctx_.storage_manager()); + REQUIRE_THROWS(array->load_all_enumerations()); +} + +/* ********************************* */ +/* Testing ArrayDirectory */ +/* ********************************* */ + +TEST_CASE_METHOD( + EnumerationFx, + "ArrayDirectory - Load Enumeration", + "[enumeration][array-directory][load-enumeration]") { + create_array(); + + auto schema = get_array_schema_latest(); + auto ad = get_array_directory(); + auto enmr_name = schema->attribute("attr1")->get_enumeration_name(); + REQUIRE(enmr_name.has_value()); + + auto enmr = ad->load_enumeration(schema, enmr_name.value(), enc_key_); + std::vector values = {"ant", "bat", "cat", "dog", "emu"}; + check_enumeration( + enmr, + "test_enmr", + values, + Datatype::STRING_ASCII, + constants::var_num, + false); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArrayDirectory - Load Enumeration Error", + "[enumeration][array-directory][error]") { + create_array(); + + auto schema = get_array_schema_latest(); + auto ad = get_array_directory(); + + // Check that this function throws an exception when attempting to load + // an unknown enumeration + REQUIRE_THROWS(ad->load_enumeration(schema, "not_an_enumeration", enc_key_)); +} + +/* ********************************* */ +/* Testing ArraySchema */ +/* ********************************* */ + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Add Enumeration - Enumeration nullptr Error", + "[enumeration][array-schema][error]") { + auto schema = make_shared(HERE()); + REQUIRE_THROWS(schema->add_enumeration(nullptr)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Add Basic Enumeration", + "[enumeration][array-schema][basic]") { + auto schema = make_shared(HERE()); + + std::vector values = {1, 2, 3, 4, 5}; + auto enmr = create_enumeration(values); + schema->add_enumeration(enmr); + + auto attr = make_shared(HERE(), "foo", Datatype::INT8); + attr->set_enumeration_name(enmr->name()); + CHECK_NOTHROW(schema->add_attribute(attr)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Get Enumeration", + "[enumeration][array-schema][get]") { + auto schema = make_shared(HERE(), ArrayType::DENSE); + + std::vector values = {1, 2, 3, 4, 5}; + auto enmr1 = create_enumeration(values); + schema->add_enumeration(enmr1); + + auto enmr2 = schema->get_enumeration(default_enmr_name); + check_enumeration(enmr2, enmr1->name(), values, Datatype::INT32, 1, false); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Get Missing Enumeration Error", + "[enumeration][array-schema][error]") { + auto schema = make_shared(HERE(), ArrayType::SPARSE); + REQUIRE_THROWS(schema->get_enumeration("not_an_enumeration")); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Add Enumeration with Existing Enumeration of same Name", + "[enumeration][array-schema][eror]") { + auto schema = make_shared(HERE(), ArrayType::SPARSE); + std::vector values = {1, 2, 3, 4, 5}; + auto enmr = create_enumeration(values); + + schema->add_enumeration(enmr); + REQUIRE_THROWS(schema->add_enumeration(enmr)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Add Attribute with Missing Enumeration Error", + "[enumeration][array-schema][eror]") { + auto schema = make_shared(HERE(), ArrayType::SPARSE); + auto attr = make_shared(HERE(), "an_attr", Datatype::INT32); + attr->set_enumeration_name("not_an_enumeration"); + REQUIRE(!schema->add_attribute(attr).ok()); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Get All Enumeration Names Empty", + "[enumeration][array-schema][get-all][empty]") { + auto schema = make_shared(HERE(), ArrayType::DENSE); + auto enmr_names = schema->get_enumeration_names(); + REQUIRE(enmr_names.size() == 0); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Get All Enumeration Names", + "[enumeration][array-schema][get-all]") { + auto schema = make_shared(HERE(), ArrayType::DENSE); + + std::vector values = {1.0f, 1.1f, 1.2f, 1.3f, 1.4f}; + auto enmr1 = create_enumeration(values); + schema->add_enumeration(enmr1); + + auto enmr_names = schema->get_enumeration_names(); + REQUIRE(enmr_names.size() == 1); + + auto enmr2 = schema->get_enumeration(enmr_names[0]); + REQUIRE(enmr2 == enmr1); + check_enumeration( + enmr2, default_enmr_name, values, Datatype::FLOAT32, 1, false); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Attribute with Invalid Datatype", + "[enumeration][array-schema][error][bad-attr-datatype]") { + auto schema = make_shared(HERE(), ArrayType::DENSE); + + std::vector values = {1, 2, 3, 4, 5}; + auto enmr = create_enumeration(values); + schema->add_enumeration(enmr); + + auto attr = make_shared(HERE(), "ohai", Datatype::FLOAT32); + attr->set_enumeration_name(default_enmr_name); + REQUIRE(schema->add_attribute(attr).ok() == false); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Attribute with Invalid Cell Val Num", + "[enumeration][array-schema][error][bad-attr-cell-val-num]") { + auto schema = make_shared(HERE(), ArrayType::DENSE); + + std::vector values = {1, 2, 3, 4, 5}; + auto enmr = create_enumeration(values); + schema->add_enumeration(enmr); + + auto attr = make_shared( + HERE(), "ohai", Datatype::INT32, 2, DataOrder::UNORDERED_DATA); + attr->set_enumeration_name(default_enmr_name); + REQUIRE(schema->add_attribute(attr).ok() == false); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Store nullptr Enumeration Error", + "[enumeration][array-schema][error][store-nullptr-enumeration]") { + auto schema = make_shared(HERE(), ArrayType::DENSE); + REQUIRE_THROWS(schema->store_enumeration(nullptr)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Store Enumeration Error", + "[enumeration][array-schema][error][store-unknown-enumeration]") { + auto schema = make_shared(HERE(), ArrayType::DENSE); + std::vector values = {1, 2, 3, 4, 5}; + auto enmr = + create_enumeration(values, false, Datatype::INT32, "unknown_enmr"); + REQUIRE_THROWS(schema->store_enumeration(enmr)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Store Enumeration Error - Already Loaded", + "[enumeration][array-schema][error][store-loaded-enumeration]") { + auto schema = make_shared(HERE(), ArrayType::DENSE); + + std::vector values = {0, 1, 2, 100000000}; + auto enmr = create_enumeration(values); + schema->add_enumeration(enmr); + + // This is an error because there's already an enumeration stored in the + // ArraySchema::enumeration_map_. When deserializing schemas from disk the + // entries in ArraySchema::enumeration_map_ are (std::string, nullptr). + REQUIRE_THROWS(schema->store_enumeration(enmr)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Attribute Get Enumeration Name From Attribute", + "[enumeration][array-schema][has-enumeration]") { + auto schema = make_shared(HERE(), ArrayType::SPARSE); + + std::vector values = {"a", "spot", "of", "tea", "perhaps?"}; + auto enmr = create_enumeration(values); + schema->add_enumeration(enmr); + + REQUIRE(schema->get_enumeration(enmr->name()) == enmr); + REQUIRE(schema->get_enumeration_path_name(enmr->name()) == enmr->path_name()); + + auto attr = make_shared(HERE(), "ohai", Datatype::INT64); + attr->set_enumeration_name(default_enmr_name); + throw_if_not_ok(schema->add_attribute(attr)); + + REQUIRE(schema->attribute("ohai")->get_enumeration_name().has_value()); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Schema Copy Constructor", + "[enumeration][array-schema][copy-ctor]") { + auto schema = create_schema(); + + // Check that the schema is valid and that we can copy it using the + // copy constructor. + CHECK_NOTHROW(schema->check()); + CHECK_NOTHROW(make_shared(HERE(), *(schema.get()))); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Mismatched path name error", + "[enumeration][array-schema][error]") { + create_array(); + auto schema = get_array_schema_latest(); + + // Creating a new Enumeration will give it a different UUID path name. + std::vector values = {1, 2, 3, 4, 5}; + auto enmr = create_enumeration(values, false, Datatype::INT32, "test_enmr"); + + // Storing an enumeration with a known name but different path is an error + REQUIRE_THROWS(schema->store_enumeration(enmr)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Drop Enumeration - Empty Name", + "[enumeration][array-schema][error]") { + create_array(); + auto schema = get_array_schema_latest(); + REQUIRE_THROWS(schema->drop_enumeration("")); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Drop Enumeration - Unknown Enumeration Name", + "[enumeration][array-schema][error]") { + create_array(); + auto schema = get_array_schema_latest(); + REQUIRE_THROWS(schema->drop_enumeration("not_an_enumeration")); +} + +/* ********************************* */ +/* Testing ArraySchemaEvolution */ +/* ********************************* */ + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Simple No Enumeration", + "[enumeration][array-schema-evolution][simple]") { + create_array(); + auto orig_schema = get_array_schema_latest(true); + auto ase = make_shared(HERE()); + auto attr3 = make_shared(HERE(), "attr3", Datatype::UINT32); + ase->add_attribute(attr3.get()); + CHECK_NOTHROW(ase->evolve_schema(orig_schema)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Simple With Enumeration", + "[enumeration][array-schema-evolution][simple]") { + create_array(); + auto orig_schema = get_array_schema_latest(); + auto ase = make_shared(HERE()); + + std::vector values{0, 1, 2, 3, 4, 1000}; + auto enmr = create_enumeration(values); + ase->add_enumeration(enmr); + + auto attr3 = make_shared(HERE(), "attr3", Datatype::UINT32); + attr3->set_enumeration_name(default_enmr_name); + ase->add_attribute(attr3.get()); + + ase->drop_attribute("attr2"); + + CHECK_NOTHROW(ase->evolve_schema(orig_schema)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Drop Attribute After Add", + "[enumeration][array-schema-evolution][drop-add]") { + create_array(); + auto orig_schema = get_array_schema_latest(); + auto ase = make_shared(HERE()); + + std::vector values{0, 1, 2, 3, 4, 1000}; + auto enmr = create_enumeration(values); + ase->add_enumeration(enmr); + + auto attr3 = make_shared(HERE(), "attr3", Datatype::UINT32); + attr3->set_enumeration_name(default_enmr_name); + ase->add_attribute(attr3.get()); + + CHECK_NOTHROW(ase->drop_attribute("attr3")); + CHECK_NOTHROW(ase->evolve_schema(orig_schema)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Enumeration Attribute Names to Add", + "[enumeration][array-schema-evolution][simple]") { + create_array(); + auto orig_schema = get_array_schema_latest(); + + auto ase = make_shared(HERE()); + + std::vector values{0, 1, 2, 3, 4, 1000}; + auto enmr = create_enumeration(values); + ase->add_enumeration(enmr); + + auto enmr_names = ase->enumeration_names_to_add(); + REQUIRE(enmr_names.size() == 1); + REQUIRE(enmr_names[0] == enmr->name()); + + CHECK_NOTHROW(ase->evolve_schema(orig_schema)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Enumeration to Add", + "[enumeration][array-schema-evolution][enmr-to-add]") { + create_array(); + auto orig_schema = get_array_schema_latest(); + auto ase = make_shared(HERE()); + + std::vector values{0, 1, 2, 3, 4, 1000}; + auto enmr1 = create_enumeration(values); + ase->add_enumeration(enmr1); + + auto enmr2 = ase->enumeration_to_add(enmr1->name()); + REQUIRE(enmr2 == enmr1); + + CHECK_NOTHROW(ase->evolve_schema(orig_schema)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Drop Enumeration", + "[enumeration][array-schema-evolution][enmr-to-add]") { + create_array(); + auto orig_schema = get_array_schema_latest(); + auto ase1 = make_shared(HERE()); + + std::vector values{0, 1, 2, 3, 4, 1000}; + auto enmr1 = create_enumeration(values, false, Datatype::UINT64, "enmr"); + ase1->add_enumeration(enmr1); + + auto new_schema = ase1->evolve_schema(orig_schema); + + auto ase2 = make_shared(HERE()); + ase2->drop_enumeration("enmr"); + + CHECK_NOTHROW(ase2->evolve_schema(new_schema)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Drop Enumeration", + "[enumeration][array-schema-evolution][enmr-to-drop]") { + auto ase = make_shared(HERE()); + CHECK_NOTHROW(ase->drop_enumeration("test_enmr")); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Drop Enumeration Repeated", + "[enumeration][array-schema-evolution][enmr-to-drop-repeated]") { + auto ase = make_shared(HERE()); + CHECK_NOTHROW(ase->drop_enumeration("test_enmr")); + CHECK_NOTHROW(ase->drop_enumeration("test_enmr")); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Drop Enumeration After Add", + "[enumeration][array-schema-evolution][enmr-add-drop]") { + auto ase = make_shared(HERE()); + + std::vector values{0, 1, 2, 3, 4, 1000}; + auto enmr = create_enumeration(values, false, Datatype::UINT64, "enmr"); + ase->add_enumeration(enmr); + REQUIRE(ase->enumeration_names_to_add().size() == 1); + CHECK_NOTHROW(ase->drop_enumeration("enmr")); + REQUIRE(ase->enumeration_names_to_add().size() == 0); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Enumeration to Add - nullptr", + "[enumeration][array-schema-evolution][enmr-nullptr]") { + create_array(); + auto ase = make_shared(HERE()); + REQUIRE_THROWS(ase->add_enumeration(nullptr)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Enumeration to Add - Already Added", + "[enumeration][array-schema-evolution][enmr-already-added]") { + create_array(); + auto ase = make_shared(HERE()); + + std::vector values{0, 1, 2, 3, 4, 1000}; + auto enmr1 = create_enumeration(values, false, Datatype::UINT64, "enmr"); + auto enmr2 = create_enumeration(values, false, Datatype::UINT64, "enmr"); + ase->add_enumeration(enmr1); + REQUIRE_THROWS(ase->add_enumeration(enmr2)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Enumeration to Add - Missing Name", + "[enumeration][array-schema-evolution][missing-name]") { + create_array(); + auto ase = make_shared(HERE()); + REQUIRE(ase->enumeration_to_add("foo") == nullptr); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Drop enumeration while still in use.", + "[enumeration][array-schema-evolution][enmr-still-in-use]") { + create_array(); + auto orig_schema = get_array_schema_latest(); + auto ase = make_shared(HERE()); + ase->drop_enumeration("test_enmr"); + + REQUIRE_THROWS(ase->evolve_schema(orig_schema)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Enumeration not Loaded", + "[enumeration][array-schema-evolution][enmr-not-loaded]") { + create_array(); + + auto attr3 = make_shared(HERE(), "attr3", Datatype::UINT32); + attr3->set_enumeration_name("test_enmr"); + + auto ase = make_shared(HERE()); + ase->add_attribute(attr3.get()); + + auto orig_schema = get_array_schema_latest(); + REQUIRE_THROWS(ase->evolve_schema(orig_schema)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Enumeration to large for signed attribute type", + "[enumeration][array-schema-evolution][enmr-to-large]") { + create_array(); + + std::vector values; + values.resize(129); + std::iota(values.begin(), values.end(), 0); + auto enmr = create_enumeration(values, false, Datatype::INT32, "big_enmr"); + + auto attr3 = make_shared(HERE(), "attr3", Datatype::INT8); + attr3->set_enumeration_name("big_enmr"); + + auto ase = make_shared(HERE()); + ase->add_enumeration(enmr); + ase->add_attribute(attr3.get()); + + auto orig_schema = get_array_schema_latest(); + REQUIRE_THROWS(ase->evolve_schema(orig_schema)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Enumeration to large for unsigned attribute type", + "[enumeration][array-schema-evolution][enmr-to-large]") { + create_array(); + + std::vector values; + values.resize(257); + std::iota(values.begin(), values.end(), 0); + auto enmr = create_enumeration(values, false, Datatype::INT32, "big_enmr"); + + auto attr3 = make_shared(HERE(), "attr3", Datatype::UINT8); + attr3->set_enumeration_name("big_enmr"); + + auto ase = make_shared(HERE()); + ase->add_enumeration(enmr); + ase->add_attribute(attr3.get()); + + auto orig_schema = get_array_schema_latest(); + REQUIRE_THROWS(ase->evolve_schema(orig_schema)); +} + +/* ********************************* */ +/* Testing QueryCondition */ +/* ********************************* */ + +TEST_CASE_METHOD( + EnumerationFx, + "QueryCondition - Rewrite Enumeration Value", + "[enumeration][query-condition][rewrite-enumeration-value]") { + create_array(); + auto array = get_array(QueryType::READ); + auto schema = array->array_schema_latest_ptr(); + + // This is normally invoked by the query class when not being tested. It's + // required here so that the enumeration's data is loaded. + array->get_enumeration("test_enmr"); + + // Assert that the enumerations were loaded + auto enmr_names = schema->get_loaded_enumeration_names(); + REQUIRE(enmr_names.size() == 1); + REQUIRE(enmr_names[0] == "test_enmr"); + + // Create two copies of the same query condition for assertions + auto qc1 = create_qc("attr1", std::string("cat"), QueryConditionOp::EQ); + auto qc2 = qc1; + + qc2.rewrite_enumeration_conditions(*(schema.get())); + + // Assert that the rewritten tree matches in the right places while also + // different to verify the assertion of having been rewritten. + auto& tree1 = qc1.ast(); + auto& tree2 = qc2.ast(); + + REQUIRE(tree1->is_expr() == false); + REQUIRE(tree1->get_field_name() == "attr1"); + + REQUIRE(tree2->is_expr() == tree1->is_expr()); + REQUIRE(tree2->get_field_name() == tree1->get_field_name()); + + auto udv1 = tree1->get_condition_value_view(); + auto udv2 = tree2->get_condition_value_view(); + REQUIRE(udv2.size() != udv1.size()); + REQUIRE(udv2.content() != udv1.content()); + REQUIRE(udv2.value_as() == 2); +} + +TEST_CASE_METHOD( + EnumerationFx, + "QueryCondition - Skip enumeration rewrite", + "[enumeration][query-condition][skip-rewrite]") { + create_array(); + auto schema = get_array_schema_latest(); + + // Almost exactly the same test as before, except this time we call + // `set_use_enumeration(false)` before rewriting and assert that the + // resulting rewritten query tree matches exactly since no enumeration + // rewriting has taken place. + auto qc1 = create_qc("attr1", (int)2, QueryConditionOp::EQ); + qc1.set_use_enumeration(false); + auto qc2 = qc1; + + qc2.rewrite_enumeration_conditions(*(schema.get())); + + auto& tree1 = qc1.ast(); + auto& tree2 = qc2.ast(); + + // Check that both trees match exactly + REQUIRE(tree1->is_expr() == false); + REQUIRE(tree1->get_field_name() == "attr1"); + + REQUIRE(tree2->is_expr() == tree1->is_expr()); + REQUIRE(tree2->get_field_name() == tree1->get_field_name()); + + auto udv1 = tree1->get_condition_value_view(); + auto udv2 = tree1->get_condition_value_view(); + REQUIRE(udv2.size() == udv1.size()); + REQUIRE(udv2.content() == udv1.content()); + REQUIRE(udv2.value_as() == 2); +} + +TEST_CASE_METHOD( + EnumerationFx, + "QueryCondition - Rewrite - No failure on unknown attribute", + "[enumeration][query-condition]") { + create_array(); + auto schema = get_array_schema_latest(); + + auto qc1 = create_qc("not_an_attr", (int)2, QueryConditionOp::EQ); + qc1.rewrite_enumeration_conditions(*(schema.get())); +} + +TEST_CASE_METHOD( + EnumerationFx, + "QueryCondition - Rewrite - Enumeration Not Loaded", + "[enumeration][query-condition][error]") { + create_array(); + auto schema = get_array_schema_latest(); + + auto qc1 = create_qc("attr1", (int)2, QueryConditionOp::EQ); + REQUIRE_THROWS(qc1.rewrite_enumeration_conditions(*(schema.get()))); +} + +TEST_CASE_METHOD( + EnumerationFx, + "QueryCondition - Rewrite - Inequality on Unordered Enumeration", + "[enumeration][query-condition][error]") { + // If an enumeration isn't marked as ordered, then it should throw an error + // when attempting to use an inequality operator on the attribute. + create_array(); + auto array = get_array(QueryType::READ); + auto schema = get_array_schema_latest(); + + // This is normally invoked by the query class when not being tested. + array->get_enumeration("test_enmr"); + + auto qc1 = create_qc("attr1", (int)2, QueryConditionOp::LT); + REQUIRE_THROWS(qc1.rewrite_enumeration_conditions(*(schema.get()))); +} + +TEST_CASE_METHOD( + EnumerationFx, + "QueryCondition - Rewrite Empty QC - Coverage", + "[enumeration][query-condition][coverage]") { + // Check that qc.rewrite_enumeration_conditions doesn't throw on an empty QC + create_array(); + auto schema = get_array_schema_latest(); + + QueryCondition qc; + CHECK_NOTHROW(qc.rewrite_enumeration_conditions(*(schema.get()))); +} + +TEST_CASE_METHOD( + EnumerationFx, + "QueryCondition - use_enumeration - Check Accessor for Coverage", + "[enumeration][query-condition][coverage]") { + auto qc1 = create_qc("attr1", (int)2, QueryConditionOp::LT); + auto& node = qc1.ast(); + + REQUIRE(node->use_enumeration() == true); + qc1.set_use_enumeration(false); + REQUIRE(node->use_enumeration() == false); +} + +TEST_CASE_METHOD( + EnumerationFx, + "QueryCondition - set_use_enumeration - Affects Children", + "[enumeration][query-condition][set_use_enumeration]") { + // Check that set_use_enumeration is applied to the entire tree if applied + // to an expression node. + auto qc1 = create_qc("attr1", (int)2, QueryConditionOp::EQ); + auto qc2 = create_qc("attr2", 3.0f, QueryConditionOp::LT); + QueryCondition qc3; + throw_if_not_ok(qc1.combine(qc2, QueryConditionCombinationOp::AND, &qc3)); + + auto& tree1 = qc3.ast(); + for (auto& child : tree1->get_children()) { + REQUIRE(child->use_enumeration() == true); + } + + qc3.set_use_enumeration(false); + + auto& tree2 = qc3.ast(); + for (auto& child : tree2->get_children()) { + REQUIRE(child->use_enumeration() == false); + } +} + +TEST_CASE_METHOD( + EnumerationFx, + "QueryCondition - use_enumeration - Error on ASTNodeExpr", + "[enumeration][query-condition][error]") { + // Check that an ASTNodeExpr throws correctly when calling + // `use_enumeration()`. + auto qc1 = create_qc("attr1", (int)2, QueryConditionOp::EQ); + auto qc2 = create_qc("attr2", 3.0f, QueryConditionOp::LT); + QueryCondition qc3; + throw_if_not_ok(qc1.combine(qc2, QueryConditionCombinationOp::AND, &qc3)); + auto& node = qc3.ast(); + + REQUIRE_THROWS(node->use_enumeration()); +} + +/* ********************************* */ +/* Testing Cap'N Proto Serialization */ +/* ********************************* */ + +#ifdef TILEDB_SERIALIZATION + +TEST_CASE_METHOD( + EnumerationFx, + "Cap'N Proto - Basic New ArraySchema Serialization", + "[enumeration][capnp][basic][initialized-in-ram") { + auto client_side = GENERATE(true, false); + auto ser_type = GENERATE(SerializationType::CAPNP, SerializationType::JSON); + + auto schema1 = create_schema(); + auto schema2 = ser_des_array_schema(schema1, client_side, ser_type); + + auto all_names1 = schema1->get_enumeration_names(); + auto all_names2 = schema2.get_enumeration_names(); + REQUIRE(vec_cmp(all_names1, all_names2)); + + auto loaded_names1 = schema1->get_loaded_enumeration_names(); + auto loaded_names2 = schema2.get_loaded_enumeration_names(); + REQUIRE(vec_cmp(loaded_names1, loaded_names2)); + + // This is a new schema in RAM, so the loaded names should be the same + // as all names. + REQUIRE(vec_cmp(all_names1, loaded_names1)); + REQUIRE(vec_cmp(all_names2, loaded_names2)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Cap'N Proto - Basic Exisitng ArraySchema Serialization", + "[enumeration][capnp][basic][deserialized-from-disk]") { + create_array(); + + auto client_side = GENERATE(true, false); + auto ser_type = GENERATE(SerializationType::CAPNP, SerializationType::JSON); + + auto schema1 = get_array_schema_latest(); + auto schema2 = ser_des_array_schema(schema1, client_side, ser_type); + + auto all_names1 = schema1->get_enumeration_names(); + auto all_names2 = schema2.get_enumeration_names(); + REQUIRE(vec_cmp(all_names1, all_names2)); + + // This schema was deserialized from disk without any enumerations loaded + // so both of these should be empty. + auto loaded_names1 = schema1->get_loaded_enumeration_names(); + auto loaded_names2 = schema2.get_loaded_enumeration_names(); + + REQUIRE(loaded_names1.empty()); + REQUIRE(loaded_names2.empty()); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Cap'N Proto - Basic ArraySchemaEvolution Serialization", + "[enumeration][capnp][basic][array-schema-evolution]") { + auto client_side = GENERATE(true, false); + auto ser_type = GENERATE(SerializationType::CAPNP, SerializationType::JSON); + + std::vector values1 = {1, 2, 3, 4, 5}; + auto enmr1 = create_enumeration(values1, false, Datatype::INT32, "enmr1"); + + std::vector values2 = {1.0, 2.0, 3.0, 4.0, 5.0}; + auto enmr2 = create_enumeration(values2, true, Datatype::FLOAT64, "enmr2"); + + auto attr = make_shared(HERE(), "ohai", Datatype::INT64); + attr->set_enumeration_name("enmr2"); + + ArraySchemaEvolution ase1; + ase1.add_attribute(attr.get()); + ase1.add_enumeration(enmr1); + ase1.add_enumeration(enmr2); + ase1.drop_attribute("some_attr"); + ase1.drop_enumeration("an_enumeration"); + + auto ase2 = ser_des_array_schema_evolution(&ase1, client_side, ser_type); + + auto enmrs_to_add1 = ase1.enumeration_names_to_add(); + auto enmrs_to_add2 = ase2->enumeration_names_to_add(); + REQUIRE(vec_cmp(enmrs_to_add1, enmrs_to_add2)); + + for (auto& name : enmrs_to_add1) { + REQUIRE(ase1.enumeration_to_add(name) != nullptr); + REQUIRE(ase2->enumeration_to_add(name) != nullptr); + REQUIRE(ase1.enumeration_to_add(name) != ase2->enumeration_to_add(name)); + } +} + +TEST_CASE_METHOD( + EnumerationFx, + "Cap'N Proto - Basic Backwards Compatible Query Serialization", + "[enumeration][capnp][basic][query-condition-old]") { + auto client_side = GENERATE(true, false); + + // Query does not support serialization to JSON + auto ser_type = SerializationType::CAPNP; + + create_array(); + auto array = get_array(QueryType::READ); + + // This is normally invoked by the query class when not being tested. + array->get_enumeration("test_enmr"); + + auto qc1 = create_qc("attr1", (int)2, QueryConditionOp::EQ); + qc1.set_use_enumeration(false); + + Query q1(ctx_.storage_manager(), array); + throw_if_not_ok(q1.set_condition(qc1)); + + Query q2(ctx_.storage_manager(), array); + ser_des_query(&q1, &q2, client_side, ser_type); + + auto qc2 = q2.condition(); + REQUIRE(qc2.has_value()); + + auto& node = qc2.value().ast(); + REQUIRE(node->use_enumeration() == false); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Cap'N Proto - Basic New Query Serialization", + "[enumeration][capnp][basic][query-condition-new]") { + auto client_side = GENERATE(true, false); + + // Query does not support serialization to JSON + auto ser_type = SerializationType::CAPNP; + + create_array(); + auto array = get_array(QueryType::READ); + + // This is normally invoked by the query class when not being tested. + array->get_enumeration("test_enmr"); + + auto qc1 = create_qc("attr1", (int)2, QueryConditionOp::EQ); + qc1.set_use_enumeration(false); + + auto qc2 = create_qc("attr2", std::string("foo"), QueryConditionOp::NE); + QueryCondition qc3; + + throw_if_not_ok(qc1.combine(qc2, QueryConditionCombinationOp::OR, &qc3)); + + Query q1(ctx_.storage_manager(), array); + throw_if_not_ok(q1.set_condition(qc3)); + + Query q2(ctx_.storage_manager(), array); + ser_des_query(&q1, &q2, client_side, ser_type); + + auto qc4 = q2.condition(); + REQUIRE(qc4.has_value()); + + auto& node1 = qc4.value().ast()->get_children()[0]; + auto& node2 = qc4.value().ast()->get_children()[1]; + REQUIRE(node1->use_enumeration() == false); + REQUIRE(node2->use_enumeration() == true); +} + +#endif // ifdef TILEDB_SERIALIZATIONs + +/* ********************************* */ +/* VERIFY SUPPORT CODE */ +/* ********************************* */ + +TEST_CASE_METHOD( + EnumerationFx, + "Check EnumerationFx::vec_cmp", + "[enumeration][test-code][verify]") { + std::vector v1 = {1, 2, 3, 4, 5}; + std::vector v2 = {5, 3, 4, 2, 1}; + + REQUIRE(vec_cmp(v1, v2)); + + std::vector v3 = {}; + REQUIRE(!vec_cmp(v1, v3)); + + std::vector v4 = {1, 2}; + REQUIRE(!vec_cmp(v1, v4)); + + std::vector v5 = {3, 4, 5, 6, 7}; + REQUIRE(!vec_cmp(v1, v5)); +} + +/* ********************************* */ +/* TEST SUPPORT CODE */ +/* ********************************* */ + +struct TypeParams { + TypeParams(Datatype type, uint32_t cell_val_num) + : type_(type) + , cell_val_num_(cell_val_num) { + } + + template + static TypeParams get(const std::vector>&) { + return TypeParams(Datatype::STRING_ASCII, constants::var_num); + } + + static TypeParams get(const std::vector&) { + return TypeParams(Datatype::INT32, 1); + } + + static TypeParams get(const std::vector&) { + return TypeParams(Datatype::UINT32, 1); + } + + static TypeParams get(const std::vector&) { + return TypeParams(Datatype::UINT64, 1); + } + + static TypeParams get(const std::vector&) { + return TypeParams(Datatype::FLOAT32, 1); + } + + static TypeParams get(const std::vector&) { + return TypeParams(Datatype::FLOAT64, 1); + } + + Datatype type_; + uint32_t cell_val_num_; +}; + +EnumerationFx::EnumerationFx() + : uri_("enumeration_test_array") + , ctx_(cfg_) { + rm_array(); + throw_if_not_ok(enc_key_.set_key(EncryptionType::NO_ENCRYPTION, nullptr, 0)); +} + +EnumerationFx::~EnumerationFx() { + rm_array(); +} + +template +shared_ptr EnumerationFx::create_enumeration( + const std::vector& values, + bool ordered, + Datatype type, + std::string name) { + TypeParams tp = TypeParams::get(values); + + if (type != static_cast(255)) { + tp.type_ = type; + } + + if constexpr (std::is_pod_v) { + return Enumeration::create( + name, + tp.type_, + tp.cell_val_num_, + ordered, + values.data(), + values.size() * sizeof(T), + nullptr, + 0); + } else { + uint64_t total_size = 0; + for (auto v : values) { + total_size += v.size(); + } + + std::vector data(total_size, 0); + std::vector offsets; + offsets.reserve(values.size()); + uint64_t curr_offset = 0; + + for (auto v : values) { + std::memcpy(data.data() + curr_offset, v.data(), v.size()); + offsets.push_back(curr_offset); + curr_offset += v.size(); + } + + return Enumeration::create( + name, + tp.type_, + tp.cell_val_num_, + ordered, + data.data(), + total_size, + offsets.data(), + offsets.size() * sizeof(uint64_t)); + } +} + +template +void EnumerationFx::check_enumeration( + shared_ptr enmr, + const std::string& name, + const std::vector& values, + Datatype data_type, + uint32_t cell_val_num, + bool ordered) { + REQUIRE(enmr->name() == name); + REQUIRE(!enmr->path_name().empty()); + REQUIRE(enmr->type() == data_type); + REQUIRE(enmr->cell_val_num() == cell_val_num); + REQUIRE(enmr->ordered() == ordered); + + std::vector data = as_vector(enmr); + REQUIRE(data == values); +} + +template +void EnumerationFx::check_storage_serialization(const std::vector& values) { + auto enmr = create_enumeration(values); + auto tile = serialize_to_tile(enmr); + REQUIRE(tile.size() == calculate_serialized_size(enmr)); +} + +template +void EnumerationFx::check_storage_deserialization( + const std::vector& values) { + auto enmr = create_enumeration(values); + auto tile = serialize_to_tile(enmr); + + Deserializer deserializer(tile.data(), tile.size()); + auto deserialized = Enumeration::deserialize(deserializer); + + REQUIRE(deserialized->name() == enmr->name()); + REQUIRE(deserialized->path_name().empty() == false); + REQUIRE(deserialized->type() == enmr->type()); + REQUIRE(deserialized->cell_val_num() == enmr->cell_val_num()); + REQUIRE(deserialized->ordered() == enmr->ordered()); + REQUIRE(deserialized->cell_size() == enmr->cell_size()); + REQUIRE(deserialized->var_size() == enmr->var_size()); + + auto orig_dspan = enmr->data(); + auto des_dspan = deserialized->data(); + REQUIRE(des_dspan.size() == orig_dspan.size()); + REQUIRE(memcmp(des_dspan.data(), orig_dspan.data(), orig_dspan.size()) == 0); + + if (enmr->var_size()) { + auto orig_ospan = enmr->offsets(); + auto des_ospan = deserialized->offsets(); + REQUIRE(orig_ospan.size() == des_ospan.size()); + REQUIRE( + memcmp(des_ospan.data(), orig_ospan.data(), orig_ospan.size()) == 0); + } +} + +storage_size_t EnumerationFx::calculate_serialized_size( + shared_ptr enmr) { + // Size is the sum of the following sizes: + storage_size_t num_bytes = 0; + + // uint32_t - version + num_bytes += sizeof(uint32_t); + + // uint32_t - name length + num_bytes += sizeof(uint32_t); + + // name.size() bytes + num_bytes += enmr->name().size(); + + // uint32_t - path_name length; + num_bytes += sizeof(uint32_t); + + // path_name.size() bytes + num_bytes += enmr->path_name().size(); + + // uint8_t - data type + num_bytes += sizeof(uint8_t); + + // uint32_t - cell_val_num + num_bytes += sizeof(uint32_t); + + // bool - ordered + num_bytes += sizeof(bool); + + // uint64_t - data.size() + // data.size() bytes + auto dspan = enmr->data(); + num_bytes += sizeof(uint64_t); + num_bytes += dspan.size(); + + // if var_sized: + if (enmr->var_size()) { + auto ospan = enmr->offsets(); + num_bytes += sizeof(uint64_t); + num_bytes += ospan.size(); + } + + return num_bytes; +} + +WriterTile EnumerationFx::serialize_to_tile( + shared_ptr enmr) { + SizeComputationSerializer size_serializer; + enmr->serialize(size_serializer); + + WriterTile tile{WriterTile::from_generic(size_serializer.size())}; + Serializer serializer(tile.data(), tile.size()); + enmr->serialize(serializer); + + return tile; +} + +template +std::vector EnumerationFx::as_vector(shared_ptr enmr) { + std::vector ret; + + if constexpr (std::is_pod_v) { + auto dspan = enmr->data(); + + const T* elems = reinterpret_cast(dspan.data()); + size_t count = dspan.size() / sizeof(T); + + ret.reserve(count); + for (size_t i = 0; i < count; i++) { + ret.push_back(elems[i]); + } + } else { + auto dspan = enmr->data(); + auto ospan = enmr->offsets(); + + auto str_data = reinterpret_cast(dspan.data()); + auto elems = reinterpret_cast(ospan.data()); + size_t count = ospan.size() / sizeof(uint64_t); + + ret.reserve(count); + for (size_t i = 0; i < count; i++) { + uint64_t len; + if (i + 1 < count) { + len = elems[i + 1] - elems[i]; + } else { + len = dspan.size() - elems[i]; + } + ret.emplace_back(str_data + elems[i], len); + } + } + + return ret; +} + +shared_ptr EnumerationFx::create_schema() { + // Create a schema to serialize + auto schema = make_shared(HERE(), ArrayType::SPARSE); + + auto dim = make_shared(HERE(), "dim1", Datatype::INT32); + int range[2] = {0, 1000}; + throw_if_not_ok(dim->set_domain(range)); + + auto dom = make_shared(HERE()); + throw_if_not_ok(dom->add_dimension(dim)); + throw_if_not_ok(schema->set_domain(dom)); + + std::vector values = {"ant", "bat", "cat", "dog", "emu"}; + auto enmr = + create_enumeration(values, false, Datatype::STRING_ASCII, "test_enmr"); + schema->add_enumeration(enmr); + + auto attr1 = make_shared(HERE(), "attr1", Datatype::INT32); + attr1->set_enumeration_name("test_enmr"); + throw_if_not_ok(schema->add_attribute(attr1)); + + auto attr2 = make_shared(HERE(), "attr2", Datatype::STRING_ASCII); + throw_if_not_ok(schema->add_attribute(attr2)); + + return schema; +} + +void EnumerationFx::create_array() { + auto schema = create_schema(); + throw_if_not_ok(ctx_.storage_manager()->array_create(uri_, schema, enc_key_)); +} + +shared_ptr EnumerationFx::get_array(QueryType type) { + auto array = make_shared(HERE(), uri_, ctx_.storage_manager()); + throw_if_not_ok(array->open(type, EncryptionType::NO_ENCRYPTION, nullptr, 0)); + return array; +} + +shared_ptr EnumerationFx::get_array_directory() { + return make_shared( + HERE(), ctx_.resources(), uri_, 0, UINT64_MAX, ArrayDirectoryMode::READ); +} + +shared_ptr EnumerationFx::get_array_schema_latest( + bool load_enmrs) { + auto array_dir = get_array_directory(); + auto schema = array_dir->load_array_schema_latest(enc_key_); + if (load_enmrs) { + array_dir->load_all_enumerations(schema, enc_key_); + } + return schema; +} + +#ifdef TILEDB_SERIALIZATION + +ArraySchema EnumerationFx::ser_des_array_schema( + shared_ptr schema, + bool client_side, + SerializationType stype) { + Buffer buf; + throw_if_not_ok(serialization::array_schema_serialize( + *(schema.get()), stype, &buf, client_side)); + return serialization::array_schema_deserialize(stype, buf); +} + +shared_ptr EnumerationFx::ser_des_array_schema_evolution( + ArraySchemaEvolution* ase, bool client_side, SerializationType stype) { + Buffer buf; + throw_if_not_ok(serialization::array_schema_evolution_serialize( + ase, stype, &buf, client_side)); + + ArraySchemaEvolution* ret; + throw_if_not_ok( + serialization::array_schema_evolution_deserialize(&ret, stype, buf)); + + return shared_ptr(ret); +} + +void EnumerationFx::ser_des_query( + Query* q_in, Query* q_out, bool client_side, SerializationType stype) { + Buffer buf; + BufferList blist; + + throw_if_not_ok( + serialization::query_serialize(q_in, stype, client_side, &blist)); + + flatten_buffer_list(blist, buf); + + throw_if_not_ok(serialization::query_deserialize( + buf, + stype, + client_side, + nullptr, + q_out, + &(ctx_.resources().compute_tp()))); +} + +#else // No TILEDB_SERIALIZATION + +ArraySchema EnumerationFx::ser_des_array_schema( + shared_ptr, bool, SerializationType) { + throw std::logic_error("Serialization not enabled."); +} + +shared_ptr EnumerationFx::ser_des_array_schema_evolution( + ArraySchemaEvolution*, bool, SerializationType) { + throw std::logic_error("Serialization not enabled."); +} + +void EnumerationFx::ser_des_query(Query*, Query*, bool, SerializationType) { + throw std::logic_error("Serialization not enabled."); +} + +#endif // TILEDB_SERIALIZATION + +template +bool EnumerationFx::vec_cmp(std::vector v1, std::vector v2) { + std::sort(v1.begin(), v1.end()); + std::sort(v2.begin(), v2.end()); + + if (v1.size() != v2.size()) { + return false; + } + + for (size_t i = 0; i < v1.size(); i++) { + if (v1[i] != v2[i]) { + return false; + } + } + + return true; +} + +void EnumerationFx::flatten_buffer_list(BufferList& blist, Buffer& buf) { + const auto nbytes = blist.total_size(); + throw_if_not_ok(buf.realloc(nbytes)); + + blist.reset_offset(); + throw_if_not_ok(blist.read(buf.data(), nbytes)); + buf.set_size(nbytes); +} + +void EnumerationFx::rm_array() { + bool is_dir; + throw_if_not_ok(ctx_.resources().vfs().is_dir(uri_, &is_dir)); + if (is_dir) { + throw_if_not_ok(ctx_.resources().vfs().remove_dir(uri_)); + } +} + +template +QueryCondition create_qc( + const char* field_name, T condition_value, const QueryConditionOp& op) { + QueryCondition ret; + + if constexpr (std::is_pod_v) { + throw_if_not_ok(ret.init(field_name, &condition_value, sizeof(T), op)); + } else { + throw_if_not_ok(ret.init( + field_name, condition_value.data(), condition_value.size(), op)); + } + + return ret; +} diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index 5e52ceecd5a5..f14e06168c78 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -85,10 +85,12 @@ if (TILEDB_CPP_API) ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/tiledb_experimental ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/array_deprecated.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/array.h + ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/array_experimental.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/array_schema.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/array_schema_evolution.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/array_schema_experimental.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/attribute.h + ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/attribute_experimental.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/config.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/consolidation_plan_experimental.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/context.h @@ -97,6 +99,7 @@ if (TILEDB_CPP_API) ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/dimension.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/dimension_label_experimental.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/domain.h + ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/enumeration_experimental.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/error.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/exception.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/filter.h @@ -108,6 +111,7 @@ if (TILEDB_CPP_API) ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/object.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/object_iter.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/query.h + ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/query_condition_experimental.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/query_condition.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/query_experimental.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/schema_base.h @@ -148,6 +152,7 @@ set(TILEDB_CORE_SOURCES ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/array_schema/dimension.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/array_schema/dimension_label.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/array_schema/domain.cc + ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/array_schema/enumeration.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/buffer/buffer.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/buffer/buffer_list.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/c_api/api_argument_validator.cc @@ -883,14 +888,22 @@ if(TILEDB_SERIALIZATION) ${CMAKE_CURRENT_SOURCE_DIR}/sm/serialization/${TILEDB_SERIALIZATION_GENERATED_SUBDIR}) # We only need to override the include path and PATH env for EP builds + # note: We run the compiler under `cmake -E env ...` because we need to run capnp executable + # the capnp driver needs to be able to find the plugin executables (eg capnpc-c++). + # For system installed capnp binaries we rely on the user to have their PATH + # include the directory with CapNProto plugin binaries. if(TILEDB_CAPNP_EP_BUILT) - list(APPEND CAPNP_COMPILE_COMMAND "${CMAKE_COMMAND}" -E env PATH="${TILEDB_EP_BASE}/install/bin" "${CAPNP_EXECUTABLE}" compile -I "${TILEDB_EP_BASE}/install/include" -oc++:"${TILEDB_SERIALIZATION_GENERATED_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/sm/serialization/tiledb-rest.capnp" --src-prefix="${CMAKE_CURRENT_SOURCE_DIR}/sm/serialization") + list(APPEND CAPNP_COMPILE_COMMAND "${CMAKE_COMMAND}" -E env PATH="${TILEDB_EP_BASE}/install/bin" "${CAPNP_EXECUTABLE}" compile -I "${TILEDB_EP_BASE}/install/include") + elseif(TILEDB_VCPKG) + set(CAPNP_PLUGIN_DIR $) + list(APPEND CAPNP_COMPILE_COMMAND "${CMAKE_COMMAND}" -E env PATH="${CAPNP_PLUGIN_DIR}" "${CAPNP_EXECUTABLE}" compile "-I${CAPNP_INCLUDE_DIRECTORY}") else() - list(APPEND CAPNP_COMPILE_COMMAND "${CAPNP_EXECUTABLE}" compile -oc++:"${TILEDB_SERIALIZATION_GENERATED_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/sm/serialization/tiledb-rest.capnp" --src-prefix "${CMAKE_CURRENT_SOURCE_DIR}/sm/serialization") + list(APPEND CAPNP_COMPILE_COMMAND "${CAPNP_EXECUTABLE}" compile) endif() - # note: run the compiler under `cmake -E env ...` because we need to run capnp executable - # the capnp driver needs to be able to find the plugin executables (eg capnpc-c++) + # Add the rest of the capnp compile command + list(APPEND CAPNP_COMPILE_COMMAND -oc++:"${TILEDB_SERIALIZATION_GENERATED_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/sm/serialization/tiledb-rest.capnp" --src-prefix="${CMAKE_CURRENT_SOURCE_DIR}/sm/serialization") + add_custom_target( update-serialization ${CMAKE_COMMAND} -E echo "CAPNP_COMPILE_COMMAND: '${CAPNP_COMPILE_COMMAND}'" diff --git a/tiledb/api/c_api/CMakeLists.txt b/tiledb/api/c_api/CMakeLists.txt index 655bc3402249..a1d2cdf7e30d 100644 --- a/tiledb/api/c_api/CMakeLists.txt +++ b/tiledb/api/c_api/CMakeLists.txt @@ -76,6 +76,9 @@ add_subdirectory(data_order) # `dimension_label`: depends on `context`, `datatype`, `data_order` add_subdirectory(dimension_label) +# `enumeration`: depends on `buffer`, `constants`, and `context` +add_subdirectory(enumeration) + # `filesystem`: no dependencies add_subdirectory(filesystem) diff --git a/tiledb/api/c_api/dimension_label/CMakeLists.txt b/tiledb/api/c_api/dimension_label/CMakeLists.txt index 9cfe7b01e707..e8f9d0d214ba 100644 --- a/tiledb/api/c_api/dimension_label/CMakeLists.txt +++ b/tiledb/api/c_api/dimension_label/CMakeLists.txt @@ -46,6 +46,7 @@ target_link_libraries(capi_dimension_label_stub PUBLIC array_schema $(type); + + bool is_ordered = false; + if (ordered != 0) { + is_ordered = true; + } + + try { + *enumeration = tiledb_enumeration_handle_t::make_handle( + std::string(name), + datatype, + cell_val_num, + is_ordered, + data, + data_size, + offsets, + offsets_size); + } catch (...) { + *enumeration = nullptr; + throw; + } + + return TILEDB_OK; +} + +void tiledb_enumeration_free(tiledb_enumeration_t** enumeration) { + ensure_output_pointer_is_valid(enumeration); + ensure_enumeration_is_valid(*enumeration); + tiledb_enumeration_handle_t::break_handle(*enumeration); +} + +capi_return_t tiledb_enumeration_get_name( + tiledb_enumeration_t* enumeration, tiledb_string_t** name) { + ensure_enumeration_is_valid(enumeration); + ensure_output_pointer_is_valid(name); + *name = tiledb_string_handle_t::make_handle(enumeration->name()); + return TILEDB_OK; +} + +capi_return_t tiledb_enumeration_get_type( + tiledb_enumeration_t* enumeration, tiledb_datatype_t* type) { + ensure_enumeration_is_valid(enumeration); + ensure_output_pointer_is_valid(type); + *type = static_cast(enumeration->type()); + return TILEDB_OK; +} + +capi_return_t tiledb_enumeration_get_cell_val_num( + tiledb_enumeration_t* enumeration, uint32_t* cell_val_num) { + ensure_enumeration_is_valid(enumeration); + ensure_output_pointer_is_valid(cell_val_num); + *cell_val_num = enumeration->cell_val_num(); + return TILEDB_OK; +} + +capi_return_t tiledb_enumeration_get_ordered( + tiledb_enumeration_t* enumeration, int* ordered) { + ensure_enumeration_is_valid(enumeration); + ensure_output_pointer_is_valid(ordered); + *ordered = static_cast(enumeration->ordered()); + return TILEDB_OK; +} + +capi_return_t tiledb_enumeration_get_data( + tiledb_enumeration_t* enumeration, const void** data, uint64_t* data_size) { + ensure_enumeration_is_valid(enumeration); + ensure_output_pointer_is_valid(data); + ensure_output_pointer_is_valid(data_size); + auto dspan = enumeration->data(); + *data = dspan.data(); + *data_size = dspan.size(); + return TILEDB_OK; +} + +capi_return_t tiledb_enumeration_get_offsets( + tiledb_enumeration_t* enumeration, + const void** offsets, + uint64_t* offsets_size) { + ensure_enumeration_is_valid(enumeration); + ensure_output_pointer_is_valid(offsets); + ensure_output_pointer_is_valid(offsets_size); + auto ospan = enumeration->offsets(); + *offsets = ospan.data(); + *offsets_size = ospan.size(); + return TILEDB_OK; +} + +capi_return_t tiledb_enumeration_dump( + tiledb_enumeration_t* enumeration, FILE* out) { + ensure_enumeration_is_valid(enumeration); + enumeration->dump(out); + return TILEDB_OK; +} + +} // namespace tiledb::api + +using tiledb::api::api_entry_context; +using tiledb::api::api_entry_void; + +capi_return_t tiledb_enumeration_alloc( + tiledb_ctx_t* ctx, + const char* name, + tiledb_datatype_t type, + uint32_t cell_val_num, + int ordered, + const void* data, + uint64_t data_size, + const void* offsets, + uint64_t offsets_size, + tiledb_enumeration_t** enumeration) noexcept { + return api_entry_context( + ctx, + name, + type, + cell_val_num, + ordered, + data, + data_size, + offsets, + offsets_size, + enumeration); +} + +void tiledb_enumeration_free(tiledb_enumeration_t** enumeration) noexcept { + return api_entry_void(enumeration); +} + +capi_return_t tiledb_enumeration_get_name( + tiledb_ctx_t* ctx, + tiledb_enumeration_t* enumeration, + tiledb_string_t** name) noexcept { + return api_entry_context( + ctx, enumeration, name); +} + +capi_return_t tiledb_enumeration_get_type( + tiledb_ctx_t* ctx, + tiledb_enumeration_t* enumeration, + tiledb_datatype_t* type) noexcept { + return api_entry_context( + ctx, enumeration, type); +} + +capi_return_t tiledb_enumeration_get_cell_val_num( + tiledb_ctx_t* ctx, + tiledb_enumeration_t* enumeration, + uint32_t* cell_val_num) noexcept { + return api_entry_context( + ctx, enumeration, cell_val_num); +} + +capi_return_t tiledb_enumeration_get_ordered( + tiledb_ctx_t* ctx, + tiledb_enumeration_t* enumeration, + int* ordered) noexcept { + return api_entry_context( + ctx, enumeration, ordered); +} + +capi_return_t tiledb_enumeration_get_data( + tiledb_ctx_t* ctx, + tiledb_enumeration_t* enumeration, + const void** data, + uint64_t* data_size) noexcept { + return api_entry_context( + ctx, enumeration, data, data_size); +} + +capi_return_t tiledb_enumeration_get_offsets( + tiledb_ctx_t* ctx, + tiledb_enumeration_t* enumeration, + const void** offsets, + uint64_t* offsets_size) noexcept { + return api_entry_context( + ctx, enumeration, offsets, offsets_size); +} + +capi_return_t tiledb_enumeration_dump( + tiledb_ctx_t* ctx, tiledb_enumeration_t* enumeration, FILE* out) noexcept { + return api_entry_context( + ctx, enumeration, out); +} diff --git a/tiledb/api/c_api/enumeration/enumeration_api_experimental.h b/tiledb/api/c_api/enumeration/enumeration_api_experimental.h new file mode 100644 index 000000000000..46326047661b --- /dev/null +++ b/tiledb/api/c_api/enumeration/enumeration_api_experimental.h @@ -0,0 +1,281 @@ +/** + * @file tiledb/api/c_api/enumeration/enumeration_api_experimental.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares the enumeration section of the C API for TileDB. + */ + +#ifndef TILEDB_CAPI_ENUMERATION_EXPERIMENTAL_H +#define TILEDB_CAPI_ENUMERATION_EXPERIMENTAL_H + +#include + +#include "../api_external_common.h" +#include "../context/context_api_external.h" +#include "../datatype/datatype_api_external.h" +#include "../string/string_api_external.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** A TileDB dimension. */ +typedef struct tiledb_enumeration_handle_t tiledb_enumeration_t; + +/** + * Creates an Enumeration. + * + * **Example:** + * + * @code{.c} + * tiledb_enumeration_t* enumeration; + * void* data = get_data(); + * uint64_t data_size = get_data_size(); + * tiledb_enumeration_alloc( + * ctx, + * TILEDB_INT64, + * cell_val_num, + * FALSE, + * data, + * data_size, + * nullptr, + * 0, + * &enumeration); + * @endcode + * + * @param ctx The TileDB context. + * @param name The name of the enumeration. + * @param type The enumeration type. + * @param cell_val_num The number of values per enumeration value. + * @param ordered Whether this enumeration should be considered as ordered. + * @param data A pointer to the enumeration value data. + * @param data_size The length of the data buffer provided. + * @param offsets A pointer to the offsets buffer if cell_vall_num + * is TILEDB_VAR_NUM. + * @param offsets_size The length of the offsets buffer, zero if no offsets. + * @param enumeration The newly allocated enumeration. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_enumeration_alloc( + tiledb_ctx_t* ctx, + const char* name, + tiledb_datatype_t type, + uint32_t cell_val_num, + int ordered, + const void* data, + uint64_t data_size, + const void* offsets, + uint64_t offsets_size, + tiledb_enumeration_t** enumeration) TILEDB_NOEXCEPT; + +/** + * Destroys a TileDB enumeration, freeing associated memory. + * + * **Example:** + * + * @code{.c} + * tiledb_enumeration_free(&enumeration); + * @endcode + * + * @param enumeration The enumeration to be destroyed. + */ +TILEDB_EXPORT void tiledb_enumeration_free(tiledb_enumeration_t** enumeration) + TILEDB_NOEXCEPT; + +/** + * Return the datatype of the enumeration values + * + * **Example:** + * + * @code{.c} + * tiledb_string_t* name_str; + * tiledb_enumeration_get_type(ctx, enumeration, &name_str); + * const char* name; + * size_t name_len; + * tiledb_string_view(name_str, &name, &name_len); + * @endcode + * + * @param ctx The TileDB context. + * @param enumeration The enumeration. + * @param name The name of the enumeration. + * @return `TILEDB_OK` or `TILEDB_ERR`. + */ +TILEDB_EXPORT capi_return_t tiledb_enumeration_get_name( + tiledb_ctx_t* ctx, + tiledb_enumeration_t* enumeration, + tiledb_string_t** name) TILEDB_NOEXCEPT; + +/** + * Return the datatype of the enumeration values + * + * **Example:** + * + * @code{.c} + * tiledb_datatype_t type; + * tiledb_enumeration_get_type(ctx, enumeration, &type); + * @endcode + * + * @param ctx The TileDB context. + * @param enumeration The enumeration. + * @param type The data type of the enumeration. + * @return `TILEDB_OK` or `TILEDB_ERR`. + */ +TILEDB_EXPORT capi_return_t tiledb_enumeration_get_type( + tiledb_ctx_t* ctx, + tiledb_enumeration_t* enumeration, + tiledb_datatype_t* type) TILEDB_NOEXCEPT; + +/** + * Return the cell value number of the enumeration values + * + * **Example:** + * + * @code{.c} + * uint32_t cell_val_num; + * tiledb_enumeration_get_cell_val_num(ctx, enumeration, &cell_val_num); + * @endcode + * + * @param ctx The TileDB context. + * @param enumeration The enumeration. + * @param type The cell value number of the enumeration. + * @return `TILEDB_OK` or `TILEDB_ERR`. + */ +TILEDB_EXPORT capi_return_t tiledb_enumeration_get_cell_val_num( + tiledb_ctx_t* ctx, + tiledb_enumeration_t* enumeration, + uint32_t* cell_val_num) TILEDB_NOEXCEPT; + +/** + * Return whether the enumeration values should be considered ordered. + * + * **Example:** + * + * @code{.c} + * int ordered; + * tiledb_enumeration_get_ordered(ctx, enumeration, &ordered); + * @endcode + * + * The cell values are considered if the value in ordered after `TILEDB_OK` + * is returned is non-zero. I.e., this is standard `int` as `bool` behavior. + * + * @param ctx The TileDB context. + * @param enumeration The enumeration. + * @param ordered A boolean value indicating whether the values are ordered. + * @return `TILEDB_OK` or `TILEDB_ERR`. + */ +TILEDB_EXPORT capi_return_t tiledb_enumeration_get_ordered( + tiledb_ctx_t* ctx, + tiledb_enumeration_t* enumeration, + int* ordered) TILEDB_NOEXCEPT; + +/** + * Return a pointer to and size of the enumerations underlying value data + * + * **Example:** + * + * @code{.c} + * void* data = NULL; + * uint64_t data_size = 0; + * tiledb_enumeration_get_data(ctx, enumeration, &data, &data_size); + * @endcode + * + * The pointer returned from this function references internal data managed + * by the enumeration. As such, clients should not attempt to free it, or + * access it beyond the lifetime of the enumeration instance. + * + * @param ctx The TileDB context. + * @param enumeration The enumeration. + * @param data The returned pointer to this enumeration value buffer. + * @param data_size The length of the buffer pointed to by data. + * @return `TILEDB_OK` or `TILEDB_ERR`. + */ +TILEDB_EXPORT capi_return_t tiledb_enumeration_get_data( + tiledb_ctx_t* ctx, + tiledb_enumeration_t* enumeration, + const void** data, + uint64_t* data_size) TILEDB_NOEXCEPT; + +/** + * Return a pointer to and size of the enumerations underlying value offsets + * + * **Example:** + * + * @code{.c} + * void* offsets = NULL; + * uint64_t offsets_size = 0; + * tiledb_enumeration_get_offsets(ctx, enumeration, &offsets, &offsets_size); + * @endcode + * + * The pointer returned from this function references internal data managed + * by the enumeration. As such, clients should not attempt to free it, or + * access it beyond the lifetime of the enumeration instance. + * + * If the enumeration values are var sized (i.e., cell_var_num is + * TILEDB_VAR_NUM) the offsets buffer will contain a `uint64_t` value for the + * starting offset of each underlying enumeration value. Note that the number + * of offsets is calculated as `offsets_size / sizeof(uint64_t)`. + * + * @param ctx The TileDB context. + * @param enumeration The enumeration. + * @param data The returned pointer to this enumeration offsets buffer. + * @param data_size The length of the buffer pointed to by offsets. + * @return `TILEDB_OK` or `TILEDB_ERR`. + */ +TILEDB_EXPORT capi_return_t tiledb_enumeration_get_offsets( + tiledb_ctx_t* ctx, + tiledb_enumeration_t* enumeration, + const void** offsets, + uint64_t* offsets_size) TILEDB_NOEXCEPT; + +/** + * Dumps the contents of an Enumeration in ASCII form to some output (e.g., + * file or stdout). + * + * **Example:** + * + * The following prints the enumeration dump to standard output. + * + * @code{.c} + * tiledb_enumeration_dump(ctx, enmr, stdout); + * @endcode + * + * @param ctx The TileDB context. + * @param attr The attribute. + * @param out The output. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error./ + */ +TILEDB_EXPORT capi_return_t tiledb_enumeration_dump( + tiledb_ctx_t* ctx, + tiledb_enumeration_t* enumeration, + FILE* out) TILEDB_NOEXCEPT; + +#ifdef __cplusplus +} +#endif + +#endif // TILEDB_CAPI_ENUMERATION_EXPERIMENTAL_H diff --git a/tiledb/api/c_api/enumeration/enumeration_api_internal.h b/tiledb/api/c_api/enumeration/enumeration_api_internal.h new file mode 100644 index 000000000000..2a984d27e03a --- /dev/null +++ b/tiledb/api/c_api/enumeration/enumeration_api_internal.h @@ -0,0 +1,139 @@ +/** + * @file tiledb/api/c_api/enumeration/enumeration_api_internal.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares the enumeration section of the C API for TileDB. + */ + +#ifndef TILEDB_CAPI_ENUMERATION_INTERNAL_H +#define TILEDB_CAPI_ENUMERATION_INTERNAL_H + +#include "enumeration_api_experimental.h" +#include "tiledb/api/c_api_support/handle/handle.h" +#include "tiledb/common/common.h" +#include "tiledb/sm/array_schema/enumeration.h" + +/** + * Handle `struct` for API enumeration objects. + */ +struct tiledb_enumeration_handle_t + : public tiledb::api::CAPIHandle { + private: + shared_ptr enumeration_; + + public: + /** Type name. */ + static constexpr std::string_view object_type_name{"enumeration"}; + + template + tiledb_enumeration_handle_t(Arg&&... arg) + : enumeration_( + tiledb::sm::Enumeration::create(std::forward(arg)...)) { + } + + /** + * Constructor from `shared_ptr` copies the shared pointer. + */ + explicit tiledb_enumeration_handle_t( + shared_ptr& e) + : enumeration_(e) { + } + + /** + * Copy the underlying enumeration object. + */ + [[nodiscard]] shared_ptr copy() { + return enumeration_; + } + + /** + * Return the name of the enumeration. + */ + [[nodiscard]] inline const std::string& name() const { + return enumeration_->name(); + } + + /** + * Return the data type of the enumeration values. + */ + [[nodiscard]] inline tiledb_datatype_t type() const { + return static_cast(enumeration_->type()); + } + + /** + * Return the cell_val_num of the enumeration values + */ + [[nodiscard]] inline uint32_t cell_val_num() const { + return enumeration_->cell_val_num(); + } + + /** + * Return a bool indicating whether the enumeration values are ordered. + */ + [[nodiscard]] inline bool ordered() const { + return enumeration_->ordered(); + } + + /** + * Return a pointer and size tuple for the underlying data type. + */ + [[nodiscard]] inline const span data() const { + return enumeration_->data(); + } + + /** + * Return a pointer and size tuple for the underlying offsets buffer. + */ + [[nodiscard]] inline const span offsets() const { + return enumeration_->offsets(); + } + + /** + * Dump a representation of the Enumeration to out + * + * @param out Where to display the output, stdout if nullptr + */ + inline void dump(FILE* out) const { + enumeration_->dump(out); + } +}; + +namespace tiledb::api { + +/** + * Returns after successfully validating an error. Throws otherwise. + * + * @param dim A possibly-valid enumeration handle. + */ +inline void ensure_enumeration_is_valid(const tiledb_enumeration_handle_t* e) { + ensure_handle_is_valid(e); +} + +} // namespace tiledb::api + +#endif // TILEDB_CAPI_ENUMERATION_INTERNAL_H diff --git a/tiledb/api/c_api/enumeration/test/CMakeLists.txt b/tiledb/api/c_api/enumeration/test/CMakeLists.txt new file mode 100644 index 000000000000..7ace61d484d8 --- /dev/null +++ b/tiledb/api/c_api/enumeration/test/CMakeLists.txt @@ -0,0 +1,32 @@ +# +# tiledb/api/c_api/enumeration/test/CMakeLists.txt +# +# The MIT License +# +# Copyright (c) 2023 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +include(unit_test) + +commence(unit_test capi_enumeration) + this_target_sources(unit_capi_enumeration.cc) + this_target_object_libraries(capi_enumeration_stub) +conclude(unit_test) diff --git a/tiledb/api/c_api/enumeration/test/compile_capi_enumeration_stub_main.cc b/tiledb/api/c_api/enumeration/test/compile_capi_enumeration_stub_main.cc new file mode 100644 index 000000000000..0b7139553f2a --- /dev/null +++ b/tiledb/api/c_api/enumeration/test/compile_capi_enumeration_stub_main.cc @@ -0,0 +1,39 @@ +/** + * @file tiledb/api/c_api/enumeration/test/compile_capi_enumeration_stub_main.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "../enumeration_api_internal.h" +#include "tiledb/sm/enums/datatype.h" + +int main() { + try { + tiledb_enumeration_handle_t e{ + "fooo", tiledb::sm::Datatype::INT32, 1, 0, nullptr, 0, nullptr, 0}; + } catch (...) { + } + return 0; +} diff --git a/tiledb/api/c_api/enumeration/test/unit_capi_enumeration.cc b/tiledb/api/c_api/enumeration/test/unit_capi_enumeration.cc new file mode 100644 index 000000000000..a185f5e4d7ad --- /dev/null +++ b/tiledb/api/c_api/enumeration/test/unit_capi_enumeration.cc @@ -0,0 +1,471 @@ +/** + * @file tiledb/api/c_api/enumeration/test/unit_capi_enumeration.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#define CATCH_CONFIG_MAIN +#include +#include "../enumeration_api_experimental.h" +#include "tiledb/api/c_api_test_support/testsupport_capi_context.h" +#include "tiledb/sm/misc/constants.h" + +// TILEDB_VAR_NUM is defined in tiledb.h which we can't use without linking +// against the entire libtiledb. Instead we'll just declare our own. +#ifndef TILEDB_VAR_NUM +#define TILEDB_VAR_NUM tiledb::sm::constants::var_num +#endif + +using namespace tiledb::api::test_support; + +struct FixedSizeEnumeration { + FixedSizeEnumeration() { + uint32_t values[5] = {1, 2, 3, 4, 5}; + auto rc = tiledb_enumeration_alloc( + ctx_.context, + "an_enumeration", + TILEDB_UINT32, + 1, + 0, + values, + sizeof(uint32_t) * 5, + nullptr, + 0, + &enumeration_); + REQUIRE(rc == TILEDB_OK); + } + + ~FixedSizeEnumeration() { + tiledb_enumeration_free(&enumeration_); + } + + ordinary_context ctx_; + tiledb_enumeration_t* enumeration_; +}; + +struct VarSizeEnumeration { + VarSizeEnumeration() { + const char* values = "foobarbazbingobango"; + uint64_t offsets[5] = {0, 3, 6, 9, 14}; + auto rc = tiledb_enumeration_alloc( + ctx_.context, + "an_enumeration", + TILEDB_STRING_UTF8, + TILEDB_VAR_NUM, + 0, + values, + strlen(values), + offsets, + 5 * sizeof(uint64_t), + &enumeration_); + REQUIRE(rc == TILEDB_OK); + } + + ~VarSizeEnumeration() { + tiledb_enumeration_free(&enumeration_); + } + + ordinary_context ctx_; + tiledb_enumeration_t* enumeration_; +}; + +TEST_CASE( + "C API: tiledb_enumeration_alloc argument validation", + "[capi][enumeration]") { + ordinary_context ctx{}; + tiledb_enumeration_t* enumeration = nullptr; + + int32_t values[5] = {1, 2, 3, 4, 5}; + const char* data = "foobarbazbingobango"; + uint64_t offsets[5] = {0, 3, 6, 9, 14}; + + SECTION("success - fixed size") { + auto rc = tiledb_enumeration_alloc( + ctx.context, + "an_enumeration", + TILEDB_UINT32, + 1, + 0, + values, + sizeof(uint32_t) * 5, + nullptr, + 0, + &enumeration); + REQUIRE(rc == TILEDB_OK); + tiledb_enumeration_free(&enumeration); + } + + SECTION("success - var size") { + auto rc = tiledb_enumeration_alloc( + ctx.context, + "an_enumeration", + TILEDB_STRING_ASCII, + TILEDB_VAR_NUM, + 0, + (void*)data, + strlen(data), + offsets, + sizeof(uint64_t) * 5, + &enumeration); + REQUIRE(rc == TILEDB_OK); + tiledb_enumeration_free(&enumeration); + } + + SECTION("failure - null context") { + auto rc = tiledb_enumeration_alloc( + nullptr, + "an_enumeration", + TILEDB_UINT32, + 1, + 0, + values, + sizeof(uint32_t) * 5, + nullptr, + 0, + &enumeration); + REQUIRE(rc == TILEDB_INVALID_CONTEXT); + tiledb_enumeration_free(&enumeration); + } + + SECTION("failure - invalid name") { + auto rc = tiledb_enumeration_alloc( + ctx.context, + nullptr, + TILEDB_UINT32, + 1, + 0, + values, + sizeof(uint32_t) * 5, + nullptr, + 0, + &enumeration); + REQUIRE(rc == TILEDB_ERR); + tiledb_enumeration_free(&enumeration); + } + + SECTION("failure - invalid datatype") { + auto rc = tiledb_enumeration_alloc( + ctx.context, + "an_enumeration", + (tiledb_datatype_t)255, + 1, + 0, + values, + sizeof(uint32_t) * 5, + nullptr, + 0, + &enumeration); + REQUIRE(rc == TILEDB_ERR); + tiledb_enumeration_free(&enumeration); + } + + SECTION("failure - data nullptr") { + auto rc = tiledb_enumeration_alloc( + ctx.context, + "an_enumeration", + TILEDB_INT32, + 1, + 0, + nullptr, + sizeof(uint32_t) * 5, + nullptr, + 0, + &enumeration); + REQUIRE(rc == TILEDB_ERR); + tiledb_enumeration_free(&enumeration); + } + + SECTION("failure - data_size == 0") { + auto rc = tiledb_enumeration_alloc( + ctx.context, + "an_enumeration", + TILEDB_INT32, + 1, + 0, + values, + 0, + nullptr, + 0, + &enumeration); + REQUIRE(rc == TILEDB_ERR); + tiledb_enumeration_free(&enumeration); + } + + SECTION("failure - enumeration nullptr") { + auto rc = tiledb_enumeration_alloc( + ctx.context, + "an_enumeration", + TILEDB_INT32, + 1, + 0, + values, + sizeof(uint32_t) * 5, + nullptr, + 0, + nullptr); + REQUIRE(rc == TILEDB_ERR); + tiledb_enumeration_free(&enumeration); + } + + SECTION("failure - offsets nullptr") { + auto rc = tiledb_enumeration_alloc( + ctx.context, + "an_enumeration", + TILEDB_STRING_ASCII, + TILEDB_VAR_NUM, + 0, + (void*)data, + strlen(data), + nullptr, + sizeof(uint64_t) * 5, + &enumeration); + REQUIRE(rc == TILEDB_ERR); + tiledb_enumeration_free(&enumeration); + } + + SECTION("failure - offsets_size == 0") { + auto rc = tiledb_enumeration_alloc( + ctx.context, + "an_enumeration", + TILEDB_STRING_ASCII, + TILEDB_VAR_NUM, + 0, + (void*)data, + strlen(data), + offsets, + 0, + &enumeration); + REQUIRE(rc == TILEDB_ERR); + tiledb_enumeration_free(&enumeration); + } +} + +TEST_CASE( + "C API: tiledb_enumeration_free argument validation", + "[capi][enumeration]") { + REQUIRE_NOTHROW(tiledb_enumeration_free(nullptr)); +} + +TEST_CASE( + "C API: tiledb_enumeration_get_type argument validation", + "[capi][enumeration]") { + FixedSizeEnumeration fe; + VarSizeEnumeration ve; + tiledb_datatype_t dt; + + SECTION("success") { + auto rc = + tiledb_enumeration_get_type(fe.ctx_.context, fe.enumeration_, &dt); + REQUIRE(rc == TILEDB_OK); + REQUIRE(dt == TILEDB_UINT32); + + rc = tiledb_enumeration_get_type(ve.ctx_.context, ve.enumeration_, &dt); + REQUIRE(rc == TILEDB_OK); + REQUIRE(dt == TILEDB_STRING_UTF8); + } + + SECTION("failure - invalid context") { + auto rc = tiledb_enumeration_get_type(nullptr, fe.enumeration_, &dt); + REQUIRE(rc == TILEDB_INVALID_CONTEXT); + } + + SECTION("failure - invalid enumeration") { + auto rc = tiledb_enumeration_get_type(fe.ctx_.context, nullptr, &dt); + REQUIRE(rc == TILEDB_ERR); + } + + SECTION("failure - invalid type pointer") { + auto rc = + tiledb_enumeration_get_type(fe.ctx_.context, fe.enumeration_, nullptr); + REQUIRE(rc == TILEDB_ERR); + } +} + +TEST_CASE( + "C API: tiledb_enumeration_get_cell_val_num argument validation", + "[capi][enumeration]") { + FixedSizeEnumeration fe; + VarSizeEnumeration ve; + uint32_t cvn; + + SECTION("success") { + auto rc = tiledb_enumeration_get_cell_val_num( + fe.ctx_.context, fe.enumeration_, &cvn); + REQUIRE(rc == TILEDB_OK); + REQUIRE(cvn == 1); + + rc = tiledb_enumeration_get_cell_val_num( + ve.ctx_.context, ve.enumeration_, &cvn); + REQUIRE(rc == TILEDB_OK); + REQUIRE(cvn == TILEDB_VAR_NUM); + } + + SECTION("failure - invalid context") { + auto rc = + tiledb_enumeration_get_cell_val_num(nullptr, fe.enumeration_, &cvn); + REQUIRE(rc == TILEDB_INVALID_CONTEXT); + } + + SECTION("failure - invalid enumeration") { + auto rc = + tiledb_enumeration_get_cell_val_num(fe.ctx_.context, nullptr, &cvn); + REQUIRE(rc == TILEDB_ERR); + } + + SECTION("failure - invalid cell_val_num pointer") { + auto rc = tiledb_enumeration_get_cell_val_num( + fe.ctx_.context, fe.enumeration_, nullptr); + REQUIRE(rc == TILEDB_ERR); + } +} + +TEST_CASE( + "C API: tiledb_enumeration_get_ordered argument validation", + "[capi][enumeration]") { + FixedSizeEnumeration fe; + VarSizeEnumeration ve; + int o; + + SECTION("success") { + auto rc = + tiledb_enumeration_get_ordered(fe.ctx_.context, fe.enumeration_, &o); + REQUIRE(rc == TILEDB_OK); + REQUIRE(!o); + + rc = tiledb_enumeration_get_ordered(ve.ctx_.context, ve.enumeration_, &o); + REQUIRE(rc == TILEDB_OK); + REQUIRE(!o); + } + + SECTION("failure - invalid context") { + auto rc = tiledb_enumeration_get_ordered(nullptr, fe.enumeration_, &o); + REQUIRE(rc == TILEDB_INVALID_CONTEXT); + } + + SECTION("failure - invalid enumeration") { + auto rc = tiledb_enumeration_get_ordered(fe.ctx_.context, nullptr, &o); + REQUIRE(rc == TILEDB_ERR); + } + + SECTION("failure - invalid ordered pointer") { + auto rc = tiledb_enumeration_get_ordered( + fe.ctx_.context, fe.enumeration_, nullptr); + REQUIRE(rc == TILEDB_ERR); + } +} + +TEST_CASE( + "C API: tiledb_enumeration_get_data argument validation", + "[capi][enumeration]") { + FixedSizeEnumeration fe; + VarSizeEnumeration ve; + const void* d; + uint64_t ds; + + uint32_t fixed_expect[5] = {1, 2, 3, 4, 5}; + const char* var_expect = "foobarbazbingobango"; + + SECTION("success") { + auto rc = + tiledb_enumeration_get_data(fe.ctx_.context, fe.enumeration_, &d, &ds); + REQUIRE(rc == TILEDB_OK); + REQUIRE(std::memcmp(fixed_expect, d, sizeof(uint32_t) * 5) == 0); + REQUIRE(ds == sizeof(uint32_t) * 5); + + rc = tiledb_enumeration_get_data(ve.ctx_.context, ve.enumeration_, &d, &ds); + REQUIRE(rc == TILEDB_OK); + REQUIRE(std::memcmp(var_expect, d, strlen(var_expect)) == 0); + REQUIRE(ds == strlen(var_expect)); + } + + SECTION("failure - invalid context") { + auto rc = tiledb_enumeration_get_data(nullptr, fe.enumeration_, &d, &ds); + REQUIRE(rc == TILEDB_INVALID_CONTEXT); + } + + SECTION("failure - invalid enumeration") { + auto rc = tiledb_enumeration_get_data(fe.ctx_.context, nullptr, &d, &ds); + REQUIRE(rc == TILEDB_ERR); + } + + SECTION("failure - invalid data pointer") { + auto rc = tiledb_enumeration_get_data( + fe.ctx_.context, fe.enumeration_, nullptr, &ds); + REQUIRE(rc == TILEDB_ERR); + } + + SECTION("failure - invalid data size pointer") { + auto rc = tiledb_enumeration_get_data( + fe.ctx_.context, fe.enumeration_, &d, nullptr); + REQUIRE(rc == TILEDB_ERR); + } +} + +TEST_CASE( + "C API: tiledb_enumeration_get_offsets argument validation", + "[capi][enumeration]") { + FixedSizeEnumeration fe; + VarSizeEnumeration ve; + const void* o; + uint64_t os; + + uint64_t var_expect[5] = {0, 3, 6, 9, 14}; + + SECTION("success") { + auto rc = tiledb_enumeration_get_offsets( + fe.ctx_.context, fe.enumeration_, &o, &os); + REQUIRE(rc == TILEDB_OK); + REQUIRE(o == nullptr); + REQUIRE(os == 0); + + rc = tiledb_enumeration_get_offsets( + ve.ctx_.context, ve.enumeration_, &o, &os); + REQUIRE(rc == TILEDB_OK); + REQUIRE(std::memcmp(var_expect, o, sizeof(uint64_t) * 5) == 0); + REQUIRE(os == sizeof(uint64_t) * 5); + } + + SECTION("failure - invalid context") { + auto rc = tiledb_enumeration_get_offsets(nullptr, fe.enumeration_, &o, &os); + REQUIRE(rc == TILEDB_INVALID_CONTEXT); + } + + SECTION("failure - invalid enumeration") { + auto rc = tiledb_enumeration_get_offsets(fe.ctx_.context, nullptr, &o, &os); + REQUIRE(rc == TILEDB_ERR); + } + + SECTION("failure - invalid offsets pointer") { + auto rc = tiledb_enumeration_get_offsets( + fe.ctx_.context, fe.enumeration_, nullptr, &os); + REQUIRE(rc == TILEDB_ERR); + } + + SECTION("failure - invalid offsets size pointer") { + auto rc = tiledb_enumeration_get_offsets( + fe.ctx_.context, fe.enumeration_, &o, nullptr); + REQUIRE(rc == TILEDB_ERR); + } +} diff --git a/tiledb/sm/array/array.cc b/tiledb/sm/array/array.cc index f9a05e52cba8..5a01fc1008ed 100644 --- a/tiledb/sm/array/array.cc +++ b/tiledb/sm/array/array.cc @@ -581,6 +581,48 @@ void Array::delete_fragments_list( } } +shared_ptr Array::get_enumeration( + const std::string& enumeration_name) { + if (remote_) { + throw ArrayException("Unable to load all enumerations; Array is remote."); + } + + if (!is_open_) { + throw ArrayException("Cannot get enumeration; Array is not open"); + } + + if (array_schema_latest_->is_enumeration_loaded(enumeration_name)) { + return array_schema_latest_->get_enumeration(enumeration_name); + } + + return array_dir_.load_enumeration( + array_schema_latest_, enumeration_name, get_encryption_key()); +} + +void Array::load_all_enumerations(bool latest_only) { + if (remote_) { + throw ArrayException("Unable to load all enumerations; Array is remote."); + } + + if (!is_open_) { + throw ArrayException("Cannot load all enumerations; Array is not open"); + } + + std::vector> schemas; + if (latest_only) { + schemas.emplace_back(array_schema_latest_); + } else { + schemas.reserve(array_schemas_all_.size()); + for (auto& iter : array_schemas_all_) { + schemas.emplace_back(iter.second); + } + } + + for (auto& schema : schemas) { + array_dir_.load_all_enumerations(schema, get_encryption_key()); + } +} + bool Array::is_empty() const { return fragment_metadata_.empty(); } diff --git a/tiledb/sm/array/array.h b/tiledb/sm/array/array.h index 92f2fccf14f4..a005e15b4050 100644 --- a/tiledb/sm/array/array.h +++ b/tiledb/sm/array/array.h @@ -257,6 +257,31 @@ class Array { return fragment_metadata_; } + /** + * Get the enumeration for the given name. + * + * This function retrieves the enumeration for the given name. If the + * corresponding enumeration has not been loaded from storage it is + * loaded before this function returns. + * + * @param enumeration_name The name of the enumeration. + * @return shared_ptr or nullptr on failure. + */ + shared_ptr get_enumeration( + const std::string& enumeration_name); + + /** + * Load all enumerations for the array. + * + * Ensure that all enumerations have been loaded. If latest_only is true + * (the default) then only enumerations for the latest schema are loaded. + * When latest_only is false, all schemas have their enumerations loaded. + * + * @param latest_only Whether to load enumerations for just the latest + * schema or all schemas. + */ + void load_all_enumerations(bool latest_only = true); + /** * Returns `true` if the array is empty at the time it is opened. * The funciton returns `false` if the array is not open. diff --git a/tiledb/sm/array/array_directory.cc b/tiledb/sm/array/array_directory.cc index 6223d0536f2d..579c2d0c64b2 100644 --- a/tiledb/sm/array/array_directory.cc +++ b/tiledb/sm/array/array_directory.cc @@ -33,6 +33,7 @@ #include "tiledb/sm/array/array_directory.h" #include "tiledb/common/logger.h" #include "tiledb/common/stdx_string.h" +#include "tiledb/sm/array_schema/enumeration.h" #include "tiledb/sm/filesystem/vfs.h" #include "tiledb/sm/misc/constants.h" #include "tiledb/sm/misc/parallel_functions.h" @@ -187,6 +188,46 @@ ArrayDirectory::load_all_array_schemas( return array_schemas; } +shared_ptr ArrayDirectory::load_enumeration( + shared_ptr schema, + const std::string& enumeration_name, + const EncryptionKey& encryption_key) const { + auto timer_se = resources_.get().stats().start_timer("sm_load_enumeration"); + + auto path_name = schema->get_enumeration_path_name(enumeration_name); + + auto enmr_uri = uri_.join_path(constants::array_schema_dir_name) + .join_path(constants::array_enumerations_dir_name) + .join_path(path_name); + + auto&& tile = GenericTileIO::load(resources_, enmr_uri, 0, encryption_key); + resources_.get().stats().add_counter("read_enumeration_size", tile.size()); + + Deserializer deserializer(tile.data(), tile.size()); + auto enum_ptr = Enumeration::deserialize(deserializer); + + schema->store_enumeration(enum_ptr); + + return enum_ptr; +} + +void ArrayDirectory::load_all_enumerations( + shared_ptr schema, const EncryptionKey& encryption_key) const { + std::vector enmr_names; + for (auto& enmr_name : schema->get_enumeration_names()) { + if (schema->is_enumeration_loaded(enmr_name)) { + continue; + } + enmr_names.push_back(enmr_name); + } + + auto& tp = resources_.get().compute_tp(); + throw_if_not_ok(parallel_for(&tp, 0, enmr_names.size(), [&](size_t i) { + load_enumeration(schema, enmr_names[i], encryption_key); + return Status::Ok(); + })); +} + const URI& ArrayDirectory::uri() const { return uri_; } @@ -1205,10 +1246,12 @@ Status ArrayDirectory::compute_array_schema_uris( if (!array_schema_dir_uris.empty()) { array_schema_uris_.reserve( array_schema_uris_.size() + array_schema_dir_uris.size()); - std::copy( - array_schema_dir_uris.begin(), - array_schema_dir_uris.end(), - std::back_inserter(array_schema_uris_)); + for (auto& uri : array_schema_dir_uris) { + if (uri.last_path_part() == constants::array_enumerations_dir_name) { + continue; + } + array_schema_uris_.push_back(uri); + } } return Status::Ok(); diff --git a/tiledb/sm/array/array_directory.h b/tiledb/sm/array/array_directory.h index 76adc29f0393..8d2e35348b34 100644 --- a/tiledb/sm/array/array_directory.h +++ b/tiledb/sm/array/array_directory.h @@ -385,6 +385,29 @@ class ArrayDirectory { std::unordered_map> load_all_array_schemas(const EncryptionKey& encryption_key) const; + /** + * Load an enumeration from schema with the given name. + * + * @param schema The ArraySchema that references the enumeration name. + * @param enumeration_name The name of the enumeration to load. + * @param encryption_key The encryption key to use. + * @return shared_ptr The loaded enumeration. + */ + shared_ptr load_enumeration( + shared_ptr schema, + const std::string& enumeration_name, + const EncryptionKey& encryption_key) const; + + /** + * Load all enumerations for the given schema. + * + * @param schema The ArraySchema to load Enumerations for + * @param encryption_key The encryption key to use. + */ + void load_all_enumerations( + shared_ptr schema, + const EncryptionKey& encryption_key) const; + /** Returns the array URI. */ const URI& uri() const; diff --git a/tiledb/sm/array/test/CMakeLists.txt b/tiledb/sm/array/test/CMakeLists.txt index 3f06e3922eeb..62b222de11ea 100644 --- a/tiledb/sm/array/test/CMakeLists.txt +++ b/tiledb/sm/array/test/CMakeLists.txt @@ -28,7 +28,7 @@ include(unit_test) commence(unit_test array) this_target_sources(main.cc unit_array_directory.cc) - this_target_link_libraries(array) + this_target_link_libraries(array context_resources) conclude(unit_test) commence(unit_test consistency) diff --git a/tiledb/sm/array_schema/CMakeLists.txt b/tiledb/sm/array_schema/CMakeLists.txt index f9616f3b7092..bff3c61bb334 100644 --- a/tiledb/sm/array_schema/CMakeLists.txt +++ b/tiledb/sm/array_schema/CMakeLists.txt @@ -32,7 +32,14 @@ include(object_library) # commence(object_library attribute) this_target_sources(attribute.cc) - this_target_object_libraries(baseline buffer constants filter_pipeline range stringx) + this_target_object_libraries( + baseline + buffer + constants + filter_pipeline + range + stringx + uuid) conclude(object_library) # @@ -51,13 +58,27 @@ commence(object_library domain) this_target_object_libraries(datum dimension math) conclude(object_library) +# +# `enumeration` object library +# +commence(object_library enumeration) + this_target_sources(enumeration.cc) + this_target_object_libraries(buffer constants uuid) +conclude(object_library) + # # `array_schema` object library # commence(object_library array_schema) this_target_sources(array_schema.cc dimension_label.cc) this_target_object_libraries( - attribute domain time uri_format uuid vfs) + attribute domain enumeration time uri_format uuid vfs) conclude(object_library) +# This is linked outside the object_library scope because ContextResources +# is recompiled as part of the capi_context_stub. Including context_resources +# here like this prevents many headaches revolving around duplicate symbols +# when linking executables. +target_link_libraries(compile_array_schema PRIVATE context_resources generic_tile_io) + add_test_subdirectory() diff --git a/tiledb/sm/array_schema/array_schema.cc b/tiledb/sm/array_schema/array_schema.cc index f9c8aa8d5010..ff299b722198 100644 --- a/tiledb/sm/array_schema/array_schema.cc +++ b/tiledb/sm/array_schema/array_schema.cc @@ -39,6 +39,7 @@ #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/array_schema/dimension_label.h" #include "tiledb/sm/array_schema/domain.h" +#include "tiledb/sm/array_schema/enumeration.h" #include "tiledb/sm/buffer/buffer.h" #include "tiledb/sm/enums/array_type.h" #include "tiledb/sm/enums/compressor.h" @@ -49,7 +50,10 @@ #include "tiledb/sm/filter/compression_filter.h" #include "tiledb/sm/filter/webp_filter.h" #include "tiledb/sm/misc/hilbert.h" +#include "tiledb/sm/misc/integral_type_casts.h" #include "tiledb/sm/misc/tdb_time.h" +#include "tiledb/sm/tile/generic_tile_io.h" +#include "tiledb/storage_format/uri/generate_uri.h" #include "tiledb/storage_format/uri/parse_uri.h" #include @@ -120,6 +124,45 @@ ArraySchema::ArraySchema( uint64_t capacity, std::vector> attributes, std::vector> dim_label_refs, + std::unordered_map enumeration_path_map, + FilterPipeline cell_var_offsets_filters, + FilterPipeline cell_validity_filters, + FilterPipeline coords_filters) + : ArraySchema( + uri, + version, + timestamp_range, + name, + array_type, + allows_dups, + domain, + cell_order, + tile_order, + capacity, + attributes, + dim_label_refs, + {}, + enumeration_path_map, + cell_var_offsets_filters, + cell_validity_filters, + coords_filters) { +} + +ArraySchema::ArraySchema( + URI uri, + uint32_t version, + std::pair timestamp_range, + std::string name, + ArrayType array_type, + bool allows_dups, + shared_ptr domain, + Layout cell_order, + Layout tile_order, + uint64_t capacity, + std::vector> attributes, + std::vector> dim_label_refs, + std::vector> enumerations, + std::unordered_map enumeration_path_map, FilterPipeline cell_var_offsets_filters, FilterPipeline cell_validity_filters, FilterPipeline coords_filters) @@ -135,6 +178,7 @@ ArraySchema::ArraySchema( , capacity_(capacity) , attributes_(attributes) , dimension_labels_(dim_label_refs) + , enumeration_path_map_(enumeration_path_map) , cell_var_offsets_filters_(cell_var_offsets_filters) , cell_validity_filters_(cell_validity_filters) , coords_filters_(coords_filters) { @@ -160,6 +204,15 @@ ArraySchema::ArraySchema( dimension_label_map_[label->name()] = label.get(); } + for (auto& [enmr_name, enmr_uri] : enumeration_path_map_) { + (void)enmr_uri; + enumeration_map_[enmr_name] = nullptr; + } + + for (const auto& enmr : enumerations) { + enumeration_map_[enmr->name()] = enmr; + } + // Check array schema is valid. st = check_double_delta_compressor(coords_filters_); if (!st.ok()) { @@ -195,9 +248,13 @@ ArraySchema::ArraySchema(const ArraySchema& array_schema) { throw_if_not_ok(set_domain(array_schema.domain_)); - attribute_map_.clear(); - for (auto attr : array_schema.attributes_) - throw_if_not_ok(add_attribute(attr, false)); + enumeration_map_ = array_schema.enumeration_map_; + enumeration_path_map_ = array_schema.enumeration_path_map_; + + for (auto attr : array_schema.attributes_) { + attributes_.emplace_back(attr); + attribute_map_[attr->name()] = attr.get(); + } // Create dimension label map for (const auto& label : array_schema.dimension_labels_) { @@ -646,6 +703,11 @@ void ArraySchema::dump(FILE* out) const { attr->dump(out); } + for (auto& enmr_iter : enumeration_map_) { + fprintf(out, "\n"); + enmr_iter.second->dump(out); + } + for (auto& label : dimension_labels_) { fprintf(out, "\n"); label->dump(out); @@ -768,6 +830,21 @@ void ArraySchema::serialize(Serializer& serializer) const { for (auto& label : dimension_labels_) { label->serialize(serializer, version); } + + // Write Enumeration path map + auto enmr_num = + utils::safe_integral_cast(enumeration_map_.size()); + + serializer.write(enmr_num); + for (auto& [enmr_name, enmr_uri] : enumeration_path_map_) { + auto enmr_name_size = static_cast(enmr_name.size()); + serializer.write(enmr_name_size); + serializer.write(enmr_name.data(), enmr_name_size); + + auto enmr_uri_size = static_cast(enmr_uri.size()); + serializer.write(enmr_uri_size); + serializer.write(enmr_uri.data(), enmr_uri_size); + } } Layout ArraySchema::tile_order() const { @@ -844,6 +921,51 @@ Status ArraySchema::add_attribute( return LOG_STATUS(Status_ArraySchemaError(msg)); } + auto enmr_name = attr->get_enumeration_name(); + if (enmr_name.has_value()) { + // The referenced enumeration must exist when the attribut is added + auto iter = enumeration_map_.find(enmr_name.value()); + if (iter == enumeration_map_.end()) { + std::string msg = + "Cannot add attribute; Attribute refers to an " + "unknown enumeration named '" + + enmr_name.value() + "'."; + return LOG_STATUS(Status_ArraySchemaError(msg)); + } + + // This attribute must have an integral datatype to support Enumerations + if (!datatype_is_integer(attr->type())) { + std::string msg = "Unable to use enumeration with attribute '" + + attr->name() + + "', attribute must have an integral data type, not " + + datatype_str(attr->type()); + return LOG_STATUS(Status_ArraySchemaError(msg)); + } + + // The attribute must have a cell_val_num of 1 + if (attr->cell_val_num() != 1) { + std::string msg = + "Attributes with enumerations must have a cell_val_num of 1."; + return LOG_STATUS(Status_ArraySchemaError(msg)); + } + + auto enmr = get_enumeration(enmr_name.value()); + if (enmr == nullptr) { + throw ArraySchemaException( + "Cannot add attribute referencing enumeration '" + enmr_name.value() + + "' as the enumeration has not been loaded."); + } + + // The +1 here is because of 0 being a valid index into the enumeration. + if (datatype_max_integral_value(attr->type()) <= enmr->elem_count()) { + throw ArraySchemaException( + "Unable to use enumeration '" + enmr_name.value() + + "' for attribute '" + attr->name() + + "' because the attribute's type is not large enough to represent " + "all enumeration values."); + } + } + // Create new attribute and potentially set a default name attributes_.emplace_back(attr); attribute_map_[attr->name()] = attr.get(); @@ -955,6 +1077,150 @@ Status ArraySchema::drop_attribute(const std::string& attr_name) { return Status::Ok(); } +void ArraySchema::add_enumeration(shared_ptr enmr) { + if (enmr == nullptr) { + throw ArraySchemaException( + "Error adding enumeration. Enumeration " + "must not be nullptr."); + } + + if (enumeration_map_.find(enmr->name()) != enumeration_map_.end()) { + throw ArraySchemaException( + "Error adding enumeration. Enumeration with name '" + enmr->name() + + "' already exists in this ArraySchema."); + } + + enumeration_map_[enmr->name()] = enmr; + enumeration_path_map_[enmr->name()] = enmr->path_name(); +} + +void ArraySchema::store_enumeration(shared_ptr enmr) { + if (enmr == nullptr) { + throw ArraySchemaException( + "Error storing enumeration. Enumeration must not be nullptr."); + } + + auto name_iter = enumeration_map_.find(enmr->name()); + if (name_iter == enumeration_map_.end()) { + throw ArraySchemaException( + "Error storing enumeration. Unknown enumeration name '" + enmr->name() + + "'."); + } + + if (name_iter->second != nullptr) { + throw ArraySchemaException( + "Error storing enumeration. Enumeration named '" + enmr->name() + + "' has already been stored."); + } + + auto path_iter = enumeration_path_map_.find(enmr->name()); + if (path_iter == enumeration_path_map_.end()) { + throw ArraySchemaException( + "Error storing enumeration. Missing path name map entry."); + } + + if (path_iter->second != enmr->path_name()) { + throw ArraySchemaException( + "Error storing enumeration. Path name mismatch for enumeration " + "named '" + + enmr->name() + "'."); + } + + name_iter->second = enmr; +} + +bool ArraySchema::has_enumeration(const std::string& enmr_name) const { + return enumeration_map_.find(enmr_name) != enumeration_map_.end(); +} + +std::vector ArraySchema::get_enumeration_names() const { + std::vector enmr_names; + for (auto& entry : enumeration_path_map_) { + enmr_names.emplace_back(entry.first); + } + return enmr_names; +} + +std::vector ArraySchema::get_loaded_enumeration_names() const { + std::vector enmr_names; + for (auto& entry : enumeration_map_) { + if (entry.second != nullptr) { + enmr_names.emplace_back(entry.first); + } + } + return enmr_names; +} + +bool ArraySchema::is_enumeration_loaded( + const std::string& enumeration_name) const { + auto iter = enumeration_map_.find(enumeration_name); + + if (iter == enumeration_map_.end()) { + throw ArraySchemaException( + "Unable to check if unknown enumeration is loaded. No enumeration " + "named '" + + enumeration_name + "'."); + } + + return iter->second != nullptr; +} + +shared_ptr ArraySchema::get_enumeration( + const std::string& enmr_name) const { + auto iter = enumeration_map_.find(enmr_name); + if (iter == enumeration_map_.end()) { + throw ArraySchemaException( + "Unable to get enumeration. Unknown enumeration named '" + enmr_name + + "'."); + } + + return iter->second; +} + +const std::string& ArraySchema::get_enumeration_path_name( + const std::string& enmr_name) const { + auto iter = enumeration_path_map_.find(enmr_name); + if (iter == enumeration_path_map_.end()) { + throw ArraySchemaException( + "Unable to get enumeration path name. Unknown enumeration named '" + + enmr_name + "'."); + } + + return iter->second; +} + +void ArraySchema::drop_enumeration(const std::string& enmr_name) { + std::lock_guard lock(mtx_); + + if (enmr_name.empty()) { + throw ArraySchemaException( + "Error dropping enumeration, empty names are invalid."); + } + + auto it = enumeration_map_.find(enmr_name); + if (it == enumeration_map_.end()) { + throw ArraySchemaException( + "Error dropping enumeration, no enumeration named '" + enmr_name + + "'."); + } + + for (auto attr : attributes_) { + auto attr_enmr_name = attr->get_enumeration_name(); + if (!attr_enmr_name.has_value()) { + continue; + } + if (attr_enmr_name.value() == enmr_name) { + throw ArraySchemaException( + "Unable to drop enumeration '" + enmr_name + "' as it is used by " + + " attribute '" + attr->name() + "'."); + } + } + + // Drop from both the path and name maps. + enumeration_path_map_.erase(it->first); + enumeration_map_.erase(it); +} + // #TODO Add security validation on incoming URI ArraySchema ArraySchema::deserialize( Deserializer& deserializer, const URI& uri) { @@ -1050,6 +1316,23 @@ ArraySchema ArraySchema::deserialize( } } + // Load enumeration name to path map + std::unordered_map enumeration_path_map; + if (version >= constants::enumerations_min_format_version) { + uint32_t enmr_num = deserializer.read(); + for (uint32_t i = 0; i < enmr_num; i++) { + auto enmr_name_size = deserializer.read(); + std::string enmr_name( + deserializer.get_ptr(enmr_name_size), enmr_name_size); + + auto enmr_path_size = deserializer.read(); + std::string enmr_path_name( + deserializer.get_ptr(enmr_path_size), enmr_path_size); + + enumeration_path_map[enmr_name] = enmr_path_name; + } + } + // Validate if (cell_order == Layout::HILBERT && domain->dim_num() > Hilbert::HC_MAX_DIM) { @@ -1092,6 +1375,7 @@ ArraySchema ArraySchema::deserialize( capacity, attributes, dimension_labels, + enumeration_path_map, cell_var_filters, cell_validity_filters, coords_filters); @@ -1291,6 +1575,7 @@ const std::string& ArraySchema::name() const { /* ****************************** */ /* PRIVATE METHODS */ /* ****************************** */ + void ArraySchema::check_attribute_dimension_label_names() const { std::set names; // Check attribute and dimension names are unique. diff --git a/tiledb/sm/array_schema/array_schema.h b/tiledb/sm/array_schema/array_schema.h index d4a8bc64e15a..f9a15720bfa1 100644 --- a/tiledb/sm/array_schema/array_schema.h +++ b/tiledb/sm/array_schema/array_schema.h @@ -43,6 +43,7 @@ #include "tiledb/sm/misc/constants.h" #include "tiledb/sm/misc/hilbert.h" #include "tiledb/sm/misc/uuid.h" +#include "tiledb/sm/storage_manager/context_resources.h" using namespace tiledb::common; @@ -55,6 +56,7 @@ class ConstBuffer; class Dimension; class DimensionLabel; class Domain; +class Enumeration; enum class ArrayType : uint8_t; enum class Compressor : uint8_t; @@ -108,6 +110,8 @@ class ArraySchema { * @param tile_order The tile order. * @param capacity The tile capacity for the case of sparse fragments. * @param attributes The array attributes. + * @param dimension_labels The array dimension labels. + * @param enumeration_path_map The array enumeration path map * @param cell_var_offsets_filters * The filter pipeline run on offset tiles for var-length attributes. * @param cell_validity_filters @@ -127,6 +131,47 @@ class ArraySchema { uint64_t capacity, std::vector> attributes, std::vector> dimension_labels, + std::unordered_map enumeration_path_map, + FilterPipeline cell_var_offsets_filters, + FilterPipeline cell_validity_filters, + FilterPipeline coords_filters); + + /** Constructor. + * @param uri The URI of the array schema file. + * @param version The format version of this array schema. + * @param timestamp_range The timestamp the array schema was written. + * @param name The file name of the schema in timestamp_timestamp_uuid format. + * @param array_type The array type. + * @param allows_dups True if the (sparse) array allows coordinate duplicates. + * @param domain The array domain. + * @param cell_order The cell order. + * @param tile_order The tile order. + * @param capacity The tile capacity for the case of sparse fragments. + * @param attributes The array attributes. + * @param dimension_labels The array dimension labels. + * @param enumerations The array enumerations + * @param enumeration_path_map The array enumeration path map + * @param cell_var_offsets_filters + * The filter pipeline run on offset tiles for var-length attributes. + * @param cell_validity_filters + * The filter pipeline run on validity tiles for nullable attributes. + * @param coords_filters The filter pipeline run on coordinate tiles. + **/ + ArraySchema( + URI uri, + uint32_t version, + std::pair timestamp_range, + std::string name, + ArrayType array_type, + bool allows_dups, + shared_ptr domain, + Layout cell_order, + Layout tile_order, + uint64_t capacity, + std::vector> attributes, + std::vector> dimension_labels, + std::vector> enumerations, + std::unordered_map enumeration_path_map, FilterPipeline cell_var_offsets_filters, FilterPipeline cell_validity_filters, FilterPipeline coords_filters); @@ -363,6 +408,78 @@ class ArraySchema { */ Status drop_attribute(const std::string& attr_name); + /** + * Add an Enumeration to this ArraySchema. + * + * @param enmr The enumeration to add. + */ + void add_enumeration(shared_ptr enmr); + + /** + * Check if an enumeration exists with the given name. + * + * @param enmr_name The name to check + * @return bool Whether the enumeration exists. + */ + bool has_enumeration(const std::string& enmr_name) const; + + /** + * Store a known enumeration on this ArraySchema after the schema + * was loaded. This allows for only incuring the cost of loading an + * enumeration when it is needed. An exception is thrown if the + * Enumeration is unknown to this ArraySchema. + * + * @param enmr The Enumeration to store. + */ + void store_enumeration(shared_ptr enmr); + + /** + * Get a vector of Enumeration names. + * + * @return A vector of enumeration names. + */ + std::vector get_enumeration_names() const; + + /** + * Get a vector of loaded Enumeration names. + * + * @return A vector of loaded enumeration names. + */ + std::vector get_loaded_enumeration_names() const; + + /** + * Check if a given enumeration has already been loaded. + * + * @param enumeration_name The name of the enumeration to check + * @return bool Whether the enumeration has been loaded or not. + */ + bool is_enumeration_loaded(const std::string& enumeration_name) const; + + /** + * Get an Enumeration by name. Throws if the attribute is unknown. + * + * @param enmr_name The name of the Enumeration. + * @return shared_ptr + */ + shared_ptr get_enumeration( + const std::string& enmr_name) const; + + /** + * Get an Enumeration's object name. Throws if the attribute is unknown. + * + * @param enmr_name The name of the Enumeration. + * @return The path name of the enumeration on disk + */ + const std::string& get_enumeration_path_name( + const std::string& enmr_name) const; + + /** + * Drop an enumeration + * + * @param enumeration_name The enumeration to drop. + */ + void drop_enumeration(const std::string& enmr_name); + /** * It assigns values to the members of the object from the input buffer. * @@ -546,6 +663,13 @@ class ArraySchema { /** A map from the dimension label names to the label schemas. */ std::unordered_map dimension_label_map_; + /** A map of Enumeration names to Enumeration pointers. */ + std::unordered_map> + enumeration_map_; + + /** A map of Enumeration names to Enumeration URIs */ + std::unordered_map enumeration_path_map_; + /** The filter pipeline run on offset tiles for var-length attributes. */ FilterPipeline cell_var_offsets_filters_; @@ -591,6 +715,9 @@ class ArraySchema { void check_webp_filter() const; + // Check whether attributes referencing enumerations are valid. + void check_enumerations() const; + /** Clears all members. Use with caution! */ void clear(); }; diff --git a/tiledb/sm/array_schema/array_schema_evolution.cc b/tiledb/sm/array_schema/array_schema_evolution.cc index bf285733c71e..f2c59846ee6b 100644 --- a/tiledb/sm/array_schema/array_schema_evolution.cc +++ b/tiledb/sm/array_schema/array_schema_evolution.cc @@ -35,10 +35,12 @@ #include "tiledb/common/common.h" #include "tiledb/common/heap_memory.h" #include "tiledb/common/logger.h" +#include "tiledb/common/status.h" #include "tiledb/sm/array_schema/array_schema.h" #include "tiledb/sm/array_schema/attribute.h" #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/array_schema/domain.h" +#include "tiledb/sm/array_schema/enumeration.h" #include "tiledb/sm/buffer/buffer.h" #include "tiledb/sm/enums/array_type.h" #include "tiledb/sm/enums/compressor.h" @@ -58,6 +60,14 @@ using namespace tiledb::common; namespace tiledb { namespace sm { +/** Class for locally generated exceptions. */ +class ArraySchemaEvolutionException : public StatusException { + public: + explicit ArraySchemaEvolutionException(const std::string& msg) + : StatusException("ArraySchemaEvolution", msg) { + } +}; + /* ****************************** */ /* CONSTRUCTORS & DESTRUCTORS */ /* ****************************** */ @@ -65,6 +75,19 @@ namespace sm { ArraySchemaEvolution::ArraySchemaEvolution() { } +ArraySchemaEvolution::ArraySchemaEvolution( + std::unordered_map> attrs_to_add, + std::unordered_set attrs_to_drop, + std::unordered_map> enmrs_to_add, + std::unordered_set enmrs_to_drop, + std::pair timestamp_range) + : attributes_to_add_map_(attrs_to_add) + , attributes_to_drop_(attrs_to_drop) + , enumerations_to_add_map_(enmrs_to_add) + , enumerations_to_drop_(enmrs_to_drop) + , timestamp_range_(timestamp_range) { +} + ArraySchemaEvolution::~ArraySchemaEvolution() { clear(); } @@ -73,59 +96,66 @@ ArraySchemaEvolution::~ArraySchemaEvolution() { /* API */ /* ****************************** */ -tuple>> -ArraySchemaEvolution::evolve_schema( +shared_ptr ArraySchemaEvolution::evolve_schema( const shared_ptr& orig_schema) { std::lock_guard lock(mtx_); if (orig_schema == nullptr) { - return { - LOG_STATUS(Status_ArraySchemaEvolutionError( - "Cannot evolve schema; Input array schema is null")), - nullopt}; + throw ArraySchemaEvolutionException( + "Cannot evolve schema; Input array schema is null"); } auto schema = make_shared(HERE(), *(orig_schema.get())); + // Add enumerations. Must be done before attributes so that any attributes + // referencing enumerations won't fail to be added. + for (auto& enmr : enumerations_to_add_map_) { + schema->add_enumeration(enmr.second); + } + // Add attributes. for (auto& attr : attributes_to_add_map_) { - RETURN_NOT_OK_TUPLE(schema->add_attribute(attr.second, false), nullopt); + throw_if_not_ok(schema->add_attribute(attr.second, false)); } // Drop attributes. for (auto& attr_name : attributes_to_drop_) { bool has_attr = false; - RETURN_NOT_OK_TUPLE(schema->has_attribute(attr_name, &has_attr), nullopt); + throw_if_not_ok(schema->has_attribute(attr_name, &has_attr)); if (has_attr) { - RETURN_NOT_OK_TUPLE(schema->drop_attribute(attr_name), nullopt); + throw_if_not_ok(schema->drop_attribute(attr_name)); } } - // Do other things + // Drop enumerations + for (auto& enmr_name : enumerations_to_drop_) { + if (schema->has_enumeration(enmr_name)) { + schema->drop_enumeration(enmr_name); + } + } // Set timestamp, if specified if (std::get<0>(timestamp_range_) != 0) { - RETURN_NOT_OK_TUPLE( - schema.get()->set_timestamp_range(timestamp_range_), nullopt); - RETURN_NOT_OK_TUPLE(schema->generate_uri(timestamp_range_), nullopt); + throw_if_not_ok(schema.get()->set_timestamp_range(timestamp_range_)); + throw_if_not_ok(schema->generate_uri(timestamp_range_)); } else { // Generate new schema URI - RETURN_NOT_OK_TUPLE(schema->generate_uri(), nullopt); + throw_if_not_ok(schema->generate_uri()); } - return {Status::Ok(), schema}; + return schema; } -Status ArraySchemaEvolution::add_attribute(const Attribute* attr) { +void ArraySchemaEvolution::add_attribute(const Attribute* attr) { std::lock_guard lock(mtx_); // Sanity check if (attr == nullptr) - return LOG_STATUS(Status_ArraySchemaEvolutionError( - "Cannot add attribute; Input attribute is null")); + throw ArraySchemaEvolutionException( + "Cannot add attribute; Input attribute is null"); if (attributes_to_add_map_.find(attr->name()) != attributes_to_add_map_.end()) { - return LOG_STATUS(Status_ArraySchemaEvolutionError( - "Cannot add attribute; Input attribute name is already there")); + throw ArraySchemaEvolutionException( + "Cannot add attribute; Input attribute name is already there"); } // Create new attribute and potentially set a default name @@ -134,18 +164,14 @@ Status ArraySchemaEvolution::add_attribute(const Attribute* attr) { if (attributes_to_drop_.find(attr->name()) != attributes_to_drop_.end()) { attributes_to_drop_.erase(attr->name()); } - - return Status::Ok(); } std::vector ArraySchemaEvolution::attribute_names_to_add() const { std::lock_guard lock(mtx_); std::vector names; names.reserve(attributes_to_add_map_.size()); - for (auto it = attributes_to_add_map_.begin(); - it != attributes_to_add_map_.end(); - ++it) { - names.push_back(it->first); + for (auto& entry : attributes_to_add_map_) { + names.emplace_back(entry.first); } return names; } @@ -157,29 +183,91 @@ const Attribute* ArraySchemaEvolution::attribute_to_add( return (it == attributes_to_add_map_.end()) ? nullptr : it->second.get(); } -Status ArraySchemaEvolution::drop_attribute(const std::string& attribute_name) { +void ArraySchemaEvolution::drop_attribute(const std::string& attribute_name) { std::lock_guard lock(mtx_); attributes_to_drop_.insert(attribute_name); - if (attributes_to_add_map_.find(attribute_name) != - attributes_to_add_map_.end()) { + auto ait = attributes_to_add_map_.find(attribute_name); + if (ait != attributes_to_add_map_.end()) { // Reset the pointer and erase it - attributes_to_add_map_.erase(attribute_name); + attributes_to_add_map_.erase(ait); } - return Status::Ok(); } std::vector ArraySchemaEvolution::attribute_names_to_drop() const { std::lock_guard lock(mtx_); std::vector names; names.reserve(attributes_to_drop_.size()); - for (auto it = attributes_to_drop_.begin(); it != attributes_to_drop_.end(); - ++it) { - names.push_back(*it); + for (auto& name : attributes_to_drop_) { + names.emplace_back(name); + } + return names; +} + +void ArraySchemaEvolution::add_enumeration(shared_ptr enmr) { + std::lock_guard lock(mtx_); + + if (enmr == nullptr) { + throw ArraySchemaEvolutionException( + "Cannot add enumeration; Input enumeration is null"); + } + + auto it = enumerations_to_add_map_.find(enmr->name()); + if (it != enumerations_to_add_map_.end()) { + throw ArraySchemaEvolutionException( + "Cannot add enumeration; Input enumeration name is already added"); + } + + enumerations_to_add_map_[enmr->name()] = enmr; +} + +std::vector ArraySchemaEvolution::enumeration_names_to_add() + const { + std::lock_guard lock(mtx_); + std::vector names; + names.reserve(enumerations_to_add_map_.size()); + for (auto elem : enumerations_to_add_map_) { + names.push_back(elem.first); + } + + return names; +} + +shared_ptr ArraySchemaEvolution::enumeration_to_add( + const std::string& name) const { + std::lock_guard lock(mtx_); + auto it = enumerations_to_add_map_.find(name); + + if (it == enumerations_to_add_map_.end()) { + return nullptr; + } + + return it->second; +} + +void ArraySchemaEvolution::drop_enumeration( + const std::string& enumeration_name) { + std::lock_guard lock(mtx_); + enumerations_to_drop_.insert(enumeration_name); + + auto it = enumerations_to_add_map_.find(enumeration_name); + if (it != enumerations_to_add_map_.end()) { + // Reset the pointer and erase it + enumerations_to_add_map_.erase(it); + } +} + +std::vector ArraySchemaEvolution::enumeration_names_to_drop() + const { + std::lock_guard lock(mtx_); + std::vector names; + names.reserve(enumerations_to_drop_.size()); + for (auto& name : enumerations_to_drop_) { + names.emplace_back(name); } return names; } -Status ArraySchemaEvolution::set_timestamp_range( +void ArraySchemaEvolution::set_timestamp_range( const std::pair& timestamp_range) { if (timestamp_range.first != timestamp_range.second) { throw std::runtime_error(std::string( @@ -188,7 +276,6 @@ Status ArraySchemaEvolution::set_timestamp_range( std::to_string(timestamp_range.second) + " are not equal!")); } timestamp_range_ = timestamp_range; - return Status::Ok(); } std::pair ArraySchemaEvolution::timestamp_range() const { @@ -203,6 +290,8 @@ std::pair ArraySchemaEvolution::timestamp_range() const { void ArraySchemaEvolution::clear() { attributes_to_add_map_.clear(); attributes_to_drop_.clear(); + enumerations_to_add_map_.clear(); + enumerations_to_drop_.clear(); timestamp_range_ = {0, 0}; } diff --git a/tiledb/sm/array_schema/array_schema_evolution.h b/tiledb/sm/array_schema/array_schema_evolution.h index 6ba64b754519..21af603e4d59 100644 --- a/tiledb/sm/array_schema/array_schema_evolution.h +++ b/tiledb/sm/array_schema/array_schema_evolution.h @@ -38,7 +38,6 @@ #include #include "tiledb/common/common.h" -#include "tiledb/common/status.h" #include "tiledb/sm/filesystem/uri.h" #include "tiledb/sm/filter/filter_pipeline.h" #include "tiledb/sm/misc/constants.h" @@ -55,6 +54,7 @@ class Buffer; class ConstBuffer; class Dimension; class Domain; +class Enumeration; class ArraySchema; enum class ArrayType : uint8_t; @@ -72,6 +72,20 @@ class ArraySchemaEvolution { /** Constructor. */ ArraySchemaEvolution(); + /** Constructor. + * @param attrs_to_add Attributes to add to the schema. + * @param enmrs_to_add Enumerations to add to the schema. + * @param attrs_to_drop Attributes to remove from the schema. + * @param timestamp_range Timestamp range to use for the new schema. + */ + ArraySchemaEvolution( + std::unordered_map> attrs_to_add, + std::unordered_set attrs_to_drop, + std::unordered_map> + enmrs_to_add, + std::unordered_set enmrs_to_drop, + std::pair timestamp_range); + /** Destructor. */ ~ArraySchemaEvolution(); @@ -79,16 +93,15 @@ class ArraySchemaEvolution { /* API */ /* ********************************* */ - tuple>> evolve_schema( + shared_ptr evolve_schema( const shared_ptr& orig_schema); /** * Adds an attribute, copying the input. * * @param attr The attribute to be added - * @return Status */ - Status add_attribute(const Attribute* attr); + void add_attribute(const Attribute* attr); /** Returns the names of attributes to add. */ std::vector attribute_names_to_add() const; @@ -102,16 +115,45 @@ class ArraySchemaEvolution { /** * Drops an attribute. * - * @param attr The attribute to be dropped - * @return Status + * @param attribute_name The attribute to be dropped. */ - Status drop_attribute(const std::string& attribute_name); + void drop_attribute(const std::string& attribute_name); /** Returns the names of attributes to drop. */ std::vector attribute_names_to_drop() const; + /** + * Adds an enumeration + * + * @param enmr The enumeration to add + */ + void add_enumeration(shared_ptr enmr); + + /** Returns the names of the enumerations to add. */ + std::vector enumeration_names_to_add() const; + + /** + * Returns a constant pointer to the selected enumeration or nullptr if + * it does not exist. + * + * @param name The name of the enumeration to add + * @return shared_ptr The enumeration to add. + */ + shared_ptr enumeration_to_add( + const std::string& name) const; + + /** + * Drops an enumeration + * + * @param enumeration_name The enumeration to be dropped. + */ + void drop_enumeration(const std::string& enumeration_name); + + /** Return the names of enumerations to drop. */ + std::vector enumeration_names_to_drop() const; + /** Set a timestamp range for the array schema evolution */ - Status set_timestamp_range( + void set_timestamp_range( const std::pair& timestamp_range); /** Returns the timestamp range. */ @@ -129,6 +171,13 @@ class ArraySchemaEvolution { /** The names of array attributes to be dropped. */ std::unordered_set attributes_to_drop_; + /** Enumerations to add with any attribute. */ + std::unordered_map> + enumerations_to_add_map_; + + /** The names of array enumerations to be dropped. */ + std::unordered_set enumerations_to_drop_; + /** * A timestamp to explicitly set the timestamp of * the evolved schema. To be consistent with @@ -151,4 +200,4 @@ class ArraySchemaEvolution { } // namespace sm } // namespace tiledb -#endif // TILEDB_SCHEMA_EVOLUTION_H \ No newline at end of file +#endif // TILEDB_SCHEMA_EVOLUTION_H diff --git a/tiledb/sm/array_schema/attribute.cc b/tiledb/sm/array_schema/attribute.cc index 8c6132a5c925..bdf23e01c44c 100644 --- a/tiledb/sm/array_schema/attribute.cc +++ b/tiledb/sm/array_schema/attribute.cc @@ -38,6 +38,7 @@ #include "tiledb/sm/enums/filter_type.h" #include "tiledb/sm/filter/compression_filter.h" #include "tiledb/sm/misc/parse_argument.h" +#include "tiledb/sm/misc/uuid.h" #include "tiledb/type/range/range.h" #include @@ -115,7 +116,8 @@ Attribute::Attribute( const FilterPipeline& filter_pipeline, const ByteVecValue& fill_value, uint8_t fill_value_validity, - DataOrder order) + DataOrder order, + std::optional enumeration_name) : cell_val_num_(cell_val_num) , nullable_(nullable) , filters_(filter_pipeline) @@ -123,7 +125,8 @@ Attribute::Attribute( , type_(type) , fill_value_(fill_value) , fill_value_validity_(fill_value_validity) - , order_(order) { + , order_(order) + , enumeration_name_(enumeration_name) { } Attribute::Attribute(const Attribute* attr) { @@ -136,6 +139,7 @@ Attribute::Attribute(const Attribute* attr) { fill_value_ = attr->fill_value_; fill_value_validity_ = attr->fill_value_validity_; order_ = attr->order_; + enumeration_name_ = attr->enumeration_name_; } Attribute::~Attribute() = default; @@ -204,6 +208,17 @@ Attribute Attribute::deserialize( order = data_order_from_int(deserializer.read()); } + std::optional enmr_name; + if (version >= 19) { + auto enmr_name_length = deserializer.read(); + if (enmr_name_length > 0) { + std::string enmr_name_value; + enmr_name_value.resize(enmr_name_length); + deserializer.read(enmr_name_value.data(), enmr_name_length); + enmr_name = enmr_name_value; + } + } + return Attribute( name, datatype, @@ -212,7 +227,8 @@ Attribute Attribute::deserialize( filterpipeline, fill_value, fill_value_validity, - order); + order, + enmr_name); } void Attribute::dump(FILE* out) const { @@ -299,6 +315,18 @@ void Attribute::serialize( if (version >= 17) { serializer.write(static_cast(order_)); } + + // Write enumeration URI + if (version >= constants::enumerations_min_format_version) { + if (enumeration_name_.has_value()) { + auto enmr_name_size = + static_cast(enumeration_name_.value().size()); + serializer.write(enmr_name_size); + serializer.write(enumeration_name_.value().data(), enmr_name_size); + } else { + serializer.write(0); + } + } } void Attribute::set_cell_val_num(unsigned int cell_val_num) { @@ -480,6 +508,18 @@ DataOrder Attribute::order() const { return order_; } +void Attribute::set_enumeration_name(std::optional enmr_name) { + if (enmr_name.has_value() && enmr_name.value().empty()) { + throw AttributeStatusException( + "Invalid enumeration name, name must not be empty."); + } + enumeration_name_ = enmr_name; +} + +std::optional Attribute::get_enumeration_name() const { + return enumeration_name_; +} + /* ********************************* */ /* PRIVATE METHODS */ /* ********************************* */ diff --git a/tiledb/sm/array_schema/attribute.h b/tiledb/sm/array_schema/attribute.h index 7e3e7a4dd302..368836646bc4 100644 --- a/tiledb/sm/array_schema/attribute.h +++ b/tiledb/sm/array_schema/attribute.h @@ -51,6 +51,7 @@ namespace sm { class Buffer; class ConstBuffer; +class Enumeration; enum class Compressor : uint8_t; enum class Datatype : uint8_t; @@ -110,7 +111,8 @@ class Attribute { const FilterPipeline& filter_pipeline, const ByteVecValue& fill_value, uint8_t fill_value_validity, - DataOrder order = DataOrder::UNORDERED_DATA); + DataOrder order = DataOrder::UNORDERED_DATA, + std::optional enumeration_name = nullopt); /** * Constructor. It clones the input attribute. @@ -242,6 +244,12 @@ class Attribute { static ByteVecValue default_fill_value( Datatype datatype, uint32_t cell_val_num); + /** Set an enumeration for this attribute. */ + void set_enumeration_name(std::optional enmr_name); + + /** Get the enumeration for this attribute. */ + std::optional get_enumeration_name() const; + private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ @@ -271,8 +279,11 @@ class Attribute { /** The required order of the data stored in the attribute. */ DataOrder order_; + /** The name of the Enumeration to use for this attribute. */ + std::optional enumeration_name_; + /* ********************************* */ - /* PRIVATE ATTRIBUTES */ + /* PRIVATE METHODS */ /* ********************************* */ /** Sets the default fill value. */ diff --git a/tiledb/sm/array_schema/enumeration.cc b/tiledb/sm/array_schema/enumeration.cc new file mode 100644 index 000000000000..825f6e9ad2e4 --- /dev/null +++ b/tiledb/sm/array_schema/enumeration.cc @@ -0,0 +1,260 @@ +/** + * @file enumeration.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file implements class Enumeration. + */ + +#include +#include + +#include "tiledb/sm/misc/uuid.h" + +#include "enumeration.h" + +namespace tiledb::sm { + +/** Class for locally generated status exceptions. */ +class EnumerationException : public StatusException { + public: + explicit EnumerationException(const std::string& msg) + : StatusException("Enumeration", msg) { + } +}; + +Enumeration::Enumeration( + const std::string& name, + const std::string& path_name, + Datatype type, + uint32_t cell_val_num, + bool ordered, + const void* data, + uint64_t data_size, + const void* offsets, + uint64_t offsets_size) + : name_(name) + , path_name_(path_name) + , type_(type) + , cell_val_num_(cell_val_num) + , ordered_(ordered) + , data_(data_size) + , offsets_(offsets_size) { + ensure_datatype_is_valid(type); + + if (name.empty()) { + throw EnumerationException("Enumeration name must not be empty"); + } + + if (path_name_.empty()) { + std::string tmp_uuid; + throw_if_not_ok(uuid::generate_uuid(&tmp_uuid, false)); + path_name_ = + "__" + tmp_uuid + "_" + std::to_string(constants::enumerations_version); + } + + if (name.find("/") != std::string::npos) { + throw EnumerationException( + "Enumeration name must not contain path separators"); + } + + if (cell_val_num == 0) { + throw EnumerationException("Invalid cell_val_num in Enumeration"); + } + + if (data == nullptr || data_size == 0) { + throw EnumerationException("No attribute value data supplied."); + } + + if (var_size() && (offsets == nullptr || offsets_size == 0)) { + throw EnumerationException( + "Variable length datatype defined but offsets are not present"); + } else if (!var_size() && (offsets != nullptr || offsets_size > 0)) { + throw EnumerationException( + "Fixed length datatype defined but offsets are present"); + } + + if (var_size()) { + if (offsets_size % sizeof(uint64_t) != 0) { + throw EnumerationException( + "Invalid offsets size is not a multiple of sizeof(uint64_t)"); + } + auto offset_values = static_cast(offsets); + uint64_t last_offset = (offsets_size / sizeof(uint64_t)) - 1; + if (offset_values[last_offset] > data_size) { + throw EnumerationException( + "Provided data buffer size is too small for the provided offsets."); + } + } else { + if (data_size % cell_size() != 0) { + throw EnumerationException( + "Invalid data size is not a multiple of the cell size."); + } + } + + throw_if_not_ok(data_.write(data, 0, data_size)); + + if (offsets_size > 0) { + throw_if_not_ok(offsets_.write(offsets, 0, offsets_size)); + } + + generate_value_map(); +} + +shared_ptr Enumeration::deserialize( + Deserializer& deserializer) { + auto disk_version = deserializer.read(); + if (disk_version > constants::enumerations_version) { + throw EnumerationException( + "Invalid Enumeration version '" + std::to_string(disk_version) + + "' is newer than supported enumeration version '" + + std::to_string(constants::enumerations_version) + "'"); + } + + auto name_size = deserializer.read(); + std::string name(deserializer.get_ptr(name_size), name_size); + + auto path_name_size = deserializer.read(); + std::string path_name( + deserializer.get_ptr(path_name_size), path_name_size); + + auto type = deserializer.read(); + auto cell_val_num = deserializer.read(); + auto ordered = deserializer.read(); + + auto data_size = deserializer.read(); + const void* data = deserializer.get_ptr(data_size); + + uint64_t offsets_size = 0; + const void* offsets = nullptr; + + if (cell_val_num == constants::var_num) { + offsets_size = deserializer.read(); + offsets = deserializer.get_ptr(offsets_size); + } + + return create( + name, + path_name, + static_cast(type), + cell_val_num, + ordered, + data, + data_size, + offsets, + offsets_size); +} + +void Enumeration::serialize(Serializer& serializer) const { + serializer.write(constants::enumerations_version); + + auto name_size = static_cast(name_.size()); + serializer.write(name_size); + serializer.write(name_.data(), name_size); + + auto path_name_size = static_cast(path_name_.size()); + serializer.write(path_name_size); + serializer.write(path_name_.data(), path_name_size); + + serializer.write(static_cast(type_)); + serializer.write(cell_val_num_); + serializer.write(ordered_); + serializer.write(data_.size()); + serializer.write(data_.data(), data_.size()); + + if (var_size()) { + serializer.write(offsets_.size()); + serializer.write(offsets_.data(), offsets_.size()); + } else { + assert(cell_val_num_ < constants::var_num); + assert(offsets_.size() == 0); + } +} + +uint64_t Enumeration::index_of(UntypedDatumView value) const { + std::string_view value_view( + static_cast(value.content()), value.size()); + + auto iter = value_map_.find(value_view); + if (iter == value_map_.end()) { + return constants::enumeration_missing_value; + } + + return iter->second; +} + +void Enumeration::dump(FILE* out) const { + if (out == nullptr) { + out = stdout; + } + std::stringstream ss; + ss << "### Enumeration ###" << std::endl; + ss << "- Name: " << name_ << std::endl; + ss << "- Type: " << datatype_str(type_) << std::endl; + ss << "- Cell Val Num: " << cell_val_num_ << std::endl; + ss << "- Ordered: " << (ordered_ ? "true" : "false") << std::endl; + ss << "- Element Count: " << value_map_.size() << std::endl; + fprintf(out, "%s", ss.str().c_str()); +} + +void Enumeration::generate_value_map() { + auto char_data = data_.data_as(); + if (var_size()) { + auto offsets = offsets_.data_as(); + uint64_t num_offsets = offsets_.size() / sizeof(uint64_t); + + for (uint64_t i = 0; i < num_offsets; i++) { + uint64_t length = 0; + if (i < num_offsets - 1) { + length = offsets[i + 1] - offsets[i]; + } else { + length = data_.size() - offsets[i]; + } + + auto sv = std::string_view(char_data + offsets[i], length); + add_value_to_map(sv, i); + } + } else { + uint64_t i = 0; + auto stride = cell_size(); + while (i * stride < data_.size()) { + auto sv = std::string_view(char_data + i * stride, stride); + add_value_to_map(sv, i); + i += 1; + } + } +} + +void Enumeration::add_value_to_map(std::string_view& sv, uint64_t index) { + if (value_map_.find(sv) != value_map_.end()) { + throw EnumerationException( + "Invalid duplicated value in enumeration '" + std::string(sv) + "'"); + } + value_map_[sv] = index; +} + +} // namespace tiledb::sm diff --git a/tiledb/sm/array_schema/enumeration.h b/tiledb/sm/array_schema/enumeration.h new file mode 100644 index 000000000000..aac6ea2dd910 --- /dev/null +++ b/tiledb/sm/array_schema/enumeration.h @@ -0,0 +1,388 @@ +/** + * @file enumeration.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file defines class Enumeration. + */ + +#ifndef TILEDB_ENUMERATION_H +#define TILEDB_ENUMERATION_H + +#include + +#include "tiledb/common/common.h" +#include "tiledb/common/types/untyped_datum.h" +#include "tiledb/sm/buffer/buffer.h" +#include "tiledb/sm/enums/datatype.h" +#include "tiledb/storage_format/serialization/serializers.h" + +namespace tiledb::sm { + +/** Defines an array enumeration */ +class Enumeration { + public: + /* ********************************* */ + /* CONSTRUCTORS & DESTRUCTORS */ + /* ********************************* */ + + /** No default constructor. Use the create factory method. */ + Enumeration() = delete; + + DISABLE_COPY(Enumeration); + DISABLE_MOVE(Enumeration); + + /** Destructor. */ + ~Enumeration() = default; + + /* ********************************* */ + /* OPERATORS */ + /* ********************************* */ + + DISABLE_COPY_ASSIGN(Enumeration); + DISABLE_MOVE_ASSIGN(Enumeration); + + /* ********************************* */ + /* API */ + /* ********************************* */ + + /** Create a new Enumeration + * + * @param name The name of this Enumeration as referenced by attributes. + * @param type The datatype of the enumeration values. + * @param cell_val_num The cell_val_num of the enumeration. + * @param ordered Whether the enumeration should be considered as ordered. + * If false, prevents inequality operators in QueryConditons from + * being used with this enumeration. + * @param data A pointer to the enumerations values. + * @param data_size The length of the buffer pointed to by data. + * @param offsets If cell_var_num is constants::var_num a pointer to the + * offsets buffer. Must be null if cell_var_num is not var_num. + * @param offsets_size The size of the buffer pointed to by offsets. Must be + * zero of cell_var_num is not var_num. + * @return shared_ptr The created enumeration. + */ + static shared_ptr create( + const std::string& name, + Datatype type, + uint32_t cell_val_num, + bool ordered, + const void* data, + uint64_t data_size, + const void* offsets, + uint64_t offsets_size) { + return create( + name, + "", + type, + cell_val_num, + ordered, + data, + data_size, + offsets, + offsets_size); + } + + /** Create a new Enumeration + * + * @param name The name of this Enumeration as referenced by attributes. + * @param path_name The last URI path component of the Enumeration. + * @param type The datatype of the enumeration values. + * @param cell_val_num The cell_val_num of the enumeration. + * @param ordered Whether the enumeration should be considered as ordered. + * If false, prevents inequality operators in QueryConditons from + * being used with this enumeration. + * @param data A pointer to the enumerations values. + * @param data_size The length of the buffer pointed to by data. + * @param offsets If cell_var_num is constants::var_num a pointer to the + * offsets buffer. Must be null if cell_var_num is not var_num. + * @param offsets_size The size of the buffer pointed to by offsets. Must be + * zero of cell_var_num is not var_num. + * @return shared_ptr The created enumeration. + */ + static shared_ptr create( + const std::string& name, + const std::string& path_name, + Datatype type, + uint32_t cell_val_num, + bool ordered, + const void* data, + uint64_t data_size, + const void* offsets, + uint64_t offsets_size) { + struct EnableMakeShared : public Enumeration { + EnableMakeShared( + const std::string& name, + const std::string& path_name, + Datatype type, + uint32_t cell_val_num, + bool ordered, + const void* data, + uint64_t data_size, + const void* offsets, + uint64_t offsets_size) + : Enumeration( + name, + path_name, + type, + cell_val_num, + ordered, + data, + data_size, + offsets, + offsets_size) { + } + }; + return make_shared( + HERE(), + name, + path_name, + type, + cell_val_num, + ordered, + data, + data_size, + offsets, + offsets_size); + } + + /** + * Deserialize an enumeration + * + * @param deserializer The deserializer to deserialize from. + * @return A new Enumeration. + */ + static shared_ptr deserialize(Deserializer& deserializer); + + /** + * Serializes the enumeration into a buffer. + * + * @param serializer The object the array schema is serialized into. + */ + void serialize(Serializer& serializer) const; + + /** + * The name of this Enumeration referenced by attributes. + * + * @return The name of this Enumeration. + */ + const std::string& name() const { + return name_; + } + + /** + * The path name for this Enumeration on disk. + * + * @return The path name of this Enumeration. + */ + const std::string& path_name() const { + return path_name_; + } + + /** + * The type of the enumeration values + * + * @return Datatype The datatype of the enumeration values. + */ + Datatype type() const { + return type_; + } + + /** + * The cell_val_num of the enumeration + * + * @return uint32_t The cell_val_num of the enumeration. + */ + uint32_t cell_val_num() const { + return cell_val_num_; + } + + /** + * Get the cell_size of the enumeration + * + * This returns constants::var_size when cell_val_num is var_num, otherwise + * it returns the cell_val_num * datatype_size(type()) + * + * @return uint64_t The cell size of this enumeration. + */ + uint64_t cell_size() const { + if (var_size()) { + return constants::var_size; + } + + return cell_val_num_ * datatype_size(type_); + } + + /** + * Get the number of elements in the Enumeration. + * + * @return uint64_t The number of elements. + */ + uint64_t elem_count() const { + if (var_size()) { + return offsets_.size() / sizeof(uint64_t); + } else { + return data_.size() / cell_size(); + } + } + + /** + * Returns whether this enumeration is variable sized. + * + * @return bool Whether this enumerations is variable sized. + */ + bool var_size() const { + return cell_val_num_ == constants::var_num; + } + + /** + * Returns whether this enumeration is considered ordered or not. + * + * If an enumeration is not ordered then inequality operators in + * QueryConditions against this Enumeration are disabled. + * + * @return bool Whether this enumeration is considered ordered. + */ + bool ordered() const { + return ordered_; + } + + /** + * Returns the data buffer + * + * @return tuple A pointer to the data buffer and the + size of the buffer pointed to. + */ + span data() const { + return {static_cast(data_.data()), data_.size()}; + } + + /** + * Returns the offsets buffer + * + * @return tuple A pointer to the offsets buffer and + * the size of the buffer pointed to. + */ + span offsets() const { + return {static_cast(offsets_.data()), offsets_.size()}; + } + + /** + * Return the index of a value in the enumeration + * + * @param data An UntypedDatumView of the value to look up. + * @return uint64_t The index of the value represented by data or + * constants::missing_enumeration_value if not found. + */ + uint64_t index_of(UntypedDatumView data) const; + + /** + * Dump a textual representation of the Enumeration to the FILE + * + * @param out A file pointer to write to. If out is nullptr, use stdout + */ + void dump(FILE* out) const; + + private: + /* ********************************* */ + /* PRIVATE CONSTRUCTORS */ + /* ********************************* */ + + /** Constructor + * + * @param name The name of this Enumeration as referenced by attributes. + * @param path_name The last URI path component of the Enumeration. + * @param type The datatype of the enumeration values. + * @param cell_val_num The cell_val_num of the enumeration. + * @param ordered Whether the enumeration should be considered as ordered. + * If false, prevents inequality operators in QueryConditons from + * being used with this enumeration. + * @param data A pointer to the enumerations values. + * @param data_size The length of the buffer pointed to by data. + * @param offsets If cell_var_num is constants::var_num a pointer to the + * offsets buffer. Must be null if cell_var_num is not var_num. + * @param offsets_size The size of the buffer pointed to by offsets. Must be + * zero of cell_var_num is not var_num. + */ + Enumeration( + const std::string& name, + const std::string& path_name, + Datatype type, + uint32_t cell_val_num, + bool ordered, + const void* data, + uint64_t data_size, + const void* offsets, + uint64_t offsets_size); + + /* ********************************* */ + /* PRIVATE ATTRIBUTES */ + /* ********************************* */ + + /** The name of this Enumeration stored in the enumerations directory. */ + std::string name_; + + /** The path name of this Enumeration as stored on disk. */ + std::string path_name_; + + /** The type of enumerated values */ + Datatype type_; + + /** Number of values per enumeration value */ + uint32_t cell_val_num_; + + /** A flag which enables or disables inequality comparisons */ + bool ordered_; + + /** The list of enumeration values */ + Buffer data_; + + /** The offsets of each enumeration value if var sized. */ + Buffer offsets_; + + /** Map of values to indices */ + std::unordered_map value_map_; + + /* ********************************* */ + /* PRIVATE METHODS */ + /* ********************************* */ + + /** Populate the value_map_ */ + void generate_value_map(); + + /** + * Add a value to value_map_ + * + * @param sv A string view of the data to add. + * @param index The index of the data in the Enumeration. + */ + void add_value_to_map(std::string_view& sv, uint64_t index); +}; + +} // namespace tiledb::sm + +#endif // TILEDB_DOMAIN_H diff --git a/tiledb/sm/array_schema/test/compile_enumeration_main.cc b/tiledb/sm/array_schema/test/compile_enumeration_main.cc new file mode 100644 index 000000000000..d35fa1cca83c --- /dev/null +++ b/tiledb/sm/array_schema/test/compile_enumeration_main.cc @@ -0,0 +1,39 @@ +/** + * @file compile_enumeration_main.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "tiledb/sm/array_schema/enumeration.h" +#include "tiledb/sm/enums/datatype.h" + +int main(int, char*[]) { + try { + tiledb::sm::Enumeration::create( + "foo", tiledb::sm::Datatype::INT32, 1, false, nullptr, 0, nullptr, 0); + } catch (...) { + } + return 0; +} diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 90b6a9d17725..4642c58408ba 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -40,6 +40,7 @@ #include "tiledb/api/c_api/buffer_list/buffer_list_api_internal.h" #include "tiledb/api/c_api/config/config_api_internal.h" #include "tiledb/api/c_api/dimension/dimension_api_internal.h" +#include "tiledb/api/c_api/enumeration/enumeration_api_internal.h" #include "tiledb/api/c_api/error/error_api_internal.h" #include "tiledb/api/c_api/filter_list/filter_list_api_internal.h" #include "tiledb/api/c_api/string/string_api_internal.h" @@ -299,6 +300,34 @@ int32_t tiledb_attribute_set_cell_val_num( return TILEDB_OK; } +capi_return_t tiledb_attribute_set_enumeration_name( + tiledb_ctx_t* ctx, tiledb_attribute_t* attr, const char* enumeration_name) { + if (sanity_check(ctx, attr) == TILEDB_ERR) { + return TILEDB_ERR; + } + attr->attr_->set_enumeration_name(enumeration_name); + return TILEDB_OK; +} + +capi_return_t tiledb_attribute_get_enumeration_name( + tiledb_ctx_t* ctx, tiledb_attribute_t* attr, tiledb_string_t** name) { + if (sanity_check(ctx, attr) == TILEDB_ERR) { + return TILEDB_ERR; + } + + ensure_output_pointer_is_valid(name); + + auto enmr_name = attr->attr_->get_enumeration_name(); + if (!enmr_name.has_value()) { + *name = nullptr; + return TILEDB_OK; + } + + *name = tiledb_string_handle_t::make_handle(enmr_name.value()); + + return TILEDB_OK; +} + int32_t tiledb_attribute_get_name( tiledb_ctx_t* ctx, const tiledb_attribute_t* attr, const char** name) { if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, attr) == TILEDB_ERR) @@ -725,6 +754,21 @@ int32_t tiledb_array_schema_timestamp_range( return TILEDB_OK; } +TILEDB_EXPORT int32_t tiledb_array_schema_add_enumeration( + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_enumeration_t* enumeration) { + if (sanity_check(ctx, array_schema) == TILEDB_ERR) { + return TILEDB_ERR; + } + + api::ensure_enumeration_is_valid(enumeration); + + array_schema->array_schema_->add_enumeration(enumeration->copy()); + + return TILEDB_OK; +} + int32_t tiledb_array_schema_set_coords_filter_list( tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, @@ -786,9 +830,10 @@ int32_t tiledb_array_schema_check( return TILEDB_OK; } -int32_t tiledb_array_schema_load( +int32_t tiledb_array_schema_load_impl( tiledb_ctx_t* ctx, const char* array_uri, + int with_enumerations, tiledb_array_schema_t** array_schema) { if (sanity_check(ctx) == TILEDB_ERR) return TILEDB_ERR; @@ -822,12 +867,23 @@ int32_t tiledb_array_schema_load( return TILEDB_ERR; } + // With enumerations not supported for remote arrays currently. + if (with_enumerations) { + delete *array_schema; + *array_schema = nullptr; + auto st = Status_Error( + "Failed to load array schema;" + "remote load with enumerations is not yet supported."); + LOG_STATUS_NO_RETURN_VALUE(st); + save_error(ctx, st); + return TILEDB_ERR; + } + auto&& [st, array_schema_rest] = rest_client->get_array_schema_from_rest(uri); if (!st.ok()) { LOG_STATUS_NO_RETURN_VALUE(st); save_error(ctx, st); - delete *array_schema; return TILEDB_ERR; } (*array_schema)->array_schema_ = array_schema_rest.value(); @@ -853,13 +909,21 @@ int32_t tiledb_array_schema_load( auto st = Status_ArrayDirectoryError(le.what()); LOG_STATUS_NO_RETURN_VALUE(st); save_error(ctx, st); - delete *array_schema; return TILEDB_ERR; } // Load latest array schema - auto&& array_schema_latest = array_dir.load_array_schema_latest(key); - (*array_schema)->array_schema_ = array_schema_latest; + try { + auto&& array_schema_latest = array_dir.load_array_schema_latest(key); + if (with_enumerations) { + array_dir.load_all_enumerations(array_schema_latest, key); + } + (*array_schema)->array_schema_ = array_schema_latest; + } catch (...) { + delete *array_schema; + *array_schema = nullptr; + throw; + } } return TILEDB_OK; } @@ -1270,12 +1334,8 @@ int32_t tiledb_array_schema_evolution_add_attribute( sanity_check(ctx, attr) == TILEDB_ERR) return TILEDB_ERR; - throw_if_not_ok( - array_schema_evolution->array_schema_evolution_->add_attribute( - attr->attr_)); - return TILEDB_OK; + array_schema_evolution->array_schema_evolution_->add_attribute(attr->attr_); - // Success return TILEDB_OK; } @@ -1287,9 +1347,41 @@ int32_t tiledb_array_schema_evolution_drop_attribute( sanity_check(ctx, array_schema_evolution) == TILEDB_ERR) return TILEDB_ERR; - throw_if_not_ok( - array_schema_evolution->array_schema_evolution_->drop_attribute( - attribute_name)); + array_schema_evolution->array_schema_evolution_->drop_attribute( + attribute_name); + return TILEDB_OK; +} + +capi_return_t tiledb_array_schema_evolution_add_enumeration( + tiledb_ctx_t* ctx, + tiledb_array_schema_evolution_t* array_schema_evolution, + tiledb_enumeration_t* enumeration) { + if (sanity_check(ctx, array_schema_evolution) == TILEDB_ERR) { + return TILEDB_ERR; + } + + api::ensure_enumeration_is_valid(enumeration); + + auto enmr = enumeration->copy(); + array_schema_evolution->array_schema_evolution_->add_enumeration(enmr); + + return TILEDB_OK; +} + +capi_return_t tiledb_array_schema_evolution_drop_enumeration( + tiledb_ctx_t* ctx, + tiledb_array_schema_evolution_t* array_schema_evolution, + const char* enmr_name) { + if (sanity_check(ctx, array_schema_evolution) == TILEDB_ERR) { + return TILEDB_ERR; + } + + if (enmr_name == nullptr) { + return TILEDB_ERR; + } + + array_schema_evolution->array_schema_evolution_->drop_enumeration(enmr_name); + return TILEDB_OK; } @@ -1302,12 +1394,8 @@ int32_t tiledb_array_schema_evolution_set_timestamp_range( sanity_check(ctx, array_schema_evolution) == TILEDB_ERR) return TILEDB_ERR; - throw_if_not_ok( - array_schema_evolution->array_schema_evolution_->set_timestamp_range( - {lo, hi})); - return TILEDB_OK; - - // Success + array_schema_evolution->array_schema_evolution_->set_timestamp_range( + {lo, hi}); return TILEDB_OK; } @@ -2506,6 +2594,19 @@ int32_t tiledb_query_condition_negate( ctx, cond, nullptr, TILEDB_NOT, negated_cond); } +capi_return_t tiledb_query_condition_set_use_enumeration( + tiledb_ctx_t* ctx, + const tiledb_query_condition_t* cond, + int use_enumeration) { + if (sanity_check(ctx, cond) == TILEDB_ERR) { + return TILEDB_ERR; + } + + cond->query_condition_->set_use_enumeration( + use_enumeration != 0 ? true : false); + return TILEDB_OK; +} + /* ****************************** */ /* ARRAY */ /* ****************************** */ @@ -3411,6 +3512,37 @@ int32_t tiledb_array_evolve( return TILEDB_OK; } +capi_return_t tiledb_array_get_enumeration( + tiledb_ctx_t* ctx, + const tiledb_array_t* array, + const char* attr_name, + tiledb_enumeration_t** enumeration) { + if (sanity_check(ctx, array) == TILEDB_ERR) { + return TILEDB_ERR; + } + + if (attr_name == nullptr) { + throw api::CAPIStatusException("'attr_name' must not be null"); + } + + api::ensure_output_pointer_is_valid(enumeration); + auto ptr = array->array_->get_enumeration(attr_name); + *enumeration = tiledb_enumeration_handle_t::make_handle(ptr); + + return TILEDB_OK; +} + +capi_return_t tiledb_array_load_all_enumerations( + tiledb_ctx_t* ctx, const tiledb_array_t* array, int latest_only) { + if (sanity_check(ctx, array) == TILEDB_ERR) { + return TILEDB_ERR; + } + + array->array_->load_all_enumerations(latest_only ? true : false); + + return TILEDB_OK; +} + int32_t tiledb_array_upgrade_version( tiledb_ctx_t* ctx, const char* array_uri, tiledb_config_t* config) { // Sanity Checks @@ -5540,6 +5672,22 @@ int32_t tiledb_attribute_set_cell_val_num( ctx, attr, cell_val_num); } +capi_return_t tiledb_attribute_set_enumeration_name( + tiledb_ctx_t* ctx, + tiledb_attribute_t* attr, + const char* enumeration_name) noexcept { + return api_entry( + ctx, attr, enumeration_name); +} + +capi_return_t tiledb_attribute_get_enumeration_name( + tiledb_ctx_t* ctx, + tiledb_attribute_t* attr, + tiledb_string_t** name) noexcept { + return api_entry( + ctx, attr, name); +} + int32_t tiledb_attribute_get_name( tiledb_ctx_t* ctx, const tiledb_attribute_t* attr, @@ -5780,6 +5928,14 @@ int32_t tiledb_array_schema_timestamp_range( ctx, array_schema, lo, hi); } +int32_t tiledb_array_schema_add_enumeration( + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_enumeration_t* enumeration) noexcept { + return api_entry( + ctx, array_schema, enumeration); +} + int32_t tiledb_array_schema_set_coords_filter_list( tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, @@ -5813,8 +5969,16 @@ int32_t tiledb_array_schema_load( tiledb_ctx_t* ctx, const char* array_uri, tiledb_array_schema_t** array_schema) noexcept { - return api_entry( - ctx, array_uri, array_schema); + return api_entry( + ctx, array_uri, 0, array_schema); +} + +int32_t tiledb_array_schema_load_with_enumerations( + tiledb_ctx_t* ctx, + const char* array_uri, + tiledb_array_schema_t** array_schema) noexcept { + return api_entry( + ctx, array_uri, 1, array_schema); } int32_t tiledb_array_schema_load_with_key( @@ -5973,6 +6137,22 @@ int32_t tiledb_array_schema_evolution_drop_attribute( ctx, array_schema_evolution, attribute_name); } +int32_t tiledb_array_schema_evolution_add_enumeration( + tiledb_ctx_t* ctx, + tiledb_array_schema_evolution_t* array_schema_evolution, + tiledb_enumeration_t* enmr) noexcept { + return api_entry( + ctx, array_schema_evolution, enmr); +} + +capi_return_t tiledb_array_schema_evolution_drop_enumeration( + tiledb_ctx_t* ctx, + tiledb_array_schema_evolution_t* array_schema_evolution, + const char* enumeration_name) noexcept { + return api_entry( + ctx, array_schema_evolution, enumeration_name); +} + TILEDB_EXPORT int32_t tiledb_array_schema_evolution_set_timestamp_range( tiledb_ctx_t* ctx, tiledb_array_schema_evolution_t* array_schema_evolution, @@ -6608,6 +6788,14 @@ int32_t tiledb_query_condition_negate( ctx, cond, negated_cond); } +capi_return_t tiledb_query_condition_set_use_enumeration( + tiledb_ctx_t* const ctx, + const tiledb_query_condition_t* const cond, + int use_enumeration) noexcept { + return api_entry( + ctx, cond, use_enumeration); +} + /* ****************************** */ /* UPDATE CONDITION */ /* ****************************** */ @@ -6956,6 +7144,21 @@ int32_t tiledb_array_evolve( ctx, array_uri, array_schema_evolution); } +capi_return_t tiledb_array_get_enumeration( + tiledb_ctx_t* ctx, + const tiledb_array_t* array, + const char* attr_name, + tiledb_enumeration_t** enumeration) noexcept { + return api_entry( + ctx, array, attr_name, enumeration); +} + +capi_return_t tiledb_array_load_all_enumerations( + tiledb_ctx_t* ctx, const tiledb_array_t* array, int latest_only) noexcept { + return api_entry( + ctx, array, latest_only); +} + int32_t tiledb_array_upgrade_version( tiledb_ctx_t* ctx, const char* array_uri, diff --git a/tiledb/sm/c_api/tiledb_experimental.h b/tiledb/sm/c_api/tiledb_experimental.h index 8316351700e4..4da66d21a4a5 100644 --- a/tiledb/sm/c_api/tiledb_experimental.h +++ b/tiledb/sm/c_api/tiledb_experimental.h @@ -41,6 +41,7 @@ /* * API sections */ +#include "tiledb/api/c_api/enumeration/enumeration_api_experimental.h" #include "tiledb/api/c_api/group/group_api_external_experimental.h" #include "tiledb/api/c_api/query_plan/query_plan_api_external_experimental.h" #include "tiledb_dimension_label_experimental.h" @@ -155,6 +156,60 @@ TILEDB_EXPORT int32_t tiledb_array_schema_evolution_drop_attribute( tiledb_array_schema_evolution_t* array_schema_evolution, const char* attribute_name) TILEDB_NOEXCEPT; +/** + * Adds an enumeration to an array schema evolution. + * + * **Example:** + * + * @code{.c} + * tiledb_enumeration_t* enmr; + * void* data = get_data(); + * uint64_t data_size = get_data_size(); + * tiledb_enumeration_alloc( + * ctx, + * TILEDB_INT64, + * cell_val_num, + * FALSE, + * data, + * data_size, + * nullptr, + * 0, + * &enumeration); + * tiledb_array_schema_evolution_add_enumeration(ctx, array_schema_evolution, + * enmr); + * @endcode + * + * @param ctx The TileDB context. + * @param array_schema_evolution The schema evolution. + * @param enumeration The enumeration to be added. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_schema_evolution_add_enumeration( + tiledb_ctx_t* ctx, + tiledb_array_schema_evolution_t* array_schema_evolution, + tiledb_enumeration_t* enumeration) TILEDB_NOEXCEPT; + +/** + * Drops an enumeration from an array schema evolution. + * + * **Example:** + * + * @code{.c} + * const char* enumeration_name = "enumeration_1"; + * tiledb_array_schema_evolution_drop_enumeration(ctx, array_schema_evolution, + * enumeration_name); + * @endcode + * + * @param ctx The TileDB context. + * @param array_schema_evolution The schema evolution. + * @param enumeration_name The name of the enumeration to be dropped. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_schema_evolution_drop_enumeration( + tiledb_ctx_t* ctx, + tiledb_array_schema_evolution_t* array_schema_evolution, + const char* enumeration_name) TILEDB_NOEXCEPT; + /** * Sets timestamp range in an array schema evolution * This function sets the output timestamp of the committed array schema after @@ -210,6 +265,103 @@ TILEDB_EXPORT int32_t tiledb_array_schema_timestamp_range( uint64_t* lo, uint64_t* hi) TILEDB_NOEXCEPT; +/** + * Adds an enumeration to an array schema. + * + * **Example:** + * + * @code{.c} + * tiledb_enumeration_t* enumeration; + * tiledb_enumeration_alloc( + * ctx, + * "enumeration_name", + * TILEDB_INT64, + * 1, + * FALSE, + * data, + * data_size, + * nullptr, + * 0, + * &enumeration); + * tiledb_array_schema_add_enumeration(ctx, array_schema, enumeration); + * @endcode + * + * @param ctx The TileDB context. + * @param array_schema The array schema. + * @param enumeration The enumeration to add with the attribute + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT int32_t tiledb_array_schema_add_enumeration( + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_enumeration_t* enumeration) TILEDB_NOEXCEPT; + +/** + * Retrieves the schema of an array from the disk with all enumerations loaded, + * creating an array schema struct. + * + * **Example:** + * + * @code{.c} + * tiledb_array_schema_t* array_schema; + * tiledb_array_schema_load(ctx, "s3://tiledb_bucket/my_array", &array_schema); + * // Make sure to free the array schema in the end + * @endcode + * + * @param ctx The TileDB context. + * @param array_uri The array whose schema will be retrieved. + * @param array_schema The array schema to be retrieved, or `NULL` upon error. + * @return `TILEDB_OK` for success and `TILEDB_OOM` or `TILEDB_ERR` for error. + */ +TILEDB_EXPORT int32_t tiledb_array_schema_load_with_enumerations( + tiledb_ctx_t* ctx, + const char* array_uri, + tiledb_array_schema_t** array_schema) TILEDB_NOEXCEPT; + +/* ********************************* */ +/* ATTRIBUTE ENUMERATIONS */ +/* ********************************* */ + +/** + * Set the enumeration name on an attribute. + * + * **Example:** + * + * @code{.c} + * tiledb_attribute_set_enumeration_name(ctx, attr, "enumeration_name"); + * @endcode + * + * @param ctx The TileDB context. + * @param attr The target attribute. + * @param enumeration_name The name of the enumeration to use for the attribute. + * @return `TILEDB_OK` for success, and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_attribute_set_enumeration_name( + tiledb_ctx_t* ctx, + tiledb_attribute_t* attr, + const char* enumeration_name) TILEDB_NOEXCEPT; + +/** + * Get the attribute's enumeration name if it has one. + * + * **Example:** + * + * @code{.c} + * tiledb_string_t* name; + * tiledb_attribute_get_enumeration_name(ctx, attr, &name); + * @endcode + * + * @param ctx The TileDB context. + * @param attr The target attribute. + * @param name The name of the attribute, nullptr if the attribute does not + * have an associated enumeration. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_attribute_get_enumeration_name( + tiledb_ctx_t* ctx, + tiledb_attribute_t* attr, + tiledb_string_t** name) TILEDB_NOEXCEPT; + /* ********************************* */ /* ARRAY */ /* ********************************* */ @@ -269,6 +421,52 @@ TILEDB_EXPORT int32_t tiledb_array_evolve( const char* array_uri, tiledb_array_schema_evolution_t* array_schema_evolution) TILEDB_NOEXCEPT; +/** + * Retrieves an attribute's enumeration given the attribute name (key). + * + * **Example:** + * + * The following retrieves the first attribute in the schema. + * + * @code{.c} + * tiledb_attribute_t* attr; + * tiledb_array_schema_get_enumeration( + * ctx, array_schema, "attr_0", &enumeration); + * // Make sure to delete the retrieved attribute in the end. + * @endcode + * + * @param ctx The TileDB context. + * @param array The TileDB array. + * @param name The name (key) of the attribute from which to + * retrieve the enumeration. + * @param enumeration The enumeration object to retrieve. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_get_enumeration( + tiledb_ctx_t* ctx, + const tiledb_array_t* array, + const char* name, + tiledb_enumeration_t** enumeration) TILEDB_NOEXCEPT; + +/** + * Load all enumerations for the array. + * + * **Example:** + * + * @code{.c} + * tiledb_array_load_all_enumerations(ctx, array); + * @endcode + * + * @param ctx The TileDB context. + * @param array The TileDB array. + * @param latest_only If non-zero, only load enumerations for the latest schema. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_load_all_enumerations( + tiledb_ctx_t* ctx, + const tiledb_array_t* array, + int latest_only) TILEDB_NOEXCEPT; + /** * Upgrades an array to the latest format version. * @@ -378,6 +576,39 @@ TILEDB_EXPORT int32_t tiledb_query_get_relevant_fragment_num( const tiledb_query_t* query, uint64_t* relevant_fragment_num) TILEDB_NOEXCEPT; +/* ********************************* */ +/* QUERY CONDITION */ +/* ********************************* */ + +/** + * Disable the use of enumerations on the given QueryCondition + * + * **Example:** + * + * @code{.c} + * tiledb_query_condition_t* query_condition; + * tiledb_query_condition_alloc(ctx, &query_condition); + * uint32_t value = 5; + * tiledb_query_condition_init( + * ctx, + * query_condition, + * "longitude", + * &value, + * sizeof(value), + * TILEDB_LT); + * tiledb_query_condition_set_use_enumeration(ctx, query_condition, 0); + * @endcode + * + * @param[in] ctx The TileDB context. + * @param[in] cond The query condition + * @param[in] use_enumeration Non-zero to use the associated enumeration + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT int32_t tiledb_query_condition_set_use_enumeration( + tiledb_ctx_t* ctx, + const tiledb_query_condition_t* cond, + int use_enumeration) TILEDB_NOEXCEPT; + /* ********************************* */ /* QUERY STATUS DETAILS */ /* ********************************* */ diff --git a/tiledb/sm/cpp_api/array_experimental.h b/tiledb/sm/cpp_api/array_experimental.h new file mode 100644 index 000000000000..34df1696d898 --- /dev/null +++ b/tiledb/sm/cpp_api/array_experimental.h @@ -0,0 +1,77 @@ +/** + * @file array_experimental.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares the experimental C++ API for arrays. + */ + +#ifndef TILEDB_CPP_API_ARRAY_EXPERIMENTAL_H +#define TILEDB_CPP_API_ARRAY_EXPERIMENTAL_H + +#include "array.h" +#include "enumeration_experimental.h" +#include "tiledb_experimental.h" + +#include + +namespace tiledb { +class ArrayExperimental { + public: + /** + * Get the Enumeration from the attribute with name `attr_name`. + * + * @param ctx The context to use. + * @param array The array containing the attribute. + * @param attr_name The name of the attribute to get the enumeration from. + * @return Enumeration The enumeration. + */ + static Enumeration get_enumeration( + const Context& ctx, const Array& array, const std::string& attr_name) { + tiledb_enumeration_t* enmr; + ctx.handle_error(tiledb_array_get_enumeration( + ctx.ptr().get(), array.ptr().get(), attr_name.c_str(), &enmr)); + return Enumeration(ctx, enmr); + } + + /** + * Load all enumerations for the array + * + * @param ctx The context to use. + * @param array The array to load enumerations for. + * @param latest_only Whether to only load enumerations for the latest schema. + */ + static void load_all_enumerations( + const Context& ctx, const Array& array, bool latest_only = true) { + ctx.handle_error(tiledb_array_load_all_enumerations( + ctx.ptr().get(), array.ptr().get(), latest_only ? 1 : 0)); + } +}; + +} // namespace tiledb + +#endif diff --git a/tiledb/sm/cpp_api/array_schema_evolution.h b/tiledb/sm/cpp_api/array_schema_evolution.h index 33cec0454661..a8397974e0a4 100644 --- a/tiledb/sm/cpp_api/array_schema_evolution.h +++ b/tiledb/sm/cpp_api/array_schema_evolution.h @@ -151,6 +151,48 @@ class ArraySchemaEvolution { return *this; } + /** + * Adds an Enumeration to the array schema evolution. + * + * **Example:** + * @code{.cpp} + * tiledb::Context ctx; + * tiledb::ArraySchemaEvolution schema_evolution(ctx); + * std::vector values = {"red", "green", "blue"}; + * schema_evolution.add_enumeration(Enumeration::create(ctx, "an_enumeration", + * values)); + * @endcode + * + * @param attr The Attribute to add + * @return Reference to this `ArraySchemaEvolution` instance. + */ + ArraySchemaEvolution& add_enumeration(const Enumeration& enmr) { + auto& ctx = ctx_.get(); + ctx.handle_error(tiledb_array_schema_evolution_add_enumeration( + ctx.ptr().get(), evolution_.get(), enmr.ptr().get())); + return *this; + } + + /** + * Drops an enumeration. + * + * **Example:** + * @code{.cpp} + * tiledb::Context ctx; + * tiledb::ArraySchemaEvolution schema_evolution(ctx); + * schema_evolution.drop_enumeration("enumeration_name"); + * @endcode + * + * @param enumeration_name The enumeration to be dropped + * @return Reference to this `ArraySchemaEvolution` instance. + */ + ArraySchemaEvolution& drop_enumeration(const std::string& enumeration_name) { + auto& ctx = ctx_.get(); + ctx.handle_error(tiledb_array_schema_evolution_drop_enumeration( + ctx.ptr().get(), evolution_.get(), enumeration_name.c_str())); + return *this; + } + /** * Sets timestamp range. * @@ -217,4 +259,4 @@ class ArraySchemaEvolution { } // namespace tiledb -#endif // TILEDB_CPP_API_ARRAY_SCHEMA_EVOLUTION_H \ No newline at end of file +#endif // TILEDB_CPP_API_ARRAY_SCHEMA_EVOLUTION_H diff --git a/tiledb/sm/cpp_api/array_schema_experimental.h b/tiledb/sm/cpp_api/array_schema_experimental.h index e712860126a6..1bbee96ed689 100644 --- a/tiledb/sm/cpp_api/array_schema_experimental.h +++ b/tiledb/sm/cpp_api/array_schema_experimental.h @@ -35,6 +35,7 @@ #include "array_schema.h" #include "dimension_label_experimental.h" +#include "enumeration_experimental.h" #include "filter_list.h" #include "tiledb_experimental.h" @@ -43,6 +44,22 @@ namespace tiledb { class ArraySchemaExperimental { public: + /** + * Load an ArraySchema from the given URI with all of its enumerations. + * + * @param ctx The TileDB context. + * @param uri The URI to load from. + * @return ArraySchema The loaded array schema. + */ + static ArraySchema load_with_enumerations( + const Context& ctx, const std::string& uri) { + tiledb_ctx_t* c_ctx = ctx.ptr().get(); + tiledb_array_schema_t* schema; + ctx.handle_error(tiledb_array_schema_load_with_enumerations( + c_ctx, uri.c_str(), &schema)); + return ArraySchema(ctx, schema); + } + /** * Adds a DimensionLabel to the array. * @@ -154,6 +171,21 @@ class ArraySchemaExperimental { ctx.ptr().get(), array_schema.ptr().get(), name.c_str(), &dim_label)); return DimensionLabel(ctx, dim_label); } + + /** + * Add an enumeration to the array schema. + * + * @param ctx TileDB context. + * @param array_schema Target array schema. + * @param enmr The enumeration to add. + */ + static void add_enumeration( + const Context& ctx, + const ArraySchema& array_schema, + const Enumeration& enmr) { + ctx.handle_error(tiledb_array_schema_add_enumeration( + ctx.ptr().get(), array_schema.ptr().get(), enmr.ptr().get())); + } }; } // namespace tiledb diff --git a/tiledb/sm/cpp_api/attribute_experimental.h b/tiledb/sm/cpp_api/attribute_experimental.h new file mode 100644 index 000000000000..7070f1188cd3 --- /dev/null +++ b/tiledb/sm/cpp_api/attribute_experimental.h @@ -0,0 +1,94 @@ +/** + * @file attribute_experimental.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares the experimental C++ API for attributes. + */ + +#ifndef TILEDB_CPP_API_ATTRIBUTE_EXPERIMENTAL_H +#define TILEDB_CPP_API_ATTRIBUTE_EXPERIMENTAL_H + +#include "attribute.h" +#include "context.h" +#include "enumeration_experimental.h" + +#include + +namespace tiledb { +class AttributeExperimental { + public: + /** + * Set the enumeration name for an attribute. + * + * @param ctx TileDB context. + * @param attribute Target attribute. + * @param enumeration_name The name of the enumeration to set. + */ + static void set_enumeration_name( + const Context& ctx, + Attribute& attribute, + const std::string& enumeration_name) { + ctx.handle_error(tiledb_attribute_set_enumeration_name( + ctx.ptr().get(), attribute.ptr().get(), enumeration_name.c_str())); + } + + /** + * Get the attribute's enumeration name. + * + * @param ctx TileDB context. + * @param attribute Target attribute. + * @return std::optional The enumeration name if one exists. + */ + static std::optional get_enumeration_name( + const Context& ctx, Attribute& attribute) { + // Get the enumeration name as a string handle + tiledb_string_t* enmr_name; + tiledb_ctx_t* c_ctx = ctx.ptr().get(); + ctx.handle_error(tiledb_attribute_get_enumeration_name( + c_ctx, attribute.ptr().get(), &enmr_name)); + + if (enmr_name == nullptr) { + return std::nullopt; + } + + // Convert string handle to a std::string + const char* name_ptr; + size_t name_len; + ctx.handle_error(tiledb_string_view(enmr_name, &name_ptr, &name_len)); + std::string ret(name_ptr, name_len); + + // Release the string handle + ctx.handle_error(tiledb_string_free(&enmr_name)); + + return ret; + } +}; + +} // namespace tiledb + +#endif diff --git a/tiledb/sm/cpp_api/deleter.h b/tiledb/sm/cpp_api/deleter.h index c72d8ef71a1f..61f9a5d2c92f 100644 --- a/tiledb/sm/cpp_api/deleter.h +++ b/tiledb/sm/cpp_api/deleter.h @@ -111,6 +111,10 @@ class Deleter { tiledb_domain_free(&p); } + void operator()(tiledb_enumeration_t* p) const { + tiledb_enumeration_free(&p); + } + void operator()(tiledb_vfs_t* p) const { tiledb_vfs_free(&p); } diff --git a/tiledb/sm/cpp_api/enumeration_experimental.h b/tiledb/sm/cpp_api/enumeration_experimental.h new file mode 100644 index 000000000000..af4b8aa1bf80 --- /dev/null +++ b/tiledb/sm/cpp_api/enumeration_experimental.h @@ -0,0 +1,369 @@ +/** + * @file enumeration_experimental.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares the C++ API for the TileDB Enumeration class. + */ + +#ifndef TILEDB_CPP_API_ENUMERATION_EXPERIMENTAL_H +#define TILEDB_CPP_API_ENUMERATION_EXPERIMENTAL_H + +#include "context.h" +#include "tiledb.h" +#include "tiledb_experimental.h" + +namespace tiledb { + +class Enumeration { + public: + /* ********************************* */ + /* CONSTRUCTORS & DESTRUCTORS */ + /* ********************************* */ + + /** + * Create an enumeration by wrapping a pointer allocated by the C API + * + * @param ctx The Context to use. + * @param enmr The tiledb_enumeration_t allocated by the C API. + */ + Enumeration(const Context& ctx, tiledb_enumeration_t* enmr) + : ctx_(ctx) { + enumeration_ = std::shared_ptr(enmr, deleter_); + } + + Enumeration(const Enumeration&) = default; + Enumeration(Enumeration&&) = default; + Enumeration& operator=(const Enumeration&) = default; + Enumeration& operator=(Enumeration&&) = default; + + /** Destructor. */ + ~Enumeration() = default; + + /* ********************************* */ + /* API */ + /* ********************************* */ + + /** Get the C TileDB context object. + * + * @return shared_ptr + */ + std::shared_ptr ptr() const { + return enumeration_; + } + + /** + * Get the name of the enumeration + * + * @return std::string The name of the enumeration. + */ + std::string name() const { + // Get the enumeration name as a string handle + tiledb_string_t* enmr_name; + tiledb_ctx_t* c_ctx = ctx_.get().ptr().get(); + ctx_.get().handle_error( + tiledb_enumeration_get_name(c_ctx, enumeration_.get(), &enmr_name)); + + // Convert string handle to a std::string + const char* name_ptr; + size_t name_len; + ctx_.get().handle_error( + tiledb_string_view(enmr_name, &name_ptr, &name_len)); + std::string ret(name_ptr, name_len); + + // Release the string handle + ctx_.get().handle_error(tiledb_string_free(&enmr_name)); + + return ret; + } + + /** + * Get the type of the enumeration + * + * @return tiledb_datatype_t The datatype of the enumeration values. + */ + tiledb_datatype_t type() const { + tiledb_datatype_t ret; + tiledb_ctx_t* c_ctx = ctx_.get().ptr().get(); + ctx_.get().handle_error( + tiledb_enumeration_get_type(c_ctx, enumeration_.get(), &ret)); + return ret; + } + + /** + * Get the cell_val_num of the enumeration + * + * @return uint32_t The cell_val_num of the enumeration. + */ + uint32_t cell_val_num() const { + uint32_t ret; + tiledb_ctx_t* c_ctx = ctx_.get().ptr().get(); + ctx_.get().handle_error( + tiledb_enumeration_get_cell_val_num(c_ctx, enumeration_.get(), &ret)); + return ret; + } + + /** + * Get whether this enumeration is considered ordered. + * + * If an enumeration is not considered ordered inequality operators are + * disabled in QueryConditions applied against the enumeration values. + * + * @return bool Whether the enumeration is considered ordered. + */ + bool ordered() const { + int is_ordered; + tiledb_ctx_t* c_ctx = ctx_.get().ptr().get(); + ctx_.get().handle_error( + tiledb_enumeration_get_ordered(c_ctx, enumeration_.get(), &is_ordered)); + return is_ordered ? true : false; + } + + /** + * Convert the enumeration values into a vector + * + * @param vec The vector that will store the values of the enumeration. + */ + template + std::vector, T>> + as_vector() { + tiledb_ctx_t* c_ctx = ctx_.get().ptr().get(); + + const void* data; + uint64_t data_size; + ctx_.get().handle_error(tiledb_enumeration_get_data( + c_ctx, enumeration_.get(), &data, &data_size)); + + const T* elems = static_cast(data); + size_t count = data_size / sizeof(T); + + std::vector ret; + ret.reserve(count); + for (size_t i = 0; i < count; i++) { + ret.push_back(elems[i]); + } + + return ret; + } + + /** + * Convert the enumeration values into a vector of std::string values + * + * @param vec The vector used to return the string data. + */ + template + std::vector, T>> + as_vector() { + tiledb_ctx_t* c_ctx = ctx_.get().ptr().get(); + + const void* data; + uint64_t data_size; + ctx_.get().handle_error(tiledb_enumeration_get_data( + c_ctx, enumeration_.get(), &data, &data_size)); + + const void* offsets; + uint64_t offsets_size; + ctx_.get().handle_error(tiledb_enumeration_get_offsets( + c_ctx, enumeration_.get(), &offsets, &offsets_size)); + + const char* str_data = static_cast(data); + const uint64_t* elems = static_cast(offsets); + size_t count = offsets_size / sizeof(uint64_t); + + std::vector ret; + ret.reserve(count); + for (size_t i = 0; i < count; i++) { + uint64_t len; + if (i + 1 < count) { + len = elems[i + 1] - elems[i]; + } else { + len = data_size - elems[i]; + } + ret.emplace_back(str_data + elems[i], len); + } + + return ret; + } + + /** + * Dump a string representation of the Enumeration to the given FILE pointer. + * + * @param out A FILE pointer to write to. stdout is used if nullptr is passed. + */ + void dump(FILE* out = nullptr) const { + ctx_.get().handle_error(tiledb_enumeration_dump( + ctx_.get().ptr().get(), enumeration_.get(), out)); + } + + /* ********************************* */ + /* STATIC FUNCTIONS */ + /* ********************************* */ + + /** + * Create an enumeration from a vector of trivial values (i.e., int's or other + * integral or floating point values) + * + * @param ctx The context to use. + * @param values The list of values to use for this enumeration. + * @param ordered Whether or not to consider this enumeration ordered. + * @param type The datatype of the enumeration values. This is automatically + * deduced if not provided. + */ + template * = nullptr> + static Enumeration create( + const Context& ctx, + const std::string& name, + std::vector& values, + bool ordered = false, + tiledb_datatype_t type = static_cast(255)) { + using DataT = impl::TypeHandler; + + if (type == static_cast(255)) { + type = DataT::tiledb_type; + } + + return create( + ctx, + name, + type, + DataT::tiledb_num, + ordered, + values.data(), + values.size() * sizeof(T), + nullptr, + 0); + } + + /** + * Create an enumeration from a vector of strings + * + * @param ctx The context to use. + * @param values The vector of values for the enumeration. + * @param ordered Whether to consider the enumerationv alues as ordered. + * @param type The datatype of the enumeration values. This is automatically + * deduced if not provided. However, this can be used to override the + * deduced type if need be. For instance, TILEDB_STRING_ASCII is the + * default type for strings but TILEDB_STRING_UTF8 can be specified. + */ + template * = nullptr> + static Enumeration create( + const Context& ctx, + const std::string& name, + std::vector>& values, + bool ordered = false, + tiledb_datatype_t type = static_cast(255)) { + using DataT = impl::TypeHandler; + static_assert( + DataT::tiledb_num == 1, "Enumeration string values cannot be compound"); + + if (type == static_cast(255)) { + type = DataT::tiledb_type; + } + + uint64_t total_size = 0; + for (auto v : values) { + total_size += v.size() * sizeof(T); + } + + std::vector data(total_size, 0); + std::vector offsets; + offsets.reserve(values.size()); + uint64_t curr_offset = 0; + + for (auto v : values) { + std::memcpy(data.data() + curr_offset, v.data(), v.size() * sizeof(T)); + offsets.push_back(curr_offset); + curr_offset += v.size() * sizeof(T); + } + + return create( + ctx, + name, + type, + TILEDB_VAR_NUM, + ordered, + data.data(), + total_size, + offsets.data(), + offsets.size() * sizeof(uint64_t)); + } + + /** + * Create an enumeration + * + * @param ctx The context to use. + * @param type The datatype of the enumeration values. + * @param cell_val_num The cell_val_num of the enumeration. + * @param ordered Whether this enumeration should be considered ordered. + * @param data A pointer to a buffer of values for this enumeration. + * @param data_size The size of the buffer pointed to by data. + * @param offsets A pointer to the offsets buffer if required. + * @param offsets_size The size of the buffer pointed to by offsets. + */ + static Enumeration create( + const Context& ctx, + const std::string& name, + tiledb_datatype_t type, + uint32_t cell_val_num, + bool ordered, + const void* data, + uint64_t data_size, + const void* offsets, + uint64_t offsets_size) { + tiledb_enumeration_t* enumeration; + ctx.handle_error(tiledb_enumeration_alloc( + ctx.ptr().get(), + name.c_str(), + type, + cell_val_num, + ordered, + data, + data_size, + offsets, + offsets_size, + &enumeration)); + return Enumeration(ctx, enumeration); + } + + private: + /* ********************************* */ + /* PRIVATE ATTRIBUTES */ + /* ********************************* */ + + /** The TileDB context. */ + std::reference_wrapper ctx_; + + /** Deleter wrapper. */ + impl::Deleter deleter_; + + /** Pointer to the TileDB C Enumeration object. */ + std::shared_ptr enumeration_; +}; + +} // namespace tiledb + +#endif // TILEDB_CPP_API_ENUMERATION_EXPERIMENTAL_H diff --git a/tiledb/sm/cpp_api/query_condition_experimental.h b/tiledb/sm/cpp_api/query_condition_experimental.h new file mode 100644 index 000000000000..8d982b4e5624 --- /dev/null +++ b/tiledb/sm/cpp_api/query_condition_experimental.h @@ -0,0 +1,57 @@ +/** + * @file query_condition_experimental.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares the experimental C++ API for the query condition. + */ + +#ifndef TILEDB_CPP_API_QUERY_CONDITION_EXPERIMENTAL_H +#define TILEDB_CPP_API_QUERY_CONDITION_EXPERIMENTAL_H + +#include "query_condition.h" + +namespace tiledb { +class QueryConditionExperimental { + public: + /** + * Set whether or not to use the associated enumeration. + * + * @param ctx TileDB Context. + * @param cond TileDB Query Condition. + * @param use_enumeration Whether to use the associated enumeration. + */ + static void set_use_enumeration( + const Context& ctx, QueryCondition& cond, bool use_enumeration) { + ctx.handle_error(tiledb_query_condition_set_use_enumeration( + ctx.ptr().get(), cond.ptr().get(), use_enumeration ? 1 : 0)); + } +}; + +} // namespace tiledb + +#endif diff --git a/tiledb/sm/cpp_api/tiledb_experimental b/tiledb/sm/cpp_api/tiledb_experimental index df23d84ae518..2bdb0d673e4b 100644 --- a/tiledb/sm/cpp_api/tiledb_experimental +++ b/tiledb/sm/cpp_api/tiledb_experimental @@ -33,11 +33,15 @@ #ifndef TILEDB_EXPERIMENTAL_CPP_H #define TILEDB_EXPERIMENTAL_CPP_H +#include "array_experimental.h" #include "array_schema_evolution.h" #include "array_schema_experimental.h" +#include "attribute_experimental.h" #include "dimension_label_experimental.h" +#include "enumeration_experimental.h" #include "consolidation_plan_experimental.h" #include "group_experimental.h" +#include "query_condition_experimental.h" #include "query_experimental.h" #include "subarray_experimental.h" diff --git a/tiledb/sm/enums/datatype.h b/tiledb/sm/enums/datatype.h index c2050d6d674b..dfa59ef3f420 100644 --- a/tiledb/sm/enums/datatype.h +++ b/tiledb/sm/enums/datatype.h @@ -45,6 +45,12 @@ using namespace tiledb::common; namespace tiledb { namespace sm { +#if defined(_WIN32) && defined(TIME_MS) +#pragma message("WARNING: Windows.h may have already been included before") +#pragma message(" tiledb/sm/enums/datatype.h which will likely cause a") +#pragma message(" syntax error while defining the Datatype enum class.") +#endif + /** Defines a datatype. */ enum class Datatype : uint8_t { #define TILEDB_DATATYPE_ENUM(id) id @@ -325,11 +331,43 @@ inline bool datatype_is_string(Datatype type) { /** Returns true if the input datatype is an integer type. */ inline bool datatype_is_integer(Datatype type) { return ( - type == Datatype::BLOB || type == Datatype::BOOL || - type == Datatype::INT8 || type == Datatype::UINT8 || - type == Datatype::INT16 || type == Datatype::UINT16 || - type == Datatype::INT32 || type == Datatype::UINT32 || - type == Datatype::INT64 || type == Datatype::UINT64); + type == Datatype::BOOL || type == Datatype::INT8 || + type == Datatype::UINT8 || type == Datatype::INT16 || + type == Datatype::UINT16 || type == Datatype::INT32 || + type == Datatype::UINT32 || type == Datatype::INT64 || + type == Datatype::UINT64); +} + +/** + * Return the largest integral value for the provided type. + * + * @param type The Datatype to return the maximum value of. + * @returns uint64_t The maximum integral value for the provided type. + */ +inline uint64_t datatype_max_integral_value(Datatype type) { + switch (type) { + case Datatype::BOOL: + return static_cast(std::numeric_limits::max()); + case Datatype::INT8: + return static_cast(std::numeric_limits::max()); + case Datatype::UINT8: + return static_cast(std::numeric_limits::max()); + case Datatype::INT16: + return static_cast(std::numeric_limits::max()); + case Datatype::UINT16: + return static_cast(std::numeric_limits::max()); + case Datatype::INT32: + return static_cast(std::numeric_limits::max()); + case Datatype::UINT32: + return static_cast(std::numeric_limits::max()); + case Datatype::INT64: + return static_cast(std::numeric_limits::max()); + case Datatype::UINT64: + return static_cast(std::numeric_limits::max()); + default: + throw std::runtime_error( + "Datatype (" + datatype_str(type) + ") is not integral."); + } } /** Returns true if the input datatype is a real type. */ diff --git a/tiledb/sm/filter/bit_width_reduction_filter.cc b/tiledb/sm/filter/bit_width_reduction_filter.cc index f114382e8e4c..08c14ab68205 100644 --- a/tiledb/sm/filter/bit_width_reduction_filter.cc +++ b/tiledb/sm/filter/bit_width_reduction_filter.cc @@ -108,7 +108,8 @@ Status BitWidthReductionFilter::run_forward( auto tile_type_size = static_cast(datatype_size(tile_type)); // If bit width compression can't work, just return the input unmodified. - if (!datatype_is_integer(tile_type) || tile_type_size == 1) { + if ((!datatype_is_integer(tile_type) && tile_type != Datatype::BLOB) || + tile_type_size == 1) { RETURN_NOT_OK(output->append_view(input)); RETURN_NOT_OK(output_metadata->append_view(input_metadata)); return Status::Ok(); @@ -292,7 +293,8 @@ Status BitWidthReductionFilter::run_reverse( auto tile_type_size = static_cast(datatype_size(tile_type)); // If bit width compression wasn't applied, just return the input unmodified. - if (!datatype_is_integer(tile_type) || tile_type_size == 1) { + if ((!datatype_is_integer(tile_type) && tile_type != Datatype::BLOB) || + tile_type_size == 1) { RETURN_NOT_OK(output->append_view(input)); RETURN_NOT_OK(output_metadata->append_view(input_metadata)); return Status::Ok(); diff --git a/tiledb/sm/filter/positive_delta_filter.cc b/tiledb/sm/filter/positive_delta_filter.cc index 5b4212b8ad2f..85ed05b8b761 100644 --- a/tiledb/sm/filter/positive_delta_filter.cc +++ b/tiledb/sm/filter/positive_delta_filter.cc @@ -70,7 +70,7 @@ Status PositiveDeltaFilter::run_forward( auto tile_type = tile.type(); // If encoding can't work, just return the input unmodified. - if (!datatype_is_integer(tile_type)) { + if (!datatype_is_integer(tile_type) && tile_type != Datatype::BLOB) { RETURN_NOT_OK(output->append_view(input)); RETURN_NOT_OK(output_metadata->append_view(input_metadata)); return Status::Ok(); @@ -250,7 +250,7 @@ Status PositiveDeltaFilter::run_reverse( auto tile_type = tile.type(); // If encoding wasn't applied, just return the input unmodified. - if (!datatype_is_integer(tile_type)) { + if (!datatype_is_integer(tile_type) && tile_type != Datatype::BLOB) { RETURN_NOT_OK(output->append_view(input)); RETURN_NOT_OK(output_metadata->append_view(input_metadata)); return Status::Ok(); diff --git a/tiledb/sm/misc/constants.cc b/tiledb/sm/misc/constants.cc index bb268dfec46f..60f70639fa8c 100644 --- a/tiledb/sm/misc/constants.cc +++ b/tiledb/sm/misc/constants.cc @@ -93,6 +93,9 @@ const std::string fragment_metadata_filename = "__fragment_metadata.tdb"; /** The array dimension labels directory name. */ const std::string array_dimension_labels_dir_name = "__labels"; +/** The array enumerations directory name. */ +const std::string array_enumerations_dir_name = "__enumerations"; + /** The array directory names. */ const std::vector array_dir_names = { array_schema_dir_name, @@ -100,7 +103,8 @@ const std::vector array_dir_names = { array_fragment_meta_dir_name, array_fragments_dir_name, array_commits_dir_name, - array_dimension_labels_dir_name}; + array_dimension_labels_dir_name, + array_enumerations_dir_name}; /** The default tile capacity. */ const uint64_t capacity = 10000; @@ -222,6 +226,9 @@ const uint32_t empty_ucs4 = 0; /** The special value for an empty ANY. */ const uint8_t empty_any = 0; +/** The return value for missing entries in an Enumeration */ +const uint64_t enumeration_missing_value = std::numeric_limits::max(); + /** The file suffix used in TileDB. */ const std::string file_suffix = ".tdb"; @@ -660,7 +667,7 @@ const int32_t library_version[3] = { TILEDB_VERSION_MAJOR, TILEDB_VERSION_MINOR, TILEDB_VERSION_PATCH}; /** The TileDB serialization base format version number. */ -const format_version_t base_format_version = 19; +const format_version_t base_format_version = 20; /** * The TileDB serialization format version number. @@ -685,6 +692,12 @@ const format_version_t deletes_min_version = 16; /** The lowest version supported for updates. */ const format_version_t updates_min_version = 16; +/** The lowest version supported format version for enumerations. */ +const format_version_t enumerations_min_format_version = 20; + +/** The current enumerations version. */ +const format_version_t enumerations_version = 0; + /** The maximum size of a tile chunk (unit of compression) in bytes. */ const uint64_t max_tile_chunk_size = 64 * 1024; diff --git a/tiledb/sm/misc/constants.h b/tiledb/sm/misc/constants.h index fa059291b493..2b023dc4210c 100644 --- a/tiledb/sm/misc/constants.h +++ b/tiledb/sm/misc/constants.h @@ -83,6 +83,9 @@ extern const std::string array_commits_dir_name; /** The array dimension labels directory name. */ extern const std::string array_dimension_labels_dir_name; +/** The array enumerations directory name. */ +extern const std::string array_enumerations_dir_name; + /** The array directory names. */ extern const std::vector array_dir_names; @@ -203,6 +206,9 @@ extern const uint32_t empty_ucs4; /** The special value for an empty ANY. */ extern const uint8_t empty_any; +/** The return value for missing entries in an Enumeration */ +extern const uint64_t enumeration_missing_value; + /** The file suffix used in TileDB. */ extern const std::string file_suffix; @@ -654,6 +660,12 @@ extern const format_version_t deletes_min_version; /** The lowest version supported for updates. */ extern const format_version_t updates_min_version; +/** The lowest version supported for enumerations. */ +extern const format_version_t enumerations_min_format_version; + +/** The current Enumerations version. */ +extern const format_version_t enumerations_version; + /** The maximum size of a tile chunk (unit of compression) in bytes. */ extern const uint64_t max_tile_chunk_size; @@ -737,7 +749,6 @@ extern const uint64_t s3_min_multipart_part_size; * global order writes intermediate chunks */ extern const std::string s3_multipart_buffering_dirname; - } // namespace constants } // namespace sm diff --git a/tiledb/sm/misc/integral_type_casts.h b/tiledb/sm/misc/integral_type_casts.h new file mode 100644 index 000000000000..a82611765368 --- /dev/null +++ b/tiledb/sm/misc/integral_type_casts.h @@ -0,0 +1,144 @@ +/** + * @file type_casts.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file contains safe type casting utilities. + */ + +#ifndef TILEDB_TYPE_CASTS_H +#define TILEDB_TYPE_CASTS_H + +#include +#include + +#include "tiledb/common/unreachable.h" +#include "tiledb/sm/enums/datatype.h" +#include "tiledb/sm/misc/types.h" + +namespace tiledb::sm::utils { + +/** + * Safely convert integral values between different bit widths checking for + * invalid conversions. These casts are named "safe" because they only allow + * casts that result in the same semantic value in the target type. This means + * that some casts that would be allowed by static_cast or similar facilities + * are rejected. Most notably, casts between types with different signed-ness + * are more thoroughly checked for correctness. + * + * This would likely be significantly faster if there was a cross platform + * interface for `__builtin_clzll`. However, the current needs don't require + * the absolute fastest speed as this is currently only used once per + * Enumeration attribute per query (as opposed to once per query condition + * comparison or something crazy). + * + * @param src The value to convert. + * @return The converted value. + */ +template +Target safe_integral_cast(Source src) { + using SourceLimits = std::numeric_limits; + using TargetLimits = std::numeric_limits; + + static_assert(SourceLimits::is_integer, "Source is not an integral type"); + static_assert(TargetLimits::is_integer, "Target is not an integral type"); + + Target ret = static_cast(src); + + // If it can't round trip, its an invalid cast. Note that the converse + // is not true as a sign could have changed for types of the same bit width + // but different signed-ness. + if (static_cast(ret) != src) { + throw std::invalid_argument("Invalid integral cast: Roundtrip failed"); + } + + // If the sign changed + if ((src < 0 && ret >= 0) || (src >= 0 && ret < 0)) { + throw std::invalid_argument("Invalid integral cast: Sign changed"); + } + + return ret; +} + +/** + * Use `safe_integral_cast` to convert an integral value into a specific + * Datatype stored in a ByteVecValue. + * + * @param value The value to convert. + * @param type The datatype to convert the value to. + * @param dest A ByteVecValue to store the converted value in. + */ +template +void safe_integral_cast_to_datatype( + Source value, Datatype type, ByteVecValue& dest) { + if (!datatype_is_integer(type)) { + throw std::invalid_argument("Datatype must be integral"); + } + + if (type == Datatype::BLOB) { + throw std::invalid_argument( + "Datatype::BLOB not supported in integral conversion"); + } + + switch (type) { + case Datatype::BOOL: + dest.assign_as(safe_integral_cast(value)); + return; + case Datatype::INT8: + dest.assign_as(safe_integral_cast(value)); + return; + case Datatype::UINT8: + dest.assign_as(safe_integral_cast(value)); + return; + case Datatype::INT16: + dest.assign_as(safe_integral_cast(value)); + return; + case Datatype::UINT16: + dest.assign_as(safe_integral_cast(value)); + return; + case Datatype::INT32: + dest.assign_as(safe_integral_cast(value)); + return; + case Datatype::UINT32: + dest.assign_as(safe_integral_cast(value)); + return; + case Datatype::INT64: + dest.assign_as(safe_integral_cast(value)); + return; + case Datatype::UINT64: + dest.assign_as(safe_integral_cast(value)); + return; + default: + throw std::logic_error("Definitions of integral types are mismatched."); + } + + ::stdx::unreachable(); +} + +} // namespace tiledb::sm::utils + +#endif // TILEDB_TYPE_CASTS_H diff --git a/tiledb/sm/misc/test/CMakeLists.txt b/tiledb/sm/misc/test/CMakeLists.txt index 6b9adac5e5b3..0c6bd18206d1 100644 --- a/tiledb/sm/misc/test/CMakeLists.txt +++ b/tiledb/sm/misc/test/CMakeLists.txt @@ -30,5 +30,12 @@ commence(unit_test misc) this_target_link_libraries(tiledb_test_support_lib) # change to `this_target_include_directories` when available target_include_directories(unit_misc PRIVATE "${CMAKE_SOURCE_DIR}") - this_target_sources(main.cc unit_bytevecvalue.cc unit_hilbert.cc unit_math.cc unit_uuid.cc) + this_target_sources( + main.cc + unit_bytevecvalue.cc + unit_hilbert.cc + unit_integral_type_casts.cc + unit_math.cc + unit_uuid.cc + ) conclude(unit_test) diff --git a/tiledb/sm/misc/test/unit_integral_type_casts.cc b/tiledb/sm/misc/test/unit_integral_type_casts.cc new file mode 100644 index 000000000000..26bb9dd190a6 --- /dev/null +++ b/tiledb/sm/misc/test/unit_integral_type_casts.cc @@ -0,0 +1,511 @@ +/** + * @file unit_integral_type_casts.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * Tests the C++ API for enumeration related functions. + */ + +#include +#include + +#include "test/support/tdb_catch.h" +#include "tiledb/sm/misc/integral_type_casts.h" + +// These tests are for the safe_integral_cast and safe_itegral_cast_to_datatype +// functions found in `tiledb/sm/misc/integral_type_casts.h`. +// +// There are two main tests in this file, one for safe_integral_cast and one +// for safe_integral_cast_to_datatype. These tests work by generating a +// comprehensive sample of test data to exhaustively check that casts between +// all supported integral datatypes are covered. Generating tests in this +// manner allows us to assert whether a cast should succeed vs one that should +// fail. +// +// Both tests have the same general outline. First, pass a callback to the +// run_test function which is responsible for generating a TestCase value +// which is nothing more than a type width (i.e., sizeof(int32_t)) and a +// set of random bytes in length 1 to byte_width. This ensures that we're +// able to test casts that should succeed (i.e., 5 stored in an int32_t is +// castable to a uint8_t) and casts that should fail (i.e., 300 stored in an +// int32_t is not castable to either int8_t or uint8_t). +// +// The callback passed to `run_test` is then responsible for enumerating all +// of the test types for cast checks. Both tests have the same naming pattern +// of: +// +// 1. check_$function_name - Based on TestCase.byte_width, pick the source +// datatype for the test +// 2. dispatch_$function_name - Attempt to convert the source test data to all +// target types. +// 3. run_$functon_name - Attempt the cast for the given TestData, source, and +// target types. These use the should_cast_succeed helper to know whether +// a given cast should succeed or fail. +// +// Beyond the main test layouts, there are two important helper functions +// that will be useful for understanding. +// +// First, `should_cast_succeed` which takes the source type and value, and a +// target type and returns a boolean on whether a cast should succeed. This +// is the crux of these tests as this logic is the concise list of rules that +// dictate when a cast is safe or not. +// +// The second function is `calculate_width` which is called by +// `should_cast_succeed`. This calculates the number of bits and bytes required +// to hold a given value and informs `should_cast_succeed` when a cast is +// representable in a target type. If the code in this function looks a little +// weird, you'll want to review 2's complement representation on why it counts +// things as it does. + +using namespace tiledb::sm; + +/* ********************************* */ +/* CONSTANTS */ +/* ********************************* */ + +// Widths of integral types int8_t to int64_t +constexpr int TYPE_WIDTHS[] = {1, 2, 4, 8}; +constexpr int NUM_TYPE_WIDTHS = 4; + +// Width of widest type (i.e., sizeof(int64_t)) +constexpr int MAX_WIDTH = 8; + +// An aribtrary number of iterations to run per test +constexpr int RANDOM_ITERATIONS = 500; + +// A POD struct representing a test case +struct TestCase { + // The width of the type, i.e., sizeof(int8_t), sizeof(uint32_t), etc, this + // is obviously restricted to the values 1, 2, 4, 8. + uint8_t type_width; + + // Storage for the generated value's data. LSB is data[0] + uint8_t data[MAX_WIDTH]; +}; + +/* ********************************* */ +/* TEST CALLBACKS */ +/* ********************************* */ + +// Test implementation for util::safe_integral_cast +void check_safe_integral_cast(const TestCase& tc); + +// Test implementation for util::safe_integral_cast_to_datatype +void check_safe_integral_cast_to_datatype(const TestCase& tc); + +/* ********************************* */ +/* TEST HELPER FUNCTIONS */ +/* ********************************* */ + +// Test driver function +template +void run_test(TestFunc func); + +// Apply the test function against each source type based on tc.type_width +template +void dispatch(TestFunc func, const TestCase& tc); + +// Return a boolean indicating whether the given cast should succeed or fail +template +bool should_cast_succeed(Source src); + +template +bool should_cast_succeed(Source src, Datatype dtype); + +// Calculate the width of a value in both bits and bytes. +template +std::tuple calculate_width(Type val); + +// Fills in a TestCase's data, byte_width, and msb_set members +void generate_case(TestCase& tc, uint8_t gen_byte_width, bool set_msb); + +// Convert a uint8_t[] to integral type T +template +T buf_to_value(const uint8_t* buffer); + +/* ********************************* */ +/* safe_integral_cast */ +/* ********************************* */ + +TEST_CASE("util::safe_integral_cast", "[safe-integral-casts]") { + run_test(check_safe_integral_cast); +} + +template +void dispatch_safe_integral_cast(const TestCase& tc); + +template +void run_safe_integral_cast(const TestCase&, Source src); + +void check_safe_integral_cast(const TestCase& tc) { + switch (tc.type_width) { + case 1: + dispatch_safe_integral_cast(tc); + dispatch_safe_integral_cast(tc); + return; + case 2: + dispatch_safe_integral_cast(tc); + dispatch_safe_integral_cast(tc); + return; + case 4: + dispatch_safe_integral_cast(tc); + dispatch_safe_integral_cast(tc); + return; + case 8: + dispatch_safe_integral_cast(tc); + dispatch_safe_integral_cast(tc); + return; + default: + throw std::logic_error("Invalid type_width"); + } +} + +template +void dispatch_safe_integral_cast(const TestCase& tc) { + Source val = buf_to_value(tc.data); + + run_safe_integral_cast(tc, val); + run_safe_integral_cast(tc, val); + + run_safe_integral_cast(tc, val); + run_safe_integral_cast(tc, val); + + run_safe_integral_cast(tc, val); + run_safe_integral_cast(tc, val); + + run_safe_integral_cast(tc, val); + run_safe_integral_cast(tc, val); +} + +template +void run_safe_integral_cast(const TestCase&, Source src) { + if (should_cast_succeed(src)) { + auto tgt = utils::safe_integral_cast(src); + REQUIRE(src == utils::safe_integral_cast(tgt)); + } else { + REQUIRE_THROWS(utils::safe_integral_cast(src)); + } +} + +/* ********************************* */ +/* safe_integral_cast_to_datatype */ +/* ********************************* */ + +TEST_CASE("util::safe_integral_cast_to_datatype", "[safe-integral-casts]") { + run_test(check_safe_integral_cast_to_datatype); +} + +TEST_CASE( + "util::safe_integral_cast_to_datatype bad type", + "[safe-integral-casts][error]") { + ByteVecValue bvv; + REQUIRE_THROWS(utils::safe_integral_cast_to_datatype(5, Datatype::BLOB, bvv)); + REQUIRE_THROWS( + utils::safe_integral_cast_to_datatype(5, Datatype::STRING_ASCII, bvv)); +} + +template +void dispatch_safe_integral_cast_to_datatype(const TestCase& tc); + +template +void run_safe_integral_cast_to_datatype( + const TestCase&, Source src, Datatype dt); + +void check_safe_integral_cast_to_datatype(const TestCase& tc) { + switch (tc.type_width) { + case 1: + dispatch_safe_integral_cast_to_datatype(tc); + dispatch_safe_integral_cast_to_datatype(tc); + return; + case 2: + dispatch_safe_integral_cast_to_datatype(tc); + dispatch_safe_integral_cast_to_datatype(tc); + return; + case 4: + dispatch_safe_integral_cast_to_datatype(tc); + dispatch_safe_integral_cast_to_datatype(tc); + return; + case 8: + dispatch_safe_integral_cast_to_datatype(tc); + dispatch_safe_integral_cast_to_datatype(tc); + return; + default: + throw std::logic_error("Invalid type_width"); + } +} + +template +void dispatch_safe_integral_cast_to_datatype(const TestCase& tc) { + Source val = buf_to_value(tc.data); + + std::vector datatypes = { + Datatype::BOOL, + Datatype::INT8, + Datatype::UINT8, + Datatype::INT16, + Datatype::UINT16, + Datatype::INT32, + Datatype::UINT32, + Datatype::INT64, + Datatype::UINT64}; + + for (auto dt : datatypes) { + run_safe_integral_cast_to_datatype(tc, val, dt); + } +} + +template +void run_safe_integral_cast_to_datatype( + const TestCase&, Source src, Datatype dtype) { + ByteVecValue dest; + if (should_cast_succeed(src, dtype)) { + utils::safe_integral_cast_to_datatype(src, dtype, dest); + switch (dtype) { + case Datatype::BOOL: + REQUIRE(dest.rvalue_as() == (uint8_t)src); + return; + case Datatype::INT8: + REQUIRE(dest.rvalue_as() == (int8_t)src); + return; + case Datatype::UINT8: + REQUIRE(dest.rvalue_as() == (uint8_t)src); + return; + case Datatype::INT16: + REQUIRE(dest.rvalue_as() == (int16_t)src); + return; + case Datatype::UINT16: + REQUIRE(dest.rvalue_as() == (uint16_t)src); + return; + case Datatype::INT32: + REQUIRE(dest.rvalue_as() == (int32_t)src); + return; + case Datatype::UINT32: + REQUIRE(dest.rvalue_as() == (uint32_t)src); + return; + case Datatype::INT64: + REQUIRE(dest.rvalue_as() == (int64_t)src); + return; + case Datatype::UINT64: + REQUIRE(dest.rvalue_as() == (uint64_t)src); + return; + default: + throw std::logic_error("Invalid datatype for test"); + } + } else { + REQUIRE_THROWS( + utils::safe_integral_cast_to_datatype(src, dtype, dest)); + } +} + +/* ********************************* */ +/* TEST HELPER IMPLEMENTATIONS */ +/* ********************************* */ + +template +void run_test(TestFunc tfunc) { + TestCase tc; + for (uint8_t i = 0; i < NUM_TYPE_WIDTHS; i++) { + tc.type_width = TYPE_WIDTHS[i]; + + // Always check no bits set edge case + memset(tc.data, 0, MAX_WIDTH); + tfunc(tc); + + for (uint8_t gen_width = 1; gen_width <= tc.type_width; gen_width++) { + for (int i = 0; i < RANDOM_ITERATIONS; i++) { + generate_case(tc, gen_width, false); + tfunc(tc); + + generate_case(tc, gen_width, true); + tfunc(tc); + } + } + + // Always check all bits set edge case + memset(tc.data, 255, MAX_WIDTH); + tfunc(tc); + } +} + +template +bool should_cast_succeed(Source src) { + auto [bit_width, byte_width] = calculate_width(src); + bool tgt_signed = std::numeric_limits::is_signed; + uint8_t tgt_size = sizeof(Target); + + // Obviously, if we have more bytes in src than the Target type can hold, + // the cast will fail. + if (byte_width > tgt_size) { + return false; + } + + // Unsigned types can't hold negative values, so if src is negative + // and Target is not signed, we must fail. + if (src < 0 && !tgt_signed) { + return false; + } + + // When converting from a positive value to a signed type we have to make + // sure that the signed type has more than bit_width + 1 bits available so + // that the resulting value doesn't become negative. + if ((src > 0 && tgt_signed) && (bit_width >= (tgt_size * 8))) { + return false; + } + + // When converting to a negative value to a signed type, we have to make + // sure the target has enough bits to represent the value. + if ((src < 0 && tgt_signed) && (bit_width >= (tgt_size * 8))) { + return false; + } + + // Otherwise, the cast should succeed + return true; +} + +template +bool should_cast_succeed(Source src, Datatype dtype) { + switch (dtype) { + case Datatype::BOOL: + return should_cast_succeed(src); + case Datatype::INT8: + return should_cast_succeed(src); + case Datatype::UINT8: + return should_cast_succeed(src); + case Datatype::INT16: + return should_cast_succeed(src); + case Datatype::UINT16: + return should_cast_succeed(src); + case Datatype::INT32: + return should_cast_succeed(src); + case Datatype::UINT32: + return should_cast_succeed(src); + case Datatype::INT64: + return should_cast_succeed(src); + case Datatype::UINT64: + return should_cast_succeed(src); + default: + throw std::logic_error("Invalid datatype for test"); + } +} + +template +std::tuple calculate_width(Type val) { + // Calculate the number of bits required to hold src. For positive values, + // this is the largest numbered bit set to 1, for negative values its the + // largest numbered bit set to 0. + + uint8_t bit_width = 0; + if (val > 0) { + // For positive values, we find the highest set bit by counting the number + // of right shifts required before the value is zero. + uint8_t i = 0; + while (val > 0) { + val >>= 1; + i += 1; + } + bit_width = i; + } else if (val < 0) { + // For negative values, we find the highest unset bit by counting the + // number of left shifts before the value becomes greater than or equal + // to zero. + uint8_t i = 0; + while (val < 0) { + val <<= 1; + i += 1; + } + bit_width = sizeof(Type) * 8 - i; + } + + uint8_t byte_width = bit_width / 8; + if (bit_width % 8 != 0) { + byte_width += 1; + } + + return {bit_width, byte_width}; +} + +// A simple function for generating random uint64_t values +uint64_t generate_value() { + static std::random_device rand_dev; + static std::mt19937_64 generator(rand_dev()); + static std::uniform_int_distribution dist; + return dist(generator); +} + +// Generate a TestCase with the given gen_width number of random bytes. The +// only curisoity is the set_msb value which forces the most significant bit +// of the most significant byte to 0 or 1. Forcing the msb to 1 allows us to +// assert that we're testing two important cases. First, casting negative +// values between different types widths, and second, checking that large +// unsigned values of a given type that aren't representable in the signed +// type of the same byte width. +void generate_case(TestCase& tc, uint8_t gen_width, bool set_msb) { + // Reset TestCase state + memset(tc.data, 0, MAX_WIDTH); + + // Convince GCC that we'll not write beyond MAX_WIDTH bytes into tc.data + if (gen_width > MAX_WIDTH) { + throw std::logic_error("Invalid gen_width in test case"); + } + + // A standard bit shift approach for converting integral values into + // an array of uint8_t values. Its important to note that the << and >> + // operators are endian-ness agnostic. So this just masks everything but + // the least-significant-byte, converts that to a uint8_t and stores it + // in the array. Then drop the least-significant byte of the random value + // by shifting 8 bits to the right. + // + // TestCase.byte_width is calculated here since its perfectly valid for + // a generated value to not have any bits set in any given byte which can + // cause expected failures to unexpected succeed if its the MSB that is + // all zeroes. + uint64_t rand_value = generate_value(); + for (int i = 0; i < gen_width; i++) { + tc.data[i] = static_cast(rand_value & 0xFF); + rand_value >>= 8; + } + + // If set_msb is false, set the gen_width's byte msb to 0, else set to 1 + if (!set_msb) { + tc.data[gen_width - 1] &= 0x7F; + } else { + tc.data[gen_width - 1] |= 0x80; + } +} + +// Convert a uint8_t buffer to the given type T. This works using the standard +// bit twiddling approach to progressive bitwise-or one byte at a time in a +// most-to-least significant byte order. +template +T buf_to_value(const uint8_t* buffer) { + T ret = 0; + for (int8_t i = sizeof(T) - 1; i >= 0; i--) { + T new_byte = (T)buffer[i]; + ret = (ret << 8) | new_byte; + } + return ret; +} diff --git a/tiledb/sm/misc/utils.cc b/tiledb/sm/misc/utils.cc index a3ac5a6aa967..3124e4e0af1a 100644 --- a/tiledb/sm/misc/utils.cc +++ b/tiledb/sm/misc/utils.cc @@ -33,6 +33,7 @@ #include "tiledb/sm/misc/utils.h" #include "tiledb/common/logger.h" +#include "tiledb/common/unreachable.h" #include "tiledb/sm/enums/datatype.h" #include "tiledb/sm/filesystem/uri.h" #include "tiledb/sm/misc/constants.h" diff --git a/tiledb/sm/query/ast/CMakeLists.txt b/tiledb/sm/query/ast/CMakeLists.txt index 4568075fbeb7..c46c57fd7244 100644 --- a/tiledb/sm/query/ast/CMakeLists.txt +++ b/tiledb/sm/query/ast/CMakeLists.txt @@ -39,7 +39,7 @@ list(APPEND SOURCES # commence(object_library query_ast) this_target_sources(${SOURCES}) - this_target_object_libraries(array_schema baseline) + this_target_object_libraries(array_schema generic_tile_io baseline) conclude(object_library) add_test_subdirectory() diff --git a/tiledb/sm/query/ast/query_ast.cc b/tiledb/sm/query/ast/query_ast.cc index ddec36e555ea..eb77c708dc7b 100644 --- a/tiledb/sm/query/ast/query_ast.cc +++ b/tiledb/sm/query/ast/query_ast.cc @@ -31,6 +31,8 @@ */ #include "query_ast.h" +#include "tiledb/sm/array_schema/enumeration.h" +#include "tiledb/sm/misc/integral_type_casts.h" using namespace tiledb::common; @@ -60,10 +62,65 @@ void ASTNodeVal::get_field_names( field_name_set.insert(field_name_); } +void ASTNodeVal::get_enumeration_field_names( + std::unordered_set& field_name_set) const { + if (use_enumeration_) { + field_name_set.insert(field_name_); + } +} + bool ASTNodeVal::is_backwards_compatible() const { return true; } +void ASTNodeVal::rewrite_enumeration_conditions( + const ArraySchema& array_schema) { + // This is called by the Query class before applying a query condition. This + // works by looking up each related enumeration and translating the + // condition's value to reflect the underlying index value. I.e., if the + // query condition was created with `my_attr = "foo"`, and `my_attr` is an + // attribute with an enumeration, this will replace the condition's value + // with the index value returned by `Enumeration::index_of()`. + + if (!use_enumeration_) { + return; + } + + if (!array_schema.is_attr(field_name_)) { + return; + } + + auto attr = array_schema.attribute(field_name_); + auto enmr_name = attr->get_enumeration_name(); + if (!enmr_name.has_value()) { + return; + } + + auto enumeration = array_schema.get_enumeration(enmr_name.value()); + if (!enumeration) { + throw std::logic_error( + "Missing requried enumeration for field '" + field_name_ + "'"); + } + + if (!enumeration->ordered() && + (op_ != QueryConditionOp::EQ && op_ != QueryConditionOp::NE)) { + throw std::logic_error( + "Cannot apply an inequality operator against an unordered Enumeration"); + } + + auto idx = enumeration->index_of(condition_value_view_); + auto val_size = datatype_size(attr->type()); + + condition_value_data_ = ByteVecValue(val_size); + utils::safe_integral_cast_to_datatype( + idx, attr->type(), condition_value_data_); + + condition_value_view_ = + UntypedDatumView(condition_value_data_.data(), val_size); + + use_enumeration_ = false; +} + Status ASTNodeVal::check_node_validity(const ArraySchema& array_schema) const { const uint64_t condition_value_size = condition_value_data_.size(); @@ -186,6 +243,14 @@ const QueryConditionCombinationOp& ASTNodeVal::get_combination_op() const { "value node."); } +bool ASTNodeVal::use_enumeration() const { + return use_enumeration_; +} + +void ASTNodeVal::set_use_enumeration(bool use_enumeration) { + use_enumeration_ = use_enumeration; +} + bool ASTNodeExpr::is_expr() const { return true; } @@ -205,6 +270,13 @@ void ASTNodeExpr::get_field_names( } } +void ASTNodeExpr::get_enumeration_field_names( + std::unordered_set& field_name_set) const { + for (const auto& child : nodes_) { + child->get_enumeration_field_names(field_name_set); + } +} + bool ASTNodeExpr::is_backwards_compatible() const { if (combination_op_ != QueryConditionCombinationOp::AND) { return false; @@ -217,6 +289,13 @@ bool ASTNodeExpr::is_backwards_compatible() const { return true; } +void ASTNodeExpr::rewrite_enumeration_conditions( + const ArraySchema& array_schema) { + for (auto& child : nodes_) { + child->rewrite_enumeration_conditions(array_schema); + } +} + Status ASTNodeExpr::check_node_validity(const ArraySchema& array_schema) const { // If the node is a compound expression node, ensure there are at least // two children in the node and then run a check on each child node. @@ -275,9 +354,22 @@ const QueryConditionOp& ASTNodeExpr::get_op() const { const std::vector>& ASTNodeExpr::get_children() const { return nodes_; } + const QueryConditionCombinationOp& ASTNodeExpr::get_combination_op() const { return combination_op_; } +bool ASTNodeExpr::use_enumeration() const { + throw std::runtime_error( + "ASTNodeExpr::use_enumeration: Cannot get enumeration status from " + "an AST expression node."); +} + +void ASTNodeExpr::set_use_enumeration(bool use_enumeration) { + for (auto& child : nodes_) { + child->set_use_enumeration(use_enumeration); + } +} + } // namespace sm } // namespace tiledb diff --git a/tiledb/sm/query/ast/query_ast.h b/tiledb/sm/query/ast/query_ast.h index 30a08884d905..2a7f35e23898 100644 --- a/tiledb/sm/query/ast/query_ast.h +++ b/tiledb/sm/query/ast/query_ast.h @@ -91,11 +91,19 @@ class ASTNode { * @brief Gets the set of field names from all the value nodes in the ASTNode. * * @param field_name_set The set variable the function populates. - * @return std::unordered_set& Set of the field names in the - * node. */ virtual void get_field_names( std::unordered_set& field_name_set) const = 0; + + /** + * @brief Gets the set of field names from all the value nodes that reference + * an enumerated field. + * + * @param field_name_set The set variable the function populates. + */ + virtual void get_enumeration_field_names( + std::unordered_set& field_name_set) const = 0; + /** * @brief Returns true if the AST is previously supported by previous versions * of TileDB. This means that the AST should only have AND combination ops, @@ -106,6 +114,15 @@ class ASTNode { */ virtual bool is_backwards_compatible() const = 0; + /** + * @brief Update an node value condition values that refer to enumerated + * attributes. + * + * @param array_schema The array schema with all relevant enumerations loaded. + */ + virtual void rewrite_enumeration_conditions( + const ArraySchema& array_schema) = 0; + /** * @brief Checks whether the node is valid based on the array schema of the * array that the query condition that contains this AST node will execute a @@ -176,6 +193,26 @@ class ASTNode { */ virtual const QueryConditionCombinationOp& get_combination_op() const = 0; + /** + * @brief Return whether this node's condition should be applied against + * the attributes enumerated values or the underlying index data if + * applicable for a given attribute. + * + * @return bool If true, apply this condition against the enumerated values. + */ + virtual bool use_enumeration() const = 0; + + /** + * @brief By default, a query condition is applied against an enumeration's + * values. This can be disabled to apply a given condition against the + * underlying integer data stored for the attribute by passing `false` to + * this method. + * + * @param use_enumeration A bool indicating whether this condition should be + * applied against the enumerations values or not. + */ + virtual void set_use_enumeration(bool use_enumeration) = 0; + /** * @brief Default virtual destructor. */ @@ -196,12 +233,15 @@ class ASTNodeVal : public ASTNode { * @param condition_value_size The byte size of condition_value. * @param op The relational operation between the value of the field * and `condition_value`. + * @param use_enumeration Whether or not to use an associated + * enumeration at query time. */ ASTNodeVal( const std::string& field_name, const void* const condition_value, const uint64_t condition_value_size, - const QueryConditionOp op) + const QueryConditionOp op, + bool use_enumeration = true) : field_name_(field_name) , condition_value_data_(condition_value_size) , condition_value_view_( @@ -209,7 +249,8 @@ class ASTNodeVal : public ASTNode { (void*)"" : condition_value_data_.data()), condition_value_data_.size()) - , op_(op) { + , op_(op) + , use_enumeration_(use_enumeration) { if (condition_value_view_.size() != 0) { memcpy( condition_value_data_.data(), condition_value, condition_value_size); @@ -228,7 +269,8 @@ class ASTNodeVal : public ASTNode { (void*)"" : condition_value_data_.data()), condition_value_data_.size()) - , op_(rhs.op_) { + , op_(rhs.op_) + , use_enumeration_(rhs.use_enumeration_) { } /** @@ -243,7 +285,8 @@ class ASTNodeVal : public ASTNode { (void*)"" : condition_value_data_.data()), condition_value_data_.size()) - , op_(negate_query_condition_op(rhs.op_)) { + , op_(negate_query_condition_op(rhs.op_)) + , use_enumeration_(rhs.use_enumeration_) { } /** @@ -284,12 +327,19 @@ class ASTNodeVal : public ASTNode { * @brief Gets the set of field names from all the value nodes in the ASTNode. * * @param field_name_set The set variable the function populates. - * @return std::unordered_set& Set of the field names in the - * node. */ void get_field_names( std::unordered_set& field_name_set) const override; + /** + * @brief Gets the set of field names from all the value nodes that reference + * an enumerated field. + * + * @param field_name_set The set variable the function populates. + */ + void get_enumeration_field_names( + std::unordered_set& field_name_set) const override; + /** * @brief Returns true if the AST is previously supported by previous versions * of TileDB. This means that the AST should only have AND combination ops, @@ -300,6 +350,17 @@ class ASTNodeVal : public ASTNode { */ bool is_backwards_compatible() const override; + /** + * Update any node value condition values that refer to enumerated attributes. + * For any value condition on an attribute that has an Enumeration, the + * user specified value is passed to the attribute's enumeration `index_of` + * method to replace the user provided value with the Enumeration's value + * index. + * + * @param array_schema The array schema with all relevant enumerations loaded. + */ + void rewrite_enumeration_conditions(const ArraySchema& array_schema) override; + /** * @brief Checks whether the node is valid based on the array schema of the * array that the query condition that contains this AST node will execute a @@ -370,6 +431,26 @@ class ASTNodeVal : public ASTNode { */ const QueryConditionCombinationOp& get_combination_op() const override; + /** + * @brief Return whether this node's condition should be applied against + * the attributes enumerated values or the underlying index data if + * applicable for a given attribute. + * + * @return bool If true, apply this condition against the enumerated values. + */ + bool use_enumeration() const override; + + /** + * @brief By default, a query condition is applied against an enumeration's + * values. This can be disabled to apply a given condition against the + * underlying integer data stored for the attribute by passing `false` to + * this method. + * + * @param use_enumeration A bool indicating whether this condition should be + * applied against the enumerations values or not. + */ + void set_use_enumeration(bool use_enumeration) override; + private: /** The attribute name. */ std::string field_name_; @@ -382,6 +463,9 @@ class ASTNodeVal : public ASTNode { /** The comparison operator. */ QueryConditionOp op_; + + /** Whether this condiiton applies to enumerated values if applicable */ + bool use_enumeration_; }; /** @@ -461,12 +545,19 @@ class ASTNodeExpr : public ASTNode { * @brief Gets the set of field names from all the value nodes in the ASTNode. * * @param field_name_set The set variable the function populates. - * @return std::unordered_set& Set of the field names in the - * node. */ void get_field_names( std::unordered_set& field_name_set) const override; + /** + * @brief Gets the set of field names from all the value nodes that reference + * an enumerated field. + * + * @param field_name_set The set variable the function populates. + */ + void get_enumeration_field_names( + std::unordered_set& field_name_set) const override; + /** * @brief Returns true if the AST is previously supported by previous versions * of TileDB. This means that the AST should only have AND combination ops, @@ -477,6 +568,14 @@ class ASTNodeExpr : public ASTNode { */ bool is_backwards_compatible() const override; + /** + * @brief Update an node value condition values that refer to enumerated + * attributes. + * + * @param array_schema The array schema with all relevant enumerations loaded. + */ + void rewrite_enumeration_conditions(const ArraySchema& array_schema) override; + /** * @brief Checks whether the node is valid based on the array schema of the * array that the query condition that contains this AST node will execute a @@ -547,6 +646,31 @@ class ASTNodeExpr : public ASTNode { */ const QueryConditionCombinationOp& get_combination_op() const override; + /** + * @brief Return whether this node's condition should be applied against + * the attributes enumerated values or the underlying index data if + * applicable for a given attribute. + * + * This method always throws when called on an expression node. + * + * @return bool If true, apply this condition against the enumerated values. + */ + bool use_enumeration() const override; + + /** + * @brief By default, a query condition is applied against an enumeration's + * values. This can be disabled to apply a given condition against the + * underlying integer data stored for the attribute by passing `false` to + * this method. + * + * When called on an expression node this value is recursively applied + * against all value nodes in the AST. + * + * @param use_enumeration A bool indicating whether this condition should be + * applied against the enumerations values or not. + */ + void set_use_enumeration(bool use_enumeration) override; + private: /** The node list **/ std::vector> nodes_; @@ -558,4 +682,4 @@ class ASTNodeExpr : public ASTNode { } // namespace sm } // namespace tiledb -#endif // TILEDB_QUERY_AST_H \ No newline at end of file +#endif // TILEDB_QUERY_AST_H diff --git a/tiledb/sm/query/query.cc b/tiledb/sm/query/query.cc index c496e5244115..cc4bc0857675 100644 --- a/tiledb/sm/query/query.cc +++ b/tiledb/sm/query/query.cc @@ -795,6 +795,37 @@ Status Query::process() { } } + if (condition_.has_value()) { + auto& names = condition_->enumeration_field_names(); + std::unordered_set deduped_enmr_names; + for (auto name : names) { + auto attr = array_schema_->attribute(name); + if (attr == nullptr) { + continue; + } + auto enmr_name = attr->get_enumeration_name(); + if (enmr_name.has_value()) { + deduped_enmr_names.insert(enmr_name.value()); + } + } + std::vector enmr_names; + enmr_names.reserve(deduped_enmr_names.size()); + for (auto& enmr_name : deduped_enmr_names) { + enmr_names.emplace_back(enmr_name); + } + + throw_if_not_ok(parallel_for( + storage_manager_->compute_tp(), + 0, + enmr_names.size(), + [&](const uint64_t i) { + array_->get_enumeration(enmr_names[i]); + return Status::Ok(); + })); + + condition_->rewrite_enumeration_conditions(array_schema()); + } + // Update query status. status_ = QueryStatus::INPROGRESS; diff --git a/tiledb/sm/query/query_condition.cc b/tiledb/sm/query/query_condition.cc index a8c78c8ff409..b93f78e21af3 100644 --- a/tiledb/sm/query/query_condition.cc +++ b/tiledb/sm/query/query_condition.cc @@ -123,6 +123,15 @@ Status QueryCondition::init( return Status::Ok(); } +void QueryCondition::rewrite_enumeration_conditions( + const ArraySchema& array_schema) { + if (!tree_) { + return; + } + + tree_->rewrite_enumeration_conditions(array_schema); +} + Status QueryCondition::check(const ArraySchema& array_schema) const { if (!tree_) { return Status::Ok(); @@ -144,6 +153,7 @@ Status QueryCondition::combine( } combined_cond->field_names_.clear(); + combined_cond->enumeration_field_names_.clear(); combined_cond->tree_ = this->tree_->combine(rhs.tree_, combination_op); return Status::Ok(); } @@ -158,6 +168,7 @@ Status QueryCondition::negate( } combined_cond->field_names_.clear(); + combined_cond->enumeration_field_names_.clear(); combined_cond->tree_ = this->tree_->get_negated_tree(); return Status::Ok(); } @@ -174,6 +185,15 @@ std::unordered_set& QueryCondition::field_names() const { return field_names_; } +std::unordered_set& QueryCondition::enumeration_field_names() + const { + if (enumeration_field_names_.empty() && tree_ != nullptr) { + tree_->get_enumeration_field_names(enumeration_field_names_); + } + + return enumeration_field_names_; +} + uint64_t QueryCondition::condition_timestamp() const { if (condition_marker_.empty()) { return 0; @@ -188,6 +208,10 @@ uint64_t QueryCondition::condition_timestamp() const { return timestamps.first; } +void QueryCondition::set_use_enumeration(bool use_enumeration) { + tree_->set_use_enumeration(use_enumeration); +} + /** Full template specialization for `char*` and `QueryConditionOp::LT`. */ template <> struct QueryCondition::BinaryCmpNullChecks { diff --git a/tiledb/sm/query/query_condition.h b/tiledb/sm/query/query_condition.h index 717f047ceab0..b0c16154ce3d 100644 --- a/tiledb/sm/query/query_condition.h +++ b/tiledb/sm/query/query_condition.h @@ -112,6 +112,15 @@ class QueryCondition { uint64_t condition_value_size, const QueryConditionOp& op); + /** + * Translate any query conditions against enumerated attributes to the + * underlying attribute type. + * + * @param array_schema The current array schema with all required enumerations + * loaded. + */ + void rewrite_enumeration_conditions(const ArraySchema& array_schema); + /** * Verifies that the current state contains supported comparison * operations. Currently, we support the following: @@ -160,6 +169,12 @@ class QueryCondition { */ std::unordered_set& field_names() const; + /** + * Returns a set of all unique field names that reference an enumerated + * attribute condition in the AST representing the query condition. + */ + std::unordered_set& enumeration_field_names() const; + /** * Returns the timestamp for this condition. */ @@ -245,6 +260,17 @@ class QueryCondition { */ uint64_t condition_index() const; + /** + * By default, a query condition is applied against the enumerated values + * of an attribute. Setting use_enumeration to false prevents the translation + * and applies this query condition directly against the underlying integral + * attribute data. + * + * @param use_enumeration A bool indicating whether to use the enumeration + * values. + */ + void set_use_enumeration(bool use_enumeration); + private: /* ********************************* */ /* PRIVATE DATATYPES */ @@ -294,6 +320,9 @@ class QueryCondition { /** Caches all field names in the value nodes of the AST. */ mutable std::unordered_set field_names_; + /** Caches all field names that references enumerations in the AST. */ + mutable std::unordered_set enumeration_field_names_; + /* ********************************* */ /* PRIVATE METHODS */ /* ********************************* */ diff --git a/tiledb/sm/serialization/array_schema.cc b/tiledb/sm/serialization/array_schema.cc index 126cddb37d82..a9a57277a820 100644 --- a/tiledb/sm/serialization/array_schema.cc +++ b/tiledb/sm/serialization/array_schema.cc @@ -42,6 +42,7 @@ #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/array_schema/dimension_label.h" #include "tiledb/sm/array_schema/domain.h" +#include "tiledb/sm/array_schema/enumeration.h" #include "tiledb/sm/enums/array_type.h" #include "tiledb/sm/enums/compressor.h" #include "tiledb/sm/enums/data_order.h" @@ -352,6 +353,11 @@ void attribute_to_capnp( const auto& filters = attribute->filters(); auto filter_pipeline_builder = attribute_builder->initFilterPipeline(); throw_if_not_ok(filter_pipeline_to_capnp(&filters, &filter_pipeline_builder)); + + auto enmr_name = attribute->get_enumeration_name(); + if (enmr_name.has_value()) { + attribute_builder->setEnumerationName(enmr_name.value()); + } } shared_ptr attribute_from_capnp( @@ -399,6 +405,11 @@ shared_ptr attribute_from_capnp( datatype, attribute_reader.getCellValNum()); } + std::optional enmr_name; + if (attribute_reader.hasEnumerationName()) { + enmr_name = attribute_reader.getEnumerationName(); + } + return tiledb::common::make_shared( HERE(), attribute_reader.getName(), @@ -408,7 +419,8 @@ shared_ptr attribute_from_capnp( *(filters.get()), fill_value_vec, fill_value_validity, - data_order); + data_order, + enmr_name); } Status dimension_to_capnp( @@ -763,6 +775,61 @@ shared_ptr dimension_label_from_capnp( is_relative); } +void enumeration_to_capnp( + shared_ptr enumeration, + capnp::Enumeration::Builder& enmr_builder) { + enmr_builder.setName(enumeration->name()); + enmr_builder.setType(datatype_str(enumeration->type())); + enmr_builder.setCellValNum(enumeration->cell_val_num()); + enmr_builder.setOrdered(enumeration->ordered()); + + auto dspan = enumeration->data(); + enmr_builder.setData(::kj::arrayPtr(dspan.data(), dspan.size())); + + if (enumeration->var_size()) { + auto ospan = enumeration->offsets(); + enmr_builder.setOffsets(::kj::arrayPtr(ospan.data(), ospan.size())); + } +} + +shared_ptr enumeration_from_capnp( + const capnp::Enumeration::Reader& reader) { + auto name = reader.getName(); + auto path_name = reader.getPathName(); + Datatype datatype = Datatype::ANY; + throw_if_not_ok(datatype_enum(reader.getType(), &datatype)); + + if (!reader.hasData()) { + throw SerializationStatusException( + "[Deserialization::enumeration_from_capnp] Deserialization of " + "Enumeration is missing its data buffer."); + } + + auto data_reader = reader.getData().asBytes(); + auto data = data_reader.begin(); + auto data_size = data_reader.size(); + + const void* offsets = nullptr; + uint64_t offsets_size = 0; + + if (reader.hasOffsets()) { + auto offsets_reader = reader.getOffsets().asBytes(); + offsets = offsets_reader.begin(); + offsets_size = offsets_reader.size(); + } + + return Enumeration::create( + name, + path_name, + datatype, + reader.getCellValNum(), + reader.getOrdered(), + data, + data_size, + offsets, + offsets_size); +} + Status array_schema_to_capnp( const ArraySchema& array_schema, capnp::ArraySchema::Builder* array_schema_builder, @@ -809,9 +876,9 @@ Status array_schema_to_capnp( // Attributes const unsigned num_attrs = array_schema.attribute_num(); - auto attributes_buidler = array_schema_builder->initAttributes(num_attrs); + auto attribute_builders = array_schema_builder->initAttributes(num_attrs); for (size_t i = 0; i < num_attrs; i++) { - auto attribute_builder = attributes_buidler[i]; + auto attribute_builder = attribute_builders[i]; attribute_to_capnp(array_schema.attribute(i), &attribute_builder); } @@ -833,6 +900,27 @@ Status array_schema_to_capnp( } } + // Loaded enumerations + auto loaded_enmr_names = array_schema.get_loaded_enumeration_names(); + const unsigned num_loaded_enmrs = loaded_enmr_names.size(); + auto enmr_builders = array_schema_builder->initEnumerations(num_loaded_enmrs); + for (size_t i = 0; i < num_loaded_enmrs; i++) { + auto enmr = array_schema.get_enumeration(loaded_enmr_names[i]); + auto builder = enmr_builders[i]; + enumeration_to_capnp(enmr, builder); + } + + // Enumeration path map + auto enmr_names = array_schema.get_enumeration_names(); + const unsigned num_enmr_names = enmr_names.size(); + auto enmr_path_map_builders = + array_schema_builder->initEnumerationPathMap(num_enmr_names); + for (size_t i = 0; i < num_enmr_names; i++) { + auto enmr_path_name = array_schema.get_enumeration_path_name(enmr_names[i]); + enmr_path_map_builders[i].setKey(enmr_names[i]); + enmr_path_map_builders[i].setValue(enmr_path_name); + } + return Status::Ok(); } @@ -999,6 +1087,39 @@ ArraySchema array_schema_from_capnp( } } + // Loaded enumerations + std::vector> enumerations; + if (schema_reader.hasEnumerations()) { + auto enmr_readers = schema_reader.getEnumerations(); + enumerations.reserve(enmr_readers.size()); + try { + for (auto&& enmr_reader : enmr_readers) { + enumerations.emplace_back(enumeration_from_capnp(enmr_reader)); + } + } catch (const std::exception& e) { + std::throw_with_nested(std::runtime_error( + "[Deserialization::array_schema_from_capnp] Cannot deserialize " + "enumerations")); + } + } + + // Enumeration path map + std::unordered_map enmr_path_map; + if (schema_reader.hasEnumerationPathMap()) { + auto enmr_path_map_readers = schema_reader.getEnumerationPathMap(); + try { + for (auto&& kv_reader : enmr_path_map_readers) { + auto enmr_name = kv_reader.getKey(); + auto enmr_path_name = kv_reader.getValue(); + enmr_path_map[enmr_name] = enmr_path_name; + } + } catch (const std::exception& e) { + std::throw_with_nested(std::runtime_error( + "[Deserialization::array_schema_from_capnp] Cannot deserialize " + "enumeration path map")); + } + } + // Set the range if we have two values // #TODO Add security validation std::pair timestamp_range; @@ -1028,6 +1149,8 @@ ArraySchema array_schema_from_capnp( capacity, attributes, dimension_labels, + enumerations, + enmr_path_map, cell_var_offsets_filters, cell_validity_filters, coords_filters); diff --git a/tiledb/sm/serialization/array_schema.h b/tiledb/sm/serialization/array_schema.h index b926c3990a41..53b8be082c0a 100644 --- a/tiledb/sm/serialization/array_schema.h +++ b/tiledb/sm/serialization/array_schema.h @@ -119,6 +119,25 @@ void dimension_label_to_capnp( shared_ptr dimension_label_from_capnp( const capnp::DimensionLabel::Reader& reader); +/** + * Serialize an Enumeration do cap'n proto object + * + * @param enumeration Enumeration to serialize. + * @param enmr_builder Cap'n proto class. + */ +void enumeration_to_capnp( + shared_ptr enumeration, + capnp::Enumeration::Builder& enmr_builder); + +/** + * Deserialize a dimension label from a cap'n proto object + * + * @param reader Cap'n proto reader object + * @return A new Enumeration + */ +shared_ptr enumeration_from_capnp( + const capnp::Enumeration::Reader& reader); + #endif // TILEDB_SERIALIZATION /** diff --git a/tiledb/sm/serialization/array_schema_evolution.cc b/tiledb/sm/serialization/array_schema_evolution.cc index cb06262467c4..369b3c15f8f1 100644 --- a/tiledb/sm/serialization/array_schema_evolution.cc +++ b/tiledb/sm/serialization/array_schema_evolution.cc @@ -42,6 +42,7 @@ #include "tiledb/sm/array_schema/attribute.h" #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/array_schema/domain.h" +#include "tiledb/sm/array_schema/enumeration.h" #include "tiledb/sm/enums/array_type.h" #include "tiledb/sm/enums/compressor.h" #include "tiledb/sm/enums/datatype.h" @@ -102,6 +103,32 @@ Status array_schema_evolution_to_capnp( attribute_to_capnp(attr_to_add, &attribute_builder); } + auto enmr_names_to_add = array_schema_evolution->enumeration_names_to_add(); + auto num_enmrs = enmr_names_to_add.size(); + + if (num_enmrs > 0) { + auto enmrs_to_add_builder = + array_schema_evolution_builder->initEnumerationsToAdd(num_enmrs); + for (size_t i = 0; i < num_enmrs; i++) { + auto enmr = + array_schema_evolution->enumeration_to_add(enmr_names_to_add[i]); + auto builder = enmrs_to_add_builder[i]; + enumeration_to_capnp(enmr, builder); + } + } + + // Enumerations to drop + std::vector enmr_names_to_drop = + array_schema_evolution->enumeration_names_to_drop(); + + auto enumerations_to_drop_builder = + array_schema_evolution_builder->initEnumerationsToDrop( + enmr_names_to_drop.size()); + + for (size_t i = 0; i < enmr_names_to_drop.size(); i++) { + enumerations_to_drop_builder.set(i, enmr_names_to_drop[i]); + } + auto timestamp_builder = array_schema_evolution_builder->initTimestampRange(2); const auto& timestamp_range = array_schema_evolution->timestamp_range(); @@ -111,34 +138,55 @@ Status array_schema_evolution_to_capnp( return Status::Ok(); } -Status array_schema_evolution_from_capnp( - const capnp::ArraySchemaEvolution::Reader& evolution_reader, - tdb_unique_ptr* array_schema_evolution) { - array_schema_evolution->reset(tdb_new(ArraySchemaEvolution)); - // Set attributes to drop +tdb_unique_ptr array_schema_evolution_from_capnp( + const capnp::ArraySchemaEvolution::Reader& evolution_reader) { + // Create attributes to add + std::unordered_map> attrs_to_add; + auto attrs_to_add_reader = evolution_reader.getAttributesToAdd(); + for (auto attr_reader : attrs_to_add_reader) { + auto attr = attribute_from_capnp(attr_reader); + attrs_to_add[attr->name()] = attr; + } + + // Create attributes to drop + std::unordered_set attrs_to_drop; auto attributes_to_drop_reader = evolution_reader.getAttributesToDrop(); for (auto attr_reader : attributes_to_drop_reader) { std::string attr_name = std::string(attr_reader.cStr()); - RETURN_NOT_OK((*array_schema_evolution)->drop_attribute(attr_name)); + attrs_to_drop.insert(attr_name); } - // Set attributes to add - auto attributes_to_add_reader = evolution_reader.getAttributesToAdd(); - for (auto attr_reader : attributes_to_add_reader) { - auto attr = attribute_from_capnp(attr_reader); - RETURN_NOT_OK((*array_schema_evolution)->add_attribute(attr.get())); + // Create enumerations to add + std::unordered_map> enmrs_to_add; + auto enmrs_to_add_reader = evolution_reader.getEnumerationsToAdd(); + for (auto enmr_reader : enmrs_to_add_reader) { + auto enmr = enumeration_from_capnp(enmr_reader); + enmrs_to_add[enmr->name()] = enmr; } + // Create enumerations to drop + std::unordered_set enmrs_to_drop; + auto enmrs_to_drop_reader = evolution_reader.getEnumerationsToDrop(); + for (auto enmr_reader : enmrs_to_drop_reader) { + std::string enmr_name = std::string(enmr_reader.cStr()); + enmrs_to_drop.insert(enmr_name); + } + + std::pair ts_range; // Set the range if we have two values if (evolution_reader.hasTimestampRange() && evolution_reader.getTimestampRange().size() >= 2) { const auto& timestamp_range = evolution_reader.getTimestampRange(); - throw_if_not_ok((*array_schema_evolution) - ->set_timestamp_range(std::make_pair( - timestamp_range[0], timestamp_range[1]))); + ts_range = std::make_pair(timestamp_range[0], timestamp_range[1]); } - return Status::Ok(); + return tdb_unique_ptr(tdb_new( + ArraySchemaEvolution, + attrs_to_add, + attrs_to_drop, + enmrs_to_add, + enmrs_to_drop, + ts_range)); } Status array_schema_evolution_serialize( @@ -216,8 +264,8 @@ Status array_schema_evolution_deserialize( array_schema_evolution_builder); capnp::ArraySchemaEvolution::Reader array_schema_evolution_reader = array_schema_evolution_builder.asReader(); - RETURN_NOT_OK(array_schema_evolution_from_capnp( - array_schema_evolution_reader, &decoded_array_schema_evolution)); + decoded_array_schema_evolution = + array_schema_evolution_from_capnp(array_schema_evolution_reader); break; } case SerializationType::CAPNP: { @@ -228,8 +276,8 @@ Status array_schema_evolution_deserialize( serialized_buffer.size() / sizeof(::capnp::word))); capnp::ArraySchemaEvolution::Reader array_schema_evolution_reader = reader.getRoot(); - RETURN_NOT_OK(array_schema_evolution_from_capnp( - array_schema_evolution_reader, &decoded_array_schema_evolution)); + decoded_array_schema_evolution = + array_schema_evolution_from_capnp(array_schema_evolution_reader); break; } default: { diff --git a/tiledb/sm/serialization/posix/tiledb-rest.capnp.c++ b/tiledb/sm/serialization/posix/tiledb-rest.capnp.c++ index 08df30190b05..86e22aaddac3 100644 --- a/tiledb/sm/serialization/posix/tiledb-rest.capnp.c++ +++ b/tiledb/sm/serialization/posix/tiledb-rest.capnp.c++ @@ -635,17 +635,17 @@ const ::capnp::_::RawSchema s_facceeafd4472c68 = { 1, 2, i_facceeafd4472c68, nullptr, nullptr, { &s_facceeafd4472c68, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<274> b_d71de32f98e296fe = { +static const ::capnp::_::AlignedData<315> b_d71de32f98e296fe = { { 0, 0, 0, 0, 5, 0, 6, 0, 254, 150, 226, 152, 47, 227, 29, 215, 18, 0, 0, 0, 1, 0, 2, 0, 127, 216, 135, 181, 36, 146, 125, 181, - 13, 0, 7, 0, 0, 0, 0, 0, + 15, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 242, 0, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 0, 0, 0, 79, 3, 0, 0, + 29, 0, 0, 0, 191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, @@ -653,112 +653,126 @@ static const ::capnp::_::AlignedData<274> b_d71de32f98e296fe = { 112, 58, 65, 114, 114, 97, 121, 83, 99, 104, 101, 109, 97, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, - 60, 0, 0, 0, 3, 0, 4, 0, + 68, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 149, 1, 0, 0, 82, 0, 0, 0, + 205, 1, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 148, 1, 0, 0, 3, 0, 1, 0, - 160, 1, 0, 0, 2, 0, 1, 0, + 204, 1, 0, 0, 3, 0, 1, 0, + 216, 1, 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 157, 1, 0, 0, 90, 0, 0, 0, + 213, 1, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 156, 1, 0, 0, 3, 0, 1, 0, - 184, 1, 0, 0, 2, 0, 1, 0, + 212, 1, 0, 0, 3, 0, 1, 0, + 240, 1, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 181, 1, 0, 0, 74, 0, 0, 0, + 237, 1, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 180, 1, 0, 0, 3, 0, 1, 0, - 192, 1, 0, 0, 2, 0, 1, 0, + 236, 1, 0, 0, 3, 0, 1, 0, + 248, 1, 0, 0, 2, 0, 1, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 189, 1, 0, 0, 82, 0, 0, 0, + 245, 1, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 188, 1, 0, 0, 3, 0, 1, 0, - 200, 1, 0, 0, 2, 0, 1, 0, + 244, 1, 0, 0, 3, 0, 1, 0, + 0, 2, 0, 0, 2, 0, 1, 0, 4, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 197, 1, 0, 0, 170, 0, 0, 0, + 253, 1, 0, 0, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 200, 1, 0, 0, 3, 0, 1, 0, - 212, 1, 0, 0, 2, 0, 1, 0, + 0, 2, 0, 0, 3, 0, 1, 0, + 12, 2, 0, 0, 2, 0, 1, 0, 5, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 209, 1, 0, 0, 58, 0, 0, 0, + 9, 2, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 204, 1, 0, 0, 3, 0, 1, 0, - 216, 1, 0, 0, 2, 0, 1, 0, + 4, 2, 0, 0, 3, 0, 1, 0, + 16, 2, 0, 0, 2, 0, 1, 0, 6, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 213, 1, 0, 0, 170, 0, 0, 0, + 13, 2, 0, 0, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 216, 1, 0, 0, 3, 0, 1, 0, - 228, 1, 0, 0, 2, 0, 1, 0, + 16, 2, 0, 0, 3, 0, 1, 0, + 28, 2, 0, 0, 2, 0, 1, 0, 7, 0, 0, 0, 6, 0, 0, 0, 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 225, 1, 0, 0, 82, 0, 0, 0, + 25, 2, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 224, 1, 0, 0, 3, 0, 1, 0, - 236, 1, 0, 0, 2, 0, 1, 0, + 24, 2, 0, 0, 3, 0, 1, 0, + 36, 2, 0, 0, 2, 0, 1, 0, 8, 0, 0, 0, 7, 0, 0, 0, 0, 0, 1, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 233, 1, 0, 0, 34, 0, 0, 0, + 33, 2, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 228, 1, 0, 0, 3, 0, 1, 0, - 240, 1, 0, 0, 2, 0, 1, 0, + 28, 2, 0, 0, 3, 0, 1, 0, + 40, 2, 0, 0, 2, 0, 1, 0, 9, 0, 0, 0, 8, 0, 0, 0, 0, 0, 1, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 237, 1, 0, 0, 66, 0, 0, 0, + 37, 2, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 232, 1, 0, 0, 3, 0, 1, 0, - 4, 2, 0, 0, 2, 0, 1, 0, + 32, 2, 0, 0, 3, 0, 1, 0, + 60, 2, 0, 0, 2, 0, 1, 0, 10, 0, 0, 0, 64, 0, 0, 0, 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 2, 0, 0, 138, 0, 0, 0, + 57, 2, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 4, 2, 0, 0, 3, 0, 1, 0, - 16, 2, 0, 0, 2, 0, 1, 0, + 60, 2, 0, 0, 3, 0, 1, 0, + 72, 2, 0, 0, 2, 0, 1, 0, 11, 0, 0, 0, 9, 0, 0, 0, 0, 0, 1, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 13, 2, 0, 0, 186, 0, 0, 0, + 69, 2, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 16, 2, 0, 0, 3, 0, 1, 0, - 28, 2, 0, 0, 2, 0, 1, 0, + 72, 2, 0, 0, 3, 0, 1, 0, + 84, 2, 0, 0, 2, 0, 1, 0, 12, 0, 0, 0, 10, 0, 0, 0, 0, 0, 1, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 25, 2, 0, 0, 42, 0, 0, 0, + 81, 2, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 20, 2, 0, 0, 3, 0, 1, 0, - 32, 2, 0, 0, 2, 0, 1, 0, + 76, 2, 0, 0, 3, 0, 1, 0, + 88, 2, 0, 0, 2, 0, 1, 0, 13, 0, 0, 0, 11, 0, 0, 0, 0, 0, 1, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 2, 0, 0, 122, 0, 0, 0, + 85, 2, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 28, 2, 0, 0, 3, 0, 1, 0, - 56, 2, 0, 0, 2, 0, 1, 0, + 84, 2, 0, 0, 3, 0, 1, 0, + 112, 2, 0, 0, 2, 0, 1, 0, 14, 0, 0, 0, 12, 0, 0, 0, 0, 0, 1, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 53, 2, 0, 0, 130, 0, 0, 0, + 109, 2, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 52, 2, 0, 0, 3, 0, 1, 0, - 80, 2, 0, 0, 2, 0, 1, 0, + 108, 2, 0, 0, 3, 0, 1, 0, + 136, 2, 0, 0, 2, 0, 1, 0, + 15, 0, 0, 0, 13, 0, 0, 0, + 0, 0, 1, 0, 15, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 133, 2, 0, 0, 106, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 132, 2, 0, 0, 3, 0, 1, 0, + 160, 2, 0, 0, 2, 0, 1, 0, + 16, 0, 0, 0, 14, 0, 0, 0, + 0, 0, 1, 0, 16, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 157, 2, 0, 0, 154, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 160, 2, 0, 0, 3, 0, 1, 0, + 188, 2, 0, 0, 2, 0, 1, 0, 97, 114, 114, 97, 121, 84, 121, 112, 101, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, @@ -907,6 +921,33 @@ static const ::capnp::_::AlignedData<274> b_d71de32f98e296fe = { 222, 209, 12, 209, 98, 141, 255, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 110, 117, 109, 101, 114, 97, 116, + 105, 111, 110, 115, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 180, 185, 33, 204, 25, 47, 11, 208, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 110, 117, 109, 101, 114, 97, 116, + 105, 111, 110, 80, 97, 116, 104, 77, + 97, 112, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 151, 188, 17, 242, 43, 223, 218, 227, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, } @@ -917,13 +958,15 @@ static const ::capnp::_::RawSchema* const d_d71de32f98e296fe[] = { &s_92ad78f56de3d76a, &s_bc4583f733eac4f5, &s_ceff8d62d10cd1de, + &s_d00b2f19cc21b9b4, &s_de030f447664754c, + &s_e3dadf2bf211bc97, }; -static const uint16_t m_d71de32f98e296fe[] = {10, 0, 1, 2, 3, 4, 14, 5, 12, 6, 7, 13, 8, 11, 9}; -static const uint16_t i_d71de32f98e296fe[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}; +static const uint16_t m_d71de32f98e296fe[] = {10, 0, 1, 2, 3, 4, 14, 5, 16, 15, 12, 6, 7, 13, 8, 11, 9}; +static const uint16_t i_d71de32f98e296fe[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; const ::capnp::_::RawSchema s_d71de32f98e296fe = { - 0xd71de32f98e296fe, b_d71de32f98e296fe.words, 274, d_d71de32f98e296fe, m_d71de32f98e296fe, - 4, 15, i_d71de32f98e296fe, nullptr, nullptr, { &s_d71de32f98e296fe, nullptr, nullptr, 0, 0, nullptr } + 0xd71de32f98e296fe, b_d71de32f98e296fe.words, 315, d_d71de32f98e296fe, m_d71de32f98e296fe, + 6, 17, i_d71de32f98e296fe, nullptr, nullptr, { &s_d71de32f98e296fe, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE static const ::capnp::_::AlignedData<174> b_ceff8d62d10cd1de = { @@ -1114,17 +1157,17 @@ const ::capnp::_::RawSchema s_ceff8d62d10cd1de = { 1, 10, i_ceff8d62d10cd1de, nullptr, nullptr, { &s_ceff8d62d10cd1de, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<80> b_a1b81d67548230d4 = { +static const ::capnp::_::AlignedData<122> b_a1b81d67548230d4 = { { 0, 0, 0, 0, 5, 0, 6, 0, 212, 48, 130, 84, 103, 29, 184, 161, 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, - 3, 0, 7, 0, 0, 0, 0, 0, + 5, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 58, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 0, 0, 0, 175, 0, 0, 0, + 33, 0, 0, 0, 31, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, @@ -1133,28 +1176,42 @@ static const ::capnp::_::AlignedData<80> b_a1b81d67548230d4 = { 99, 104, 101, 109, 97, 69, 118, 111, 108, 117, 116, 105, 111, 110, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, - 12, 0, 0, 0, 3, 0, 4, 0, + 20, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 69, 0, 0, 0, 138, 0, 0, 0, + 125, 0, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 72, 0, 0, 0, 3, 0, 1, 0, - 100, 0, 0, 0, 2, 0, 1, 0, + 128, 0, 0, 0, 3, 0, 1, 0, + 156, 0, 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 97, 0, 0, 0, 130, 0, 0, 0, + 153, 0, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 96, 0, 0, 0, 3, 0, 1, 0, - 124, 0, 0, 0, 2, 0, 1, 0, + 152, 0, 0, 0, 3, 0, 1, 0, + 180, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 121, 0, 0, 0, 122, 0, 0, 0, + 177, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 120, 0, 0, 0, 3, 0, 1, 0, - 148, 0, 0, 0, 2, 0, 1, 0, + 176, 0, 0, 0, 3, 0, 1, 0, + 204, 0, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 201, 0, 0, 0, 146, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 204, 0, 0, 0, 3, 0, 1, 0, + 232, 0, 0, 0, 2, 0, 1, 0, + 4, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 229, 0, 0, 0, 154, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 232, 0, 0, 0, 3, 0, 1, 0, + 4, 1, 0, 0, 2, 0, 1, 0, 97, 116, 116, 114, 105, 98, 117, 116, 101, 115, 84, 111, 68, 114, 111, 112, 0, 0, 0, 0, 0, 0, 0, 0, @@ -1192,6 +1249,34 @@ static const ::capnp::_::AlignedData<80> b_a1b81d67548230d4 = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 110, 117, 109, 101, 114, 97, 116, + 105, 111, 110, 115, 84, 111, 65, 100, + 100, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 180, 185, 33, 204, 25, 47, 11, 208, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 110, 117, 109, 101, 114, 97, 116, + 105, 111, 110, 115, 84, 111, 68, 114, + 111, 112, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, } @@ -1200,25 +1285,26 @@ static const ::capnp::_::AlignedData<80> b_a1b81d67548230d4 = { #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_a1b81d67548230d4[] = { &s_92ad78f56de3d76a, + &s_d00b2f19cc21b9b4, }; -static const uint16_t m_a1b81d67548230d4[] = {1, 0, 2}; -static const uint16_t i_a1b81d67548230d4[] = {0, 1, 2}; +static const uint16_t m_a1b81d67548230d4[] = {1, 0, 3, 4, 2}; +static const uint16_t i_a1b81d67548230d4[] = {0, 1, 2, 3, 4}; const ::capnp::_::RawSchema s_a1b81d67548230d4 = { - 0xa1b81d67548230d4, b_a1b81d67548230d4.words, 80, d_a1b81d67548230d4, m_a1b81d67548230d4, - 1, 3, i_a1b81d67548230d4, nullptr, nullptr, { &s_a1b81d67548230d4, nullptr, nullptr, 0, 0, nullptr } + 0xa1b81d67548230d4, b_a1b81d67548230d4.words, 122, d_a1b81d67548230d4, m_a1b81d67548230d4, + 2, 5, i_a1b81d67548230d4, nullptr, nullptr, { &s_a1b81d67548230d4, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<144> b_92ad78f56de3d76a = { +static const ::capnp::_::AlignedData<160> b_92ad78f56de3d76a = { { 0, 0, 0, 0, 5, 0, 6, 0, 106, 215, 227, 109, 245, 120, 173, 146, 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, - 5, 0, 7, 0, 0, 0, 0, 0, + 6, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 226, 0, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 0, 0, 0, 199, 1, 0, 0, + 29, 0, 0, 0, 255, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, @@ -1226,63 +1312,70 @@ static const ::capnp::_::AlignedData<144> b_92ad78f56de3d76a = { 112, 58, 65, 116, 116, 114, 105, 98, 117, 116, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, - 32, 0, 0, 0, 3, 0, 4, 0, + 36, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 209, 0, 0, 0, 90, 0, 0, 0, + 237, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 208, 0, 0, 0, 3, 0, 1, 0, - 220, 0, 0, 0, 2, 0, 1, 0, + 236, 0, 0, 0, 3, 0, 1, 0, + 248, 0, 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 217, 0, 0, 0, 42, 0, 0, 0, + 245, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 212, 0, 0, 0, 3, 0, 1, 0, - 224, 0, 0, 0, 2, 0, 1, 0, + 240, 0, 0, 0, 3, 0, 1, 0, + 252, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 221, 0, 0, 0, 42, 0, 0, 0, + 249, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 216, 0, 0, 0, 3, 0, 1, 0, - 228, 0, 0, 0, 2, 0, 1, 0, + 244, 0, 0, 0, 3, 0, 1, 0, + 0, 1, 0, 0, 2, 0, 1, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 225, 0, 0, 0, 122, 0, 0, 0, + 253, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 224, 0, 0, 0, 3, 0, 1, 0, - 236, 0, 0, 0, 2, 0, 1, 0, + 252, 0, 0, 0, 3, 0, 1, 0, + 8, 1, 0, 0, 2, 0, 1, 0, 4, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 233, 0, 0, 0, 82, 0, 0, 0, + 5, 1, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 232, 0, 0, 0, 3, 0, 1, 0, - 244, 0, 0, 0, 2, 0, 1, 0, + 4, 1, 0, 0, 3, 0, 1, 0, + 16, 1, 0, 0, 2, 0, 1, 0, 5, 0, 0, 0, 32, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 241, 0, 0, 0, 74, 0, 0, 0, + 13, 1, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 240, 0, 0, 0, 3, 0, 1, 0, - 252, 0, 0, 0, 2, 0, 1, 0, + 12, 1, 0, 0, 3, 0, 1, 0, + 24, 1, 0, 0, 2, 0, 1, 0, 6, 0, 0, 0, 33, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 249, 0, 0, 0, 146, 0, 0, 0, + 21, 1, 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 252, 0, 0, 0, 3, 0, 1, 0, - 8, 1, 0, 0, 2, 0, 1, 0, + 24, 1, 0, 0, 3, 0, 1, 0, + 36, 1, 0, 0, 2, 0, 1, 0, 7, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 5, 1, 0, 0, 50, 0, 0, 0, + 33, 1, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1, 0, 0, 3, 0, 1, 0, - 12, 1, 0, 0, 2, 0, 1, 0, + 28, 1, 0, 0, 3, 0, 1, 0, + 40, 1, 0, 0, 2, 0, 1, 0, + 8, 0, 0, 0, 5, 0, 0, 0, + 0, 0, 1, 0, 8, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 37, 1, 0, 0, 130, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 36, 1, 0, 0, 3, 0, 1, 0, + 48, 1, 0, 0, 2, 0, 1, 0, 99, 101, 108, 108, 86, 97, 108, 78, 117, 109, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, @@ -1346,6 +1439,15 @@ static const ::capnp::_::AlignedData<144> b_92ad78f56de3d76a = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 114, 100, 101, 114, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 110, 117, 109, 101, 114, 97, 116, + 105, 111, 110, 78, 97, 109, 101, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -1359,11 +1461,147 @@ static const ::capnp::_::AlignedData<144> b_92ad78f56de3d76a = { static const ::capnp::_::RawSchema* const d_92ad78f56de3d76a[] = { &s_bc4583f733eac4f5, }; -static const uint16_t m_92ad78f56de3d76a[] = {0, 4, 6, 3, 1, 5, 7, 2}; -static const uint16_t i_92ad78f56de3d76a[] = {0, 1, 2, 3, 4, 5, 6, 7}; +static const uint16_t m_92ad78f56de3d76a[] = {0, 8, 4, 6, 3, 1, 5, 7, 2}; +static const uint16_t i_92ad78f56de3d76a[] = {0, 1, 2, 3, 4, 5, 6, 7, 8}; const ::capnp::_::RawSchema s_92ad78f56de3d76a = { - 0x92ad78f56de3d76a, b_92ad78f56de3d76a.words, 144, d_92ad78f56de3d76a, m_92ad78f56de3d76a, - 1, 8, i_92ad78f56de3d76a, nullptr, nullptr, { &s_92ad78f56de3d76a, nullptr, nullptr, 0, 0, nullptr } + 0x92ad78f56de3d76a, b_92ad78f56de3d76a.words, 160, d_92ad78f56de3d76a, m_92ad78f56de3d76a, + 1, 9, i_92ad78f56de3d76a, nullptr, nullptr, { &s_92ad78f56de3d76a, nullptr, nullptr, 0, 0, nullptr } +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<125> b_d00b2f19cc21b9b4 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 180, 185, 33, 204, 25, 47, 11, 208, + 18, 0, 0, 0, 1, 0, 1, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 5, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 242, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 29, 0, 0, 0, 143, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 69, 110, 117, 109, 101, 114, + 97, 116, 105, 111, 110, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 28, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 181, 0, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 176, 0, 0, 0, 3, 0, 1, 0, + 188, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 185, 0, 0, 0, 74, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 184, 0, 0, 0, 3, 0, 1, 0, + 196, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 193, 0, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 188, 0, 0, 0, 3, 0, 1, 0, + 200, 0, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 197, 0, 0, 0, 90, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 196, 0, 0, 0, 3, 0, 1, 0, + 208, 0, 0, 0, 2, 0, 1, 0, + 4, 0, 0, 0, 32, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 205, 0, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 200, 0, 0, 0, 3, 0, 1, 0, + 212, 0, 0, 0, 2, 0, 1, 0, + 5, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 5, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 209, 0, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 204, 0, 0, 0, 3, 0, 1, 0, + 216, 0, 0, 0, 2, 0, 1, 0, + 6, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 6, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 213, 0, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 208, 0, 0, 0, 3, 0, 1, 0, + 220, 0, 0, 0, 2, 0, 1, 0, + 110, 97, 109, 101, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 112, 97, 116, 104, 78, 97, 109, 101, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 121, 112, 101, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 101, 108, 108, 86, 97, 108, 78, + 117, 109, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 111, 114, 100, 101, 114, 101, 100, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 100, 97, 116, 97, 0, 0, 0, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 111, 102, 102, 115, 101, 116, 115, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; +::capnp::word const* const bp_d00b2f19cc21b9b4 = b_d00b2f19cc21b9b4.words; +#if !CAPNP_LITE +static const uint16_t m_d00b2f19cc21b9b4[] = {3, 5, 0, 6, 4, 1, 2}; +static const uint16_t i_d00b2f19cc21b9b4[] = {0, 1, 2, 3, 4, 5, 6}; +const ::capnp::_::RawSchema s_d00b2f19cc21b9b4 = { + 0xd00b2f19cc21b9b4, b_d00b2f19cc21b9b4.words, 125, nullptr, m_d00b2f19cc21b9b4, + 0, 7, i_d00b2f19cc21b9b4, nullptr, nullptr, { &s_d00b2f19cc21b9b4, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE static const ::capnp::_::AlignedData<143> b_d20a578112fa92a2 = { @@ -4271,17 +4509,17 @@ const ::capnp::_::RawSchema s_cbe1e7c13508aa2c = { 1, 4, i_cbe1e7c13508aa2c, nullptr, nullptr, { &s_cbe1e7c13508aa2c, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<65> b_dac6a7f675c57409 = { +static const ::capnp::_::AlignedData<81> b_dac6a7f675c57409 = { { 0, 0, 0, 0, 5, 0, 6, 0, 9, 116, 197, 117, 246, 167, 198, 218, - 18, 0, 0, 0, 1, 0, 0, 0, + 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 18, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 0, 0, 0, 175, 0, 0, 0, + 33, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, @@ -4290,28 +4528,35 @@ static const ::capnp::_::AlignedData<65> b_dac6a7f675c57409 = { 105, 111, 110, 67, 108, 97, 117, 115, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, - 12, 0, 0, 0, 3, 0, 4, 0, + 16, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 69, 0, 0, 0, 82, 0, 0, 0, + 97, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 68, 0, 0, 0, 3, 0, 1, 0, - 80, 0, 0, 0, 2, 0, 1, 0, + 96, 0, 0, 0, 3, 0, 1, 0, + 108, 0, 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 77, 0, 0, 0, 50, 0, 0, 0, + 105, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 72, 0, 0, 0, 3, 0, 1, 0, - 84, 0, 0, 0, 2, 0, 1, 0, + 100, 0, 0, 0, 3, 0, 1, 0, + 112, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 81, 0, 0, 0, 26, 0, 0, 0, + 109, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 76, 0, 0, 0, 3, 0, 1, 0, - 88, 0, 0, 0, 2, 0, 1, 0, + 104, 0, 0, 0, 3, 0, 1, 0, + 116, 0, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 113, 0, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 112, 0, 0, 0, 3, 0, 1, 0, + 124, 0, 0, 0, 2, 0, 1, 0, 102, 105, 101, 108, 100, 78, 97, 109, 101, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, @@ -4335,19 +4580,28 @@ static const ::capnp::_::AlignedData<65> b_dac6a7f675c57409 = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 117, 115, 101, 69, 110, 117, 109, 101, + 114, 97, 116, 105, 111, 110, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, } }; ::capnp::word const* const bp_dac6a7f675c57409 = b_dac6a7f675c57409.words; #if !CAPNP_LITE -static const uint16_t m_dac6a7f675c57409[] = {0, 2, 1}; -static const uint16_t i_dac6a7f675c57409[] = {0, 1, 2}; +static const uint16_t m_dac6a7f675c57409[] = {0, 2, 3, 1}; +static const uint16_t i_dac6a7f675c57409[] = {0, 1, 2, 3}; const ::capnp::_::RawSchema s_dac6a7f675c57409 = { - 0xdac6a7f675c57409, b_dac6a7f675c57409.words, 65, nullptr, m_dac6a7f675c57409, - 0, 3, i_dac6a7f675c57409, nullptr, nullptr, { &s_dac6a7f675c57409, nullptr, nullptr, 0, 0, nullptr } + 0xdac6a7f675c57409, b_dac6a7f675c57409.words, 81, nullptr, m_dac6a7f675c57409, + 0, 4, i_dac6a7f675c57409, nullptr, nullptr, { &s_dac6a7f675c57409, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<116> b_afc739d5c01e6496 = { +static const ::capnp::_::AlignedData<132> b_afc739d5c01e6496 = { { 0, 0, 0, 0, 5, 0, 6, 0, 150, 100, 30, 192, 213, 57, 199, 175, 18, 0, 0, 0, 1, 0, 1, 0, @@ -4357,7 +4611,7 @@ static const ::capnp::_::AlignedData<116> b_afc739d5c01e6496 = { 21, 0, 0, 0, 210, 0, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 0, 0, 0, 87, 1, 0, 0, + 29, 0, 0, 0, 143, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, @@ -4365,49 +4619,56 @@ static const ::capnp::_::AlignedData<116> b_afc739d5c01e6496 = { 112, 58, 65, 83, 84, 78, 111, 100, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, - 24, 0, 0, 0, 3, 0, 4, 0, + 28, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 153, 0, 0, 0, 106, 0, 0, 0, + 181, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 152, 0, 0, 0, 3, 0, 1, 0, - 164, 0, 0, 0, 2, 0, 1, 0, + 180, 0, 0, 0, 3, 0, 1, 0, + 192, 0, 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 161, 0, 0, 0, 82, 0, 0, 0, + 189, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 160, 0, 0, 0, 3, 0, 1, 0, - 172, 0, 0, 0, 2, 0, 1, 0, + 188, 0, 0, 0, 3, 0, 1, 0, + 200, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 169, 0, 0, 0, 50, 0, 0, 0, + 197, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 164, 0, 0, 0, 3, 0, 1, 0, - 176, 0, 0, 0, 2, 0, 1, 0, + 192, 0, 0, 0, 3, 0, 1, 0, + 204, 0, 0, 0, 2, 0, 1, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 173, 0, 0, 0, 26, 0, 0, 0, + 201, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 168, 0, 0, 0, 3, 0, 1, 0, - 180, 0, 0, 0, 2, 0, 1, 0, + 196, 0, 0, 0, 3, 0, 1, 0, + 208, 0, 0, 0, 2, 0, 1, 0, 4, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 177, 0, 0, 0, 74, 0, 0, 0, + 205, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 176, 0, 0, 0, 3, 0, 1, 0, - 204, 0, 0, 0, 2, 0, 1, 0, + 204, 0, 0, 0, 3, 0, 1, 0, + 232, 0, 0, 0, 2, 0, 1, 0, 5, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 201, 0, 0, 0, 114, 0, 0, 0, + 229, 0, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 200, 0, 0, 0, 3, 0, 1, 0, - 212, 0, 0, 0, 2, 0, 1, 0, + 228, 0, 0, 0, 3, 0, 1, 0, + 240, 0, 0, 0, 2, 0, 1, 0, + 6, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 6, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 237, 0, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 236, 0, 0, 0, 3, 0, 1, 0, + 248, 0, 0, 0, 2, 0, 1, 0, 105, 115, 69, 120, 112, 114, 101, 115, 115, 105, 111, 110, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, @@ -4462,6 +4723,15 @@ static const ::capnp::_::AlignedData<116> b_afc739d5c01e6496 = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 117, 115, 101, 69, 110, 117, 109, 101, + 114, 97, 116, 105, 111, 110, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, } }; @@ -4470,11 +4740,11 @@ static const ::capnp::_::AlignedData<116> b_afc739d5c01e6496 = { static const ::capnp::_::RawSchema* const d_afc739d5c01e6496[] = { &s_afc739d5c01e6496, }; -static const uint16_t m_afc739d5c01e6496[] = {4, 5, 1, 0, 3, 2}; -static const uint16_t i_afc739d5c01e6496[] = {0, 1, 2, 3, 4, 5}; +static const uint16_t m_afc739d5c01e6496[] = {4, 5, 1, 0, 3, 6, 2}; +static const uint16_t i_afc739d5c01e6496[] = {0, 1, 2, 3, 4, 5, 6}; const ::capnp::_::RawSchema s_afc739d5c01e6496 = { - 0xafc739d5c01e6496, b_afc739d5c01e6496.words, 116, d_afc739d5c01e6496, m_afc739d5c01e6496, - 1, 6, i_afc739d5c01e6496, nullptr, nullptr, { &s_afc739d5c01e6496, nullptr, nullptr, 0, 0, nullptr } + 0xafc739d5c01e6496, b_afc739d5c01e6496.words, 132, d_afc739d5c01e6496, m_afc739d5c01e6496, + 1, 7, i_afc739d5c01e6496, nullptr, nullptr, { &s_afc739d5c01e6496, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE static const ::capnp::_::AlignedData<73> b_eaf57cb9871fc06f = { @@ -9010,6 +9280,14 @@ constexpr ::capnp::Kind Attribute::_capnpPrivate::kind; constexpr ::capnp::_::RawSchema const* Attribute::_capnpPrivate::schema; #endif // !CAPNP_LITE +// Enumeration +constexpr uint16_t Enumeration::_capnpPrivate::dataWordSize; +constexpr uint16_t Enumeration::_capnpPrivate::pointerCount; +#if !CAPNP_LITE +constexpr ::capnp::Kind Enumeration::_capnpPrivate::kind; +constexpr ::capnp::_::RawSchema const* Enumeration::_capnpPrivate::schema; +#endif // !CAPNP_LITE + // AttributeBufferHeader constexpr uint16_t AttributeBufferHeader::_capnpPrivate::dataWordSize; constexpr uint16_t AttributeBufferHeader::_capnpPrivate::pointerCount; diff --git a/tiledb/sm/serialization/posix/tiledb-rest.capnp.h b/tiledb/sm/serialization/posix/tiledb-rest.capnp.h index 260cbaa58fb9..05ccbc0a6b37 100644 --- a/tiledb/sm/serialization/posix/tiledb-rest.capnp.h +++ b/tiledb/sm/serialization/posix/tiledb-rest.capnp.h @@ -23,6 +23,7 @@ CAPNP_DECLARE_SCHEMA(d71de32f98e296fe); CAPNP_DECLARE_SCHEMA(ceff8d62d10cd1de); CAPNP_DECLARE_SCHEMA(a1b81d67548230d4); CAPNP_DECLARE_SCHEMA(92ad78f56de3d76a); +CAPNP_DECLARE_SCHEMA(d00b2f19cc21b9b4); CAPNP_DECLARE_SCHEMA(d20a578112fa92a2); CAPNP_DECLARE_SCHEMA(95e26a84d32d8223); CAPNP_DECLARE_SCHEMA(a2a652536db09fa0); @@ -196,7 +197,7 @@ struct ArraySchema { class Pipeline; struct _capnpPrivate { - CAPNP_DECLARE_STRUCT_HEADER(d71de32f98e296fe, 2, 13) + CAPNP_DECLARE_STRUCT_HEADER(d71de32f98e296fe, 2, 15) #if !CAPNP_LITE static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; @@ -230,7 +231,7 @@ struct ArraySchemaEvolution { class Pipeline; struct _capnpPrivate { - CAPNP_DECLARE_STRUCT_HEADER(a1b81d67548230d4, 0, 3) + CAPNP_DECLARE_STRUCT_HEADER(a1b81d67548230d4, 0, 5) #if !CAPNP_LITE static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; @@ -247,7 +248,24 @@ struct Attribute { class Pipeline; struct _capnpPrivate { - CAPNP_DECLARE_STRUCT_HEADER(92ad78f56de3d76a, 1, 5) + CAPNP_DECLARE_STRUCT_HEADER(92ad78f56de3d76a, 1, 6) +#if !CAPNP_LITE + static constexpr ::capnp::_::RawBrandedSchema const* brand() { + return &schema->defaultBrand; + } +#endif // !CAPNP_LITE + }; +}; + +struct Enumeration { + Enumeration() = delete; + + class Reader; + class Builder; + class Pipeline; + + struct _capnpPrivate { + CAPNP_DECLARE_STRUCT_HEADER(d00b2f19cc21b9b4, 1, 5) #if !CAPNP_LITE static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; @@ -820,7 +838,7 @@ struct ConditionClause { class Pipeline; struct _capnpPrivate { - CAPNP_DECLARE_STRUCT_HEADER(dac6a7f675c57409, 0, 3) + CAPNP_DECLARE_STRUCT_HEADER(dac6a7f675c57409, 1, 3) #if !CAPNP_LITE static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; @@ -2441,6 +2459,18 @@ class ArraySchema::Reader { ::capnp::Kind::STRUCT>::Reader getDimensionLabels() const; + inline bool hasEnumerations() const; + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader + getEnumerations() const; + + inline bool hasEnumerationPathMap() const; + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>::Reader + getEnumerationPathMap() const; + private: ::capnp::_::StructReader _reader; template @@ -2640,6 +2670,48 @@ class ArraySchema::Builder { ::capnp::Kind::STRUCT>> disownDimensionLabels(); + inline bool hasEnumerations(); + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder + getEnumerations(); + inline void setEnumerations(::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader value); + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder + initEnumerations(unsigned int size); + inline void adoptEnumerations( + ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>&& value); + inline ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>> + disownEnumerations(); + + inline bool hasEnumerationPathMap(); + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>::Builder + getEnumerationPathMap(); + inline void setEnumerationPathMap(::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>::Reader value); + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>::Builder + initEnumerationPathMap(unsigned int size); + inline void adoptEnumerationPathMap( + ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>>&& value); + inline ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>> + disownEnumerationPathMap(); + private: ::capnp::_::StructBuilder _builder; template @@ -2883,6 +2955,16 @@ class ArraySchemaEvolution::Reader { inline ::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>::Reader getTimestampRange() const; + inline bool hasEnumerationsToAdd() const; + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader + getEnumerationsToAdd() const; + + inline bool hasEnumerationsToDrop() const; + inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Reader + getEnumerationsToDrop() const; + private: ::capnp::_::StructReader _reader; template @@ -2972,6 +3054,43 @@ class ArraySchemaEvolution::Builder { inline ::capnp::Orphan<::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>> disownTimestampRange(); + inline bool hasEnumerationsToAdd(); + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder + getEnumerationsToAdd(); + inline void setEnumerationsToAdd( + ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader value); + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder + initEnumerationsToAdd(unsigned int size); + inline void adoptEnumerationsToAdd( + ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>&& value); + inline ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>> + disownEnumerationsToAdd(); + + inline bool hasEnumerationsToDrop(); + inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Builder + getEnumerationsToDrop(); + inline void setEnumerationsToDrop( + ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Reader value); + inline void setEnumerationsToDrop( + ::kj::ArrayPtr value); + inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Builder + initEnumerationsToDrop(unsigned int size); + inline void adoptEnumerationsToDrop( + ::capnp::Orphan<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>&& + value); + inline ::capnp::Orphan<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>> + disownEnumerationsToDrop(); + private: ::capnp::_::StructBuilder _builder; template @@ -3042,6 +3161,9 @@ class Attribute::Reader { inline bool hasOrder() const; inline ::capnp::Text::Reader getOrder() const; + inline bool hasEnumerationName() const; + inline ::capnp::Text::Reader getEnumerationName() const; + private: ::capnp::_::StructReader _reader; template @@ -3131,6 +3253,13 @@ class Attribute::Builder { inline void adoptOrder(::capnp::Orphan<::capnp::Text>&& value); inline ::capnp::Orphan<::capnp::Text> disownOrder(); + inline bool hasEnumerationName(); + inline ::capnp::Text::Builder getEnumerationName(); + inline void setEnumerationName(::capnp::Text::Reader value); + inline ::capnp::Text::Builder initEnumerationName(unsigned int size); + inline void adoptEnumerationName(::capnp::Orphan<::capnp::Text>&& value); + inline ::capnp::Orphan<::capnp::Text> disownEnumerationName(); + private: ::capnp::_::StructBuilder _builder; template @@ -3163,6 +3292,153 @@ class Attribute::Pipeline { }; #endif // !CAPNP_LITE +class Enumeration::Reader { + public: + typedef Enumeration Reads; + + Reader() = default; + inline explicit Reader(::capnp::_::StructReader base) + : _reader(base) { + } + + inline ::capnp::MessageSize totalSize() const { + return _reader.totalSize().asPublic(); + } + +#if !CAPNP_LITE + inline ::kj::StringTree toString() const { + return ::capnp::_::structString(_reader, *_capnpPrivate::brand()); + } +#endif // !CAPNP_LITE + + inline bool hasName() const; + inline ::capnp::Text::Reader getName() const; + + inline bool hasPathName() const; + inline ::capnp::Text::Reader getPathName() const; + + inline bool hasType() const; + inline ::capnp::Text::Reader getType() const; + + inline ::uint32_t getCellValNum() const; + + inline bool getOrdered() const; + + inline bool hasData() const; + inline ::capnp::Data::Reader getData() const; + + inline bool hasOffsets() const; + inline ::capnp::Data::Reader getOffsets() const; + + private: + ::capnp::_::StructReader _reader; + template + friend struct ::capnp::ToDynamic_; + template + friend struct ::capnp::_::PointerHelpers; + template + friend struct ::capnp::List; + friend class ::capnp::MessageBuilder; + friend class ::capnp::Orphanage; +}; + +class Enumeration::Builder { + public: + typedef Enumeration Builds; + + Builder() = delete; // Deleted to discourage incorrect usage. + // You can explicitly initialize to nullptr instead. + inline Builder(decltype(nullptr)) { + } + inline explicit Builder(::capnp::_::StructBuilder base) + : _builder(base) { + } + inline operator Reader() const { + return Reader(_builder.asReader()); + } + inline Reader asReader() const { + return *this; + } + + inline ::capnp::MessageSize totalSize() const { + return asReader().totalSize(); + } +#if !CAPNP_LITE + inline ::kj::StringTree toString() const { + return asReader().toString(); + } +#endif // !CAPNP_LITE + + inline bool hasName(); + inline ::capnp::Text::Builder getName(); + inline void setName(::capnp::Text::Reader value); + inline ::capnp::Text::Builder initName(unsigned int size); + inline void adoptName(::capnp::Orphan<::capnp::Text>&& value); + inline ::capnp::Orphan<::capnp::Text> disownName(); + + inline bool hasPathName(); + inline ::capnp::Text::Builder getPathName(); + inline void setPathName(::capnp::Text::Reader value); + inline ::capnp::Text::Builder initPathName(unsigned int size); + inline void adoptPathName(::capnp::Orphan<::capnp::Text>&& value); + inline ::capnp::Orphan<::capnp::Text> disownPathName(); + + inline bool hasType(); + inline ::capnp::Text::Builder getType(); + inline void setType(::capnp::Text::Reader value); + inline ::capnp::Text::Builder initType(unsigned int size); + inline void adoptType(::capnp::Orphan<::capnp::Text>&& value); + inline ::capnp::Orphan<::capnp::Text> disownType(); + + inline ::uint32_t getCellValNum(); + inline void setCellValNum(::uint32_t value); + + inline bool getOrdered(); + inline void setOrdered(bool value); + + inline bool hasData(); + inline ::capnp::Data::Builder getData(); + inline void setData(::capnp::Data::Reader value); + inline ::capnp::Data::Builder initData(unsigned int size); + inline void adoptData(::capnp::Orphan<::capnp::Data>&& value); + inline ::capnp::Orphan<::capnp::Data> disownData(); + + inline bool hasOffsets(); + inline ::capnp::Data::Builder getOffsets(); + inline void setOffsets(::capnp::Data::Reader value); + inline ::capnp::Data::Builder initOffsets(unsigned int size); + inline void adoptOffsets(::capnp::Orphan<::capnp::Data>&& value); + inline ::capnp::Orphan<::capnp::Data> disownOffsets(); + + private: + ::capnp::_::StructBuilder _builder; + template + friend struct ::capnp::ToDynamic_; + friend class ::capnp::Orphanage; + template + friend struct ::capnp::_::PointerHelpers; +}; + +#if !CAPNP_LITE +class Enumeration::Pipeline { + public: + typedef Enumeration Pipelines; + + inline Pipeline(decltype(nullptr)) + : _typeless(nullptr) { + } + inline explicit Pipeline(::capnp::AnyPointer::Pipeline&& typeless) + : _typeless(kj::mv(typeless)) { + } + + private: + ::capnp::AnyPointer::Pipeline _typeless; + friend class ::capnp::PipelineHook; + template + friend struct ::capnp::ToDynamic_; +}; +#endif // !CAPNP_LITE + class AttributeBufferHeader::Reader { public: typedef AttributeBufferHeader Reads; @@ -7193,6 +7469,8 @@ class ConditionClause::Reader { inline bool hasOp() const; inline ::capnp::Text::Reader getOp() const; + inline bool getUseEnumeration() const; + private: ::capnp::_::StructReader _reader; template @@ -7253,6 +7531,9 @@ class ConditionClause::Builder { inline void adoptOp(::capnp::Orphan<::capnp::Text>&& value); inline ::capnp::Orphan<::capnp::Text> disownOp(); + inline bool getUseEnumeration(); + inline void setUseEnumeration(bool value); + private: ::capnp::_::StructBuilder _builder; template @@ -7321,6 +7602,8 @@ class ASTNode::Reader { inline bool hasCombinationOp() const; inline ::capnp::Text::Reader getCombinationOp() const; + inline bool getUseEnumeration() const; + private: ::capnp::_::StructReader _reader; template @@ -7411,6 +7694,9 @@ class ASTNode::Builder { inline void adoptCombinationOp(::capnp::Orphan<::capnp::Text>&& value); inline ::capnp::Orphan<::capnp::Text> disownCombinationOp(); + inline bool getUseEnumeration(); + inline void setUseEnumeration(bool value); + private: ::capnp::_::StructBuilder _builder; template @@ -15475,54 +15761,199 @@ ArraySchema::Builder::disownDimensionLabels() { ::capnp::POINTERS)); } -inline ::uint32_t DimensionLabel::Reader::getDimensionId() const { - return _reader.getDataField<::uint32_t>( - ::capnp::bounded<0>() * ::capnp::ELEMENTS); -} - -inline ::uint32_t DimensionLabel::Builder::getDimensionId() { - return _builder.getDataField<::uint32_t>( - ::capnp::bounded<0>() * ::capnp::ELEMENTS); -} -inline void DimensionLabel::Builder::setDimensionId(::uint32_t value) { - _builder.setDataField<::uint32_t>( - ::capnp::bounded<0>() * ::capnp::ELEMENTS, value); -} - -inline bool DimensionLabel::Reader::hasName() const { - return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) +inline bool ArraySchema::Reader::hasEnumerations() const { + return !_reader.getPointerField(::capnp::bounded<13>() * ::capnp::POINTERS) .isNull(); } -inline bool DimensionLabel::Builder::hasName() { - return !_builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) +inline bool ArraySchema::Builder::hasEnumerations() { + return !_builder.getPointerField(::capnp::bounded<13>() * ::capnp::POINTERS) .isNull(); } -inline ::capnp::Text::Reader DimensionLabel::Reader::getName() const { - return ::capnp::_::PointerHelpers<::capnp::Text>::get( - _reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); -} -inline ::capnp::Text::Builder DimensionLabel::Builder::getName() { - return ::capnp::_::PointerHelpers<::capnp::Text>::get( - _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader +ArraySchema::Reader::getEnumerations() const { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>::get(_reader + .getPointerField( + ::capnp::bounded<13>() * + ::capnp::POINTERS)); } -inline void DimensionLabel::Builder::setName(::capnp::Text::Reader value) { - ::capnp::_::PointerHelpers<::capnp::Text>::set( - _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), - value); +inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder +ArraySchema::Builder::getEnumerations() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>::get(_builder + .getPointerField( + ::capnp::bounded<13>() * + ::capnp::POINTERS)); } -inline ::capnp::Text::Builder DimensionLabel::Builder::initName( - unsigned int size) { - return ::capnp::_::PointerHelpers<::capnp::Text>::init( - _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), - size); +inline void ArraySchema::Builder::setEnumerations( + ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>:: + set(_builder.getPointerField(::capnp::bounded<13>() * ::capnp::POINTERS), + value); } -inline void DimensionLabel::Builder::adoptName( - ::capnp::Orphan<::capnp::Text>&& value) { - ::capnp::_::PointerHelpers<::capnp::Text>::adopt( - _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), - kj::mv(value)); +inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder +ArraySchema::Builder::initEnumerations(unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>:: + init( + _builder.getPointerField(::capnp::bounded<13>() * ::capnp::POINTERS), + size); } -inline ::capnp::Orphan<::capnp::Text> DimensionLabel::Builder::disownName() { +inline void ArraySchema::Builder::adoptEnumerations( + ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>&& value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>:: + adopt( + _builder.getPointerField(::capnp::bounded<13>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>> +ArraySchema::Builder::disownEnumerations() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>::disown(_builder + .getPointerField( + ::capnp::bounded<13>() * + ::capnp::POINTERS)); +} + +inline bool ArraySchema::Reader::hasEnumerationPathMap() const { + return !_reader.getPointerField(::capnp::bounded<14>() * ::capnp::POINTERS) + .isNull(); +} +inline bool ArraySchema::Builder::hasEnumerationPathMap() { + return !_builder.getPointerField(::capnp::bounded<14>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp:: + List<::tiledb::sm::serialization::capnp::KV, ::capnp::Kind::STRUCT>::Reader + ArraySchema::Reader::getEnumerationPathMap() const { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>>::get(_reader + .getPointerField( + ::capnp::bounded<14>() * + ::capnp::POINTERS)); +} +inline ::capnp:: + List<::tiledb::sm::serialization::capnp::KV, ::capnp::Kind::STRUCT>::Builder + ArraySchema::Builder::getEnumerationPathMap() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>>::get(_builder + .getPointerField( + ::capnp::bounded<14>() * + ::capnp::POINTERS)); +} +inline void ArraySchema::Builder::setEnumerationPathMap( + ::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>::Reader value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>>:: + set(_builder.getPointerField(::capnp::bounded<14>() * ::capnp::POINTERS), + value); +} +inline ::capnp:: + List<::tiledb::sm::serialization::capnp::KV, ::capnp::Kind::STRUCT>::Builder + ArraySchema::Builder::initEnumerationPathMap(unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>>:: + init( + _builder.getPointerField(::capnp::bounded<14>() * ::capnp::POINTERS), + size); +} +inline void ArraySchema::Builder::adoptEnumerationPathMap( + ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>>&& value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>>:: + adopt( + _builder.getPointerField(::capnp::bounded<14>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>> +ArraySchema::Builder::disownEnumerationPathMap() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>>::disown(_builder + .getPointerField( + ::capnp::bounded<14>() * + ::capnp::POINTERS)); +} + +inline ::uint32_t DimensionLabel::Reader::getDimensionId() const { + return _reader.getDataField<::uint32_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS); +} + +inline ::uint32_t DimensionLabel::Builder::getDimensionId() { + return _builder.getDataField<::uint32_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS); +} +inline void DimensionLabel::Builder::setDimensionId(::uint32_t value) { + _builder.setDataField<::uint32_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS, value); +} + +inline bool DimensionLabel::Reader::hasName() const { + return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) + .isNull(); +} +inline bool DimensionLabel::Builder::hasName() { + return !_builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::Text::Reader DimensionLabel::Reader::getName() const { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder DimensionLabel::Builder::getName() { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline void DimensionLabel::Builder::setName(::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers<::capnp::Text>::set( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + value); +} +inline ::capnp::Text::Builder DimensionLabel::Builder::initName( + unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::Text>::init( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + size); +} +inline void DimensionLabel::Builder::adoptName( + ::capnp::Orphan<::capnp::Text>&& value) { + ::capnp::_::PointerHelpers<::capnp::Text>::adopt( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::Text> DimensionLabel::Builder::disownName() { return ::capnp::_::PointerHelpers<::capnp::Text>::disown( _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); } @@ -15962,6 +16393,136 @@ ArraySchemaEvolution::Builder::disownTimestampRange() { _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS)); } +inline bool ArraySchemaEvolution::Reader::hasEnumerationsToAdd() const { + return !_reader.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS) + .isNull(); +} +inline bool ArraySchemaEvolution::Builder::hasEnumerationsToAdd() { + return !_builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader +ArraySchemaEvolution::Reader::getEnumerationsToAdd() const { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>::get(_reader + .getPointerField( + ::capnp::bounded<3>() * + ::capnp::POINTERS)); +} +inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder +ArraySchemaEvolution::Builder::getEnumerationsToAdd() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>::get(_builder + .getPointerField( + ::capnp::bounded<3>() * + ::capnp::POINTERS)); +} +inline void ArraySchemaEvolution::Builder::setEnumerationsToAdd( + ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>:: + set(_builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS), + value); +} +inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder +ArraySchemaEvolution::Builder::initEnumerationsToAdd(unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>:: + init( + _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS), + size); +} +inline void ArraySchemaEvolution::Builder::adoptEnumerationsToAdd( + ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>&& value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>:: + adopt( + _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>> +ArraySchemaEvolution::Builder::disownEnumerationsToAdd() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>::disown(_builder + .getPointerField( + ::capnp::bounded<3>() * + ::capnp::POINTERS)); +} + +inline bool ArraySchemaEvolution::Reader::hasEnumerationsToDrop() const { + return !_reader.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS) + .isNull(); +} +inline bool ArraySchemaEvolution::Builder::hasEnumerationsToDrop() { + return !_builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Reader +ArraySchemaEvolution::Reader::getEnumerationsToDrop() const { + return ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::get( + _reader.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS)); +} +inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Builder +ArraySchemaEvolution::Builder::getEnumerationsToDrop() { + return ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::get( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS)); +} +inline void ArraySchemaEvolution::Builder::setEnumerationsToDrop( + ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Reader value) { + ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::set( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS), + value); +} +inline void ArraySchemaEvolution::Builder::setEnumerationsToDrop( + ::kj::ArrayPtr value) { + ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::set( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS), + value); +} +inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Builder +ArraySchemaEvolution::Builder::initEnumerationsToDrop(unsigned int size) { + return ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::init( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS), + size); +} +inline void ArraySchemaEvolution::Builder::adoptEnumerationsToDrop( + ::capnp::Orphan<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>&& + value) { + ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::adopt( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>> +ArraySchemaEvolution::Builder::disownEnumerationsToDrop() { + return ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::disown( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS)); +} + inline ::uint32_t Attribute::Reader::getCellValNum() const { return _reader.getDataField<::uint32_t>( ::capnp::bounded<0>() * ::capnp::ELEMENTS); @@ -16207,6 +16768,263 @@ inline ::capnp::Orphan<::capnp::Text> Attribute::Builder::disownOrder() { _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS)); } +inline bool Attribute::Reader::hasEnumerationName() const { + return !_reader.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS) + .isNull(); +} +inline bool Attribute::Builder::hasEnumerationName() { + return !_builder.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::Text::Reader Attribute::Reader::getEnumerationName() const { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _reader.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder Attribute::Builder::getEnumerationName() { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _builder.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS)); +} +inline void Attribute::Builder::setEnumerationName( + ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers<::capnp::Text>::set( + _builder.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS), + value); +} +inline ::capnp::Text::Builder Attribute::Builder::initEnumerationName( + unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::Text>::init( + _builder.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS), + size); +} +inline void Attribute::Builder::adoptEnumerationName( + ::capnp::Orphan<::capnp::Text>&& value) { + ::capnp::_::PointerHelpers<::capnp::Text>::adopt( + _builder.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::Text> +Attribute::Builder::disownEnumerationName() { + return ::capnp::_::PointerHelpers<::capnp::Text>::disown( + _builder.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS)); +} + +inline bool Enumeration::Reader::hasName() const { + return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) + .isNull(); +} +inline bool Enumeration::Builder::hasName() { + return !_builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::Text::Reader Enumeration::Reader::getName() const { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder Enumeration::Builder::getName() { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline void Enumeration::Builder::setName(::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers<::capnp::Text>::set( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + value); +} +inline ::capnp::Text::Builder Enumeration::Builder::initName( + unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::Text>::init( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + size); +} +inline void Enumeration::Builder::adoptName( + ::capnp::Orphan<::capnp::Text>&& value) { + ::capnp::_::PointerHelpers<::capnp::Text>::adopt( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::Text> Enumeration::Builder::disownName() { + return ::capnp::_::PointerHelpers<::capnp::Text>::disown( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} + +inline bool Enumeration::Reader::hasPathName() const { + return !_reader.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS) + .isNull(); +} +inline bool Enumeration::Builder::hasPathName() { + return !_builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::Text::Reader Enumeration::Reader::getPathName() const { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _reader.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder Enumeration::Builder::getPathName() { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); +} +inline void Enumeration::Builder::setPathName(::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers<::capnp::Text>::set( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), + value); +} +inline ::capnp::Text::Builder Enumeration::Builder::initPathName( + unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::Text>::init( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), + size); +} +inline void Enumeration::Builder::adoptPathName( + ::capnp::Orphan<::capnp::Text>&& value) { + ::capnp::_::PointerHelpers<::capnp::Text>::adopt( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::Text> Enumeration::Builder::disownPathName() { + return ::capnp::_::PointerHelpers<::capnp::Text>::disown( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); +} + +inline bool Enumeration::Reader::hasType() const { + return !_reader.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS) + .isNull(); +} +inline bool Enumeration::Builder::hasType() { + return !_builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::Text::Reader Enumeration::Reader::getType() const { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _reader.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder Enumeration::Builder::getType() { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS)); +} +inline void Enumeration::Builder::setType(::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers<::capnp::Text>::set( + _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS), + value); +} +inline ::capnp::Text::Builder Enumeration::Builder::initType( + unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::Text>::init( + _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS), + size); +} +inline void Enumeration::Builder::adoptType( + ::capnp::Orphan<::capnp::Text>&& value) { + ::capnp::_::PointerHelpers<::capnp::Text>::adopt( + _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::Text> Enumeration::Builder::disownType() { + return ::capnp::_::PointerHelpers<::capnp::Text>::disown( + _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS)); +} + +inline ::uint32_t Enumeration::Reader::getCellValNum() const { + return _reader.getDataField<::uint32_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS); +} + +inline ::uint32_t Enumeration::Builder::getCellValNum() { + return _builder.getDataField<::uint32_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS); +} +inline void Enumeration::Builder::setCellValNum(::uint32_t value) { + _builder.setDataField<::uint32_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS, value); +} + +inline bool Enumeration::Reader::getOrdered() const { + return _reader.getDataField(::capnp::bounded<32>() * ::capnp::ELEMENTS); +} + +inline bool Enumeration::Builder::getOrdered() { + return _builder.getDataField( + ::capnp::bounded<32>() * ::capnp::ELEMENTS); +} +inline void Enumeration::Builder::setOrdered(bool value) { + _builder.setDataField( + ::capnp::bounded<32>() * ::capnp::ELEMENTS, value); +} + +inline bool Enumeration::Reader::hasData() const { + return !_reader.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS) + .isNull(); +} +inline bool Enumeration::Builder::hasData() { + return !_builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::Data::Reader Enumeration::Reader::getData() const { + return ::capnp::_::PointerHelpers<::capnp::Data>::get( + _reader.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS)); +} +inline ::capnp::Data::Builder Enumeration::Builder::getData() { + return ::capnp::_::PointerHelpers<::capnp::Data>::get( + _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS)); +} +inline void Enumeration::Builder::setData(::capnp::Data::Reader value) { + ::capnp::_::PointerHelpers<::capnp::Data>::set( + _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS), + value); +} +inline ::capnp::Data::Builder Enumeration::Builder::initData( + unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::Data>::init( + _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS), + size); +} +inline void Enumeration::Builder::adoptData( + ::capnp::Orphan<::capnp::Data>&& value) { + ::capnp::_::PointerHelpers<::capnp::Data>::adopt( + _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::Data> Enumeration::Builder::disownData() { + return ::capnp::_::PointerHelpers<::capnp::Data>::disown( + _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS)); +} + +inline bool Enumeration::Reader::hasOffsets() const { + return !_reader.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS) + .isNull(); +} +inline bool Enumeration::Builder::hasOffsets() { + return !_builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::Data::Reader Enumeration::Reader::getOffsets() const { + return ::capnp::_::PointerHelpers<::capnp::Data>::get( + _reader.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS)); +} +inline ::capnp::Data::Builder Enumeration::Builder::getOffsets() { + return ::capnp::_::PointerHelpers<::capnp::Data>::get( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS)); +} +inline void Enumeration::Builder::setOffsets(::capnp::Data::Reader value) { + ::capnp::_::PointerHelpers<::capnp::Data>::set( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS), + value); +} +inline ::capnp::Data::Builder Enumeration::Builder::initOffsets( + unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::Data>::init( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS), + size); +} +inline void Enumeration::Builder::adoptOffsets( + ::capnp::Orphan<::capnp::Data>&& value) { + ::capnp::_::PointerHelpers<::capnp::Data>::adopt( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::Data> Enumeration::Builder::disownOffsets() { + return ::capnp::_::PointerHelpers<::capnp::Data>::disown( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS)); +} + inline bool AttributeBufferHeader::Reader::hasName() const { return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) .isNull(); @@ -20752,6 +21570,17 @@ inline ::capnp::Orphan<::capnp::Text> ConditionClause::Builder::disownOp() { _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS)); } +inline bool ConditionClause::Reader::getUseEnumeration() const { + return _reader.getDataField(::capnp::bounded<0>() * ::capnp::ELEMENTS); +} + +inline bool ConditionClause::Builder::getUseEnumeration() { + return _builder.getDataField(::capnp::bounded<0>() * ::capnp::ELEMENTS); +} +inline void ConditionClause::Builder::setUseEnumeration(bool value) { + _builder.setDataField(::capnp::bounded<0>() * ::capnp::ELEMENTS, value); +} + inline bool ASTNode::Reader::getIsExpression() const { return _reader.getDataField(::capnp::bounded<0>() * ::capnp::ELEMENTS); } @@ -20986,6 +21815,17 @@ inline ::capnp::Orphan<::capnp::Text> ASTNode::Builder::disownCombinationOp() { _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS)); } +inline bool ASTNode::Reader::getUseEnumeration() const { + return _reader.getDataField(::capnp::bounded<1>() * ::capnp::ELEMENTS); +} + +inline bool ASTNode::Builder::getUseEnumeration() { + return _builder.getDataField(::capnp::bounded<1>() * ::capnp::ELEMENTS); +} +inline void ASTNode::Builder::setUseEnumeration(bool value) { + _builder.setDataField(::capnp::bounded<1>() * ::capnp::ELEMENTS, value); +} + inline bool Condition::Reader::hasClauses() const { return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) .isNull(); diff --git a/tiledb/sm/serialization/query.cc b/tiledb/sm/serialization/query.cc index 4e697a28e812..397eba71aa85 100644 --- a/tiledb/sm/serialization/query.cc +++ b/tiledb/sm/serialization/query.cc @@ -776,6 +776,9 @@ static Status condition_ast_to_capnp( const std::string op_str = query_condition_op_str(node->get_op()); ensure_qc_op_string_is_valid(op_str); ast_builder->setOp(op_str); + + // Store whether this expression should skip the enumeration lookup. + ast_builder->setUseEnumeration(node->use_enumeration()); } else { // Store the boolean expression tag. ast_builder->setIsExpression(true); @@ -823,6 +826,8 @@ static void clause_to_capnp( const std::string op_str = query_condition_op_str(node->get_op()); ensure_qc_op_string_is_valid(op_str); clause_builder->setOp(op_str); + + clause_builder->setUseEnumeration(node->use_enumeration()); } Status condition_to_capnp( @@ -996,8 +1001,10 @@ tdb_unique_ptr condition_ast_from_capnp( } ensure_qc_op_is_valid(op); + auto use_enumeration = ast_reader.getUseEnumeration(); + return tdb_unique_ptr( - tdb_new(ASTNodeVal, field_name, data, size, op)); + tdb_new(ASTNodeVal, field_name, data, size, op, use_enumeration)); } // Getting and validating the query condition combination operator. @@ -1049,8 +1056,10 @@ Status condition_from_capnp( } ensure_qc_op_is_valid(op); + bool use_enumeration = clause.getUseEnumeration(); + ast_nodes.push_back(tdb_unique_ptr( - tdb_new(ASTNodeVal, field_name, data, size, op))); + tdb_new(ASTNodeVal, field_name, data, size, op, use_enumeration))); } // Constructing the tree from the list of AST nodes. diff --git a/tiledb/sm/serialization/tiledb-rest.capnp b/tiledb/sm/serialization/tiledb-rest.capnp index 1bcfb60f3bcb..6a37ed9d67eb 100644 --- a/tiledb/sm/serialization/tiledb-rest.capnp +++ b/tiledb/sm/serialization/tiledb-rest.capnp @@ -122,6 +122,12 @@ struct ArraySchema { dimensionLabels @14 :List(DimensionLabel); # Dimension labels of the array + + enumerations @15: List(Enumeration); + # Enumerations of the array + + enumerationPathMap @16: List(KV); + # Enumeration name to path map } struct DimensionLabel { @@ -167,6 +173,12 @@ struct ArraySchemaEvolution { timestampRange @2 :List(UInt64); # Timestamp range of array schema + + enumerationsToAdd @3 :List(Enumeration); + # Enumerations to be added + + enumerationsToDrop @4 :List(Text); + # Enumeration names to be dropped } struct Attribute { @@ -194,6 +206,33 @@ struct Attribute { order @7 :Text; # The prescribed order of the data stored in the attribute + + enumerationName @8 :Text; + # Name of the enumeration for this attribute, if it has one +} + +struct Enumeration { +# Enumeration of values for use by Attributes + name @0 :Text; + # Enumeration name + + pathName @1 :Text; + # Enumeration path name + + type @2 :Text; + # Type of the Enumeration values + + cellValNum @3 :UInt32; + # Enumeration number of values per cell + + ordered @4 :Bool; + # Whether the enumeration is considered orderable + + data @5 :Data; + # The contents of the enumeration values + + offsets @6 :Data; + # The contents of the enumeration offsets buffer } struct AttributeBufferHeader { @@ -534,6 +573,9 @@ struct ConditionClause { op @2 :Text; # The comparison operation + + useEnumeration @3 :Bool; + # Whether or not to use the associated attribute's Enumeration } struct ASTNode { @@ -557,6 +599,9 @@ struct ASTNode { combinationOp @5 :Text; # The combination logical operator + + useEnumeration @6 :Bool; + # Whether or not to use the associated attribute's Enumeration } struct Condition { diff --git a/tiledb/sm/serialization/win32/tiledb-rest.capnp.c++ b/tiledb/sm/serialization/win32/tiledb-rest.capnp.c++ index 08df30190b05..86e22aaddac3 100644 --- a/tiledb/sm/serialization/win32/tiledb-rest.capnp.c++ +++ b/tiledb/sm/serialization/win32/tiledb-rest.capnp.c++ @@ -635,17 +635,17 @@ const ::capnp::_::RawSchema s_facceeafd4472c68 = { 1, 2, i_facceeafd4472c68, nullptr, nullptr, { &s_facceeafd4472c68, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<274> b_d71de32f98e296fe = { +static const ::capnp::_::AlignedData<315> b_d71de32f98e296fe = { { 0, 0, 0, 0, 5, 0, 6, 0, 254, 150, 226, 152, 47, 227, 29, 215, 18, 0, 0, 0, 1, 0, 2, 0, 127, 216, 135, 181, 36, 146, 125, 181, - 13, 0, 7, 0, 0, 0, 0, 0, + 15, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 242, 0, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 0, 0, 0, 79, 3, 0, 0, + 29, 0, 0, 0, 191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, @@ -653,112 +653,126 @@ static const ::capnp::_::AlignedData<274> b_d71de32f98e296fe = { 112, 58, 65, 114, 114, 97, 121, 83, 99, 104, 101, 109, 97, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, - 60, 0, 0, 0, 3, 0, 4, 0, + 68, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 149, 1, 0, 0, 82, 0, 0, 0, + 205, 1, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 148, 1, 0, 0, 3, 0, 1, 0, - 160, 1, 0, 0, 2, 0, 1, 0, + 204, 1, 0, 0, 3, 0, 1, 0, + 216, 1, 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 157, 1, 0, 0, 90, 0, 0, 0, + 213, 1, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 156, 1, 0, 0, 3, 0, 1, 0, - 184, 1, 0, 0, 2, 0, 1, 0, + 212, 1, 0, 0, 3, 0, 1, 0, + 240, 1, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 181, 1, 0, 0, 74, 0, 0, 0, + 237, 1, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 180, 1, 0, 0, 3, 0, 1, 0, - 192, 1, 0, 0, 2, 0, 1, 0, + 236, 1, 0, 0, 3, 0, 1, 0, + 248, 1, 0, 0, 2, 0, 1, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 189, 1, 0, 0, 82, 0, 0, 0, + 245, 1, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 188, 1, 0, 0, 3, 0, 1, 0, - 200, 1, 0, 0, 2, 0, 1, 0, + 244, 1, 0, 0, 3, 0, 1, 0, + 0, 2, 0, 0, 2, 0, 1, 0, 4, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 197, 1, 0, 0, 170, 0, 0, 0, + 253, 1, 0, 0, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 200, 1, 0, 0, 3, 0, 1, 0, - 212, 1, 0, 0, 2, 0, 1, 0, + 0, 2, 0, 0, 3, 0, 1, 0, + 12, 2, 0, 0, 2, 0, 1, 0, 5, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 209, 1, 0, 0, 58, 0, 0, 0, + 9, 2, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 204, 1, 0, 0, 3, 0, 1, 0, - 216, 1, 0, 0, 2, 0, 1, 0, + 4, 2, 0, 0, 3, 0, 1, 0, + 16, 2, 0, 0, 2, 0, 1, 0, 6, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 213, 1, 0, 0, 170, 0, 0, 0, + 13, 2, 0, 0, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 216, 1, 0, 0, 3, 0, 1, 0, - 228, 1, 0, 0, 2, 0, 1, 0, + 16, 2, 0, 0, 3, 0, 1, 0, + 28, 2, 0, 0, 2, 0, 1, 0, 7, 0, 0, 0, 6, 0, 0, 0, 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 225, 1, 0, 0, 82, 0, 0, 0, + 25, 2, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 224, 1, 0, 0, 3, 0, 1, 0, - 236, 1, 0, 0, 2, 0, 1, 0, + 24, 2, 0, 0, 3, 0, 1, 0, + 36, 2, 0, 0, 2, 0, 1, 0, 8, 0, 0, 0, 7, 0, 0, 0, 0, 0, 1, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 233, 1, 0, 0, 34, 0, 0, 0, + 33, 2, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 228, 1, 0, 0, 3, 0, 1, 0, - 240, 1, 0, 0, 2, 0, 1, 0, + 28, 2, 0, 0, 3, 0, 1, 0, + 40, 2, 0, 0, 2, 0, 1, 0, 9, 0, 0, 0, 8, 0, 0, 0, 0, 0, 1, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 237, 1, 0, 0, 66, 0, 0, 0, + 37, 2, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 232, 1, 0, 0, 3, 0, 1, 0, - 4, 2, 0, 0, 2, 0, 1, 0, + 32, 2, 0, 0, 3, 0, 1, 0, + 60, 2, 0, 0, 2, 0, 1, 0, 10, 0, 0, 0, 64, 0, 0, 0, 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 2, 0, 0, 138, 0, 0, 0, + 57, 2, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 4, 2, 0, 0, 3, 0, 1, 0, - 16, 2, 0, 0, 2, 0, 1, 0, + 60, 2, 0, 0, 3, 0, 1, 0, + 72, 2, 0, 0, 2, 0, 1, 0, 11, 0, 0, 0, 9, 0, 0, 0, 0, 0, 1, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 13, 2, 0, 0, 186, 0, 0, 0, + 69, 2, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 16, 2, 0, 0, 3, 0, 1, 0, - 28, 2, 0, 0, 2, 0, 1, 0, + 72, 2, 0, 0, 3, 0, 1, 0, + 84, 2, 0, 0, 2, 0, 1, 0, 12, 0, 0, 0, 10, 0, 0, 0, 0, 0, 1, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 25, 2, 0, 0, 42, 0, 0, 0, + 81, 2, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 20, 2, 0, 0, 3, 0, 1, 0, - 32, 2, 0, 0, 2, 0, 1, 0, + 76, 2, 0, 0, 3, 0, 1, 0, + 88, 2, 0, 0, 2, 0, 1, 0, 13, 0, 0, 0, 11, 0, 0, 0, 0, 0, 1, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 2, 0, 0, 122, 0, 0, 0, + 85, 2, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 28, 2, 0, 0, 3, 0, 1, 0, - 56, 2, 0, 0, 2, 0, 1, 0, + 84, 2, 0, 0, 3, 0, 1, 0, + 112, 2, 0, 0, 2, 0, 1, 0, 14, 0, 0, 0, 12, 0, 0, 0, 0, 0, 1, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 53, 2, 0, 0, 130, 0, 0, 0, + 109, 2, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 52, 2, 0, 0, 3, 0, 1, 0, - 80, 2, 0, 0, 2, 0, 1, 0, + 108, 2, 0, 0, 3, 0, 1, 0, + 136, 2, 0, 0, 2, 0, 1, 0, + 15, 0, 0, 0, 13, 0, 0, 0, + 0, 0, 1, 0, 15, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 133, 2, 0, 0, 106, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 132, 2, 0, 0, 3, 0, 1, 0, + 160, 2, 0, 0, 2, 0, 1, 0, + 16, 0, 0, 0, 14, 0, 0, 0, + 0, 0, 1, 0, 16, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 157, 2, 0, 0, 154, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 160, 2, 0, 0, 3, 0, 1, 0, + 188, 2, 0, 0, 2, 0, 1, 0, 97, 114, 114, 97, 121, 84, 121, 112, 101, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, @@ -907,6 +921,33 @@ static const ::capnp::_::AlignedData<274> b_d71de32f98e296fe = { 222, 209, 12, 209, 98, 141, 255, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 110, 117, 109, 101, 114, 97, 116, + 105, 111, 110, 115, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 180, 185, 33, 204, 25, 47, 11, 208, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 110, 117, 109, 101, 114, 97, 116, + 105, 111, 110, 80, 97, 116, 104, 77, + 97, 112, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 151, 188, 17, 242, 43, 223, 218, 227, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, } @@ -917,13 +958,15 @@ static const ::capnp::_::RawSchema* const d_d71de32f98e296fe[] = { &s_92ad78f56de3d76a, &s_bc4583f733eac4f5, &s_ceff8d62d10cd1de, + &s_d00b2f19cc21b9b4, &s_de030f447664754c, + &s_e3dadf2bf211bc97, }; -static const uint16_t m_d71de32f98e296fe[] = {10, 0, 1, 2, 3, 4, 14, 5, 12, 6, 7, 13, 8, 11, 9}; -static const uint16_t i_d71de32f98e296fe[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}; +static const uint16_t m_d71de32f98e296fe[] = {10, 0, 1, 2, 3, 4, 14, 5, 16, 15, 12, 6, 7, 13, 8, 11, 9}; +static const uint16_t i_d71de32f98e296fe[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; const ::capnp::_::RawSchema s_d71de32f98e296fe = { - 0xd71de32f98e296fe, b_d71de32f98e296fe.words, 274, d_d71de32f98e296fe, m_d71de32f98e296fe, - 4, 15, i_d71de32f98e296fe, nullptr, nullptr, { &s_d71de32f98e296fe, nullptr, nullptr, 0, 0, nullptr } + 0xd71de32f98e296fe, b_d71de32f98e296fe.words, 315, d_d71de32f98e296fe, m_d71de32f98e296fe, + 6, 17, i_d71de32f98e296fe, nullptr, nullptr, { &s_d71de32f98e296fe, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE static const ::capnp::_::AlignedData<174> b_ceff8d62d10cd1de = { @@ -1114,17 +1157,17 @@ const ::capnp::_::RawSchema s_ceff8d62d10cd1de = { 1, 10, i_ceff8d62d10cd1de, nullptr, nullptr, { &s_ceff8d62d10cd1de, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<80> b_a1b81d67548230d4 = { +static const ::capnp::_::AlignedData<122> b_a1b81d67548230d4 = { { 0, 0, 0, 0, 5, 0, 6, 0, 212, 48, 130, 84, 103, 29, 184, 161, 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, - 3, 0, 7, 0, 0, 0, 0, 0, + 5, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 58, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 0, 0, 0, 175, 0, 0, 0, + 33, 0, 0, 0, 31, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, @@ -1133,28 +1176,42 @@ static const ::capnp::_::AlignedData<80> b_a1b81d67548230d4 = { 99, 104, 101, 109, 97, 69, 118, 111, 108, 117, 116, 105, 111, 110, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, - 12, 0, 0, 0, 3, 0, 4, 0, + 20, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 69, 0, 0, 0, 138, 0, 0, 0, + 125, 0, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 72, 0, 0, 0, 3, 0, 1, 0, - 100, 0, 0, 0, 2, 0, 1, 0, + 128, 0, 0, 0, 3, 0, 1, 0, + 156, 0, 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 97, 0, 0, 0, 130, 0, 0, 0, + 153, 0, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 96, 0, 0, 0, 3, 0, 1, 0, - 124, 0, 0, 0, 2, 0, 1, 0, + 152, 0, 0, 0, 3, 0, 1, 0, + 180, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 121, 0, 0, 0, 122, 0, 0, 0, + 177, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 120, 0, 0, 0, 3, 0, 1, 0, - 148, 0, 0, 0, 2, 0, 1, 0, + 176, 0, 0, 0, 3, 0, 1, 0, + 204, 0, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 201, 0, 0, 0, 146, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 204, 0, 0, 0, 3, 0, 1, 0, + 232, 0, 0, 0, 2, 0, 1, 0, + 4, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 229, 0, 0, 0, 154, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 232, 0, 0, 0, 3, 0, 1, 0, + 4, 1, 0, 0, 2, 0, 1, 0, 97, 116, 116, 114, 105, 98, 117, 116, 101, 115, 84, 111, 68, 114, 111, 112, 0, 0, 0, 0, 0, 0, 0, 0, @@ -1192,6 +1249,34 @@ static const ::capnp::_::AlignedData<80> b_a1b81d67548230d4 = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 110, 117, 109, 101, 114, 97, 116, + 105, 111, 110, 115, 84, 111, 65, 100, + 100, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 180, 185, 33, 204, 25, 47, 11, 208, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 110, 117, 109, 101, 114, 97, 116, + 105, 111, 110, 115, 84, 111, 68, 114, + 111, 112, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, } @@ -1200,25 +1285,26 @@ static const ::capnp::_::AlignedData<80> b_a1b81d67548230d4 = { #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_a1b81d67548230d4[] = { &s_92ad78f56de3d76a, + &s_d00b2f19cc21b9b4, }; -static const uint16_t m_a1b81d67548230d4[] = {1, 0, 2}; -static const uint16_t i_a1b81d67548230d4[] = {0, 1, 2}; +static const uint16_t m_a1b81d67548230d4[] = {1, 0, 3, 4, 2}; +static const uint16_t i_a1b81d67548230d4[] = {0, 1, 2, 3, 4}; const ::capnp::_::RawSchema s_a1b81d67548230d4 = { - 0xa1b81d67548230d4, b_a1b81d67548230d4.words, 80, d_a1b81d67548230d4, m_a1b81d67548230d4, - 1, 3, i_a1b81d67548230d4, nullptr, nullptr, { &s_a1b81d67548230d4, nullptr, nullptr, 0, 0, nullptr } + 0xa1b81d67548230d4, b_a1b81d67548230d4.words, 122, d_a1b81d67548230d4, m_a1b81d67548230d4, + 2, 5, i_a1b81d67548230d4, nullptr, nullptr, { &s_a1b81d67548230d4, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<144> b_92ad78f56de3d76a = { +static const ::capnp::_::AlignedData<160> b_92ad78f56de3d76a = { { 0, 0, 0, 0, 5, 0, 6, 0, 106, 215, 227, 109, 245, 120, 173, 146, 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, - 5, 0, 7, 0, 0, 0, 0, 0, + 6, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 226, 0, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 0, 0, 0, 199, 1, 0, 0, + 29, 0, 0, 0, 255, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, @@ -1226,63 +1312,70 @@ static const ::capnp::_::AlignedData<144> b_92ad78f56de3d76a = { 112, 58, 65, 116, 116, 114, 105, 98, 117, 116, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, - 32, 0, 0, 0, 3, 0, 4, 0, + 36, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 209, 0, 0, 0, 90, 0, 0, 0, + 237, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 208, 0, 0, 0, 3, 0, 1, 0, - 220, 0, 0, 0, 2, 0, 1, 0, + 236, 0, 0, 0, 3, 0, 1, 0, + 248, 0, 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 217, 0, 0, 0, 42, 0, 0, 0, + 245, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 212, 0, 0, 0, 3, 0, 1, 0, - 224, 0, 0, 0, 2, 0, 1, 0, + 240, 0, 0, 0, 3, 0, 1, 0, + 252, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 221, 0, 0, 0, 42, 0, 0, 0, + 249, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 216, 0, 0, 0, 3, 0, 1, 0, - 228, 0, 0, 0, 2, 0, 1, 0, + 244, 0, 0, 0, 3, 0, 1, 0, + 0, 1, 0, 0, 2, 0, 1, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 225, 0, 0, 0, 122, 0, 0, 0, + 253, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 224, 0, 0, 0, 3, 0, 1, 0, - 236, 0, 0, 0, 2, 0, 1, 0, + 252, 0, 0, 0, 3, 0, 1, 0, + 8, 1, 0, 0, 2, 0, 1, 0, 4, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 233, 0, 0, 0, 82, 0, 0, 0, + 5, 1, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 232, 0, 0, 0, 3, 0, 1, 0, - 244, 0, 0, 0, 2, 0, 1, 0, + 4, 1, 0, 0, 3, 0, 1, 0, + 16, 1, 0, 0, 2, 0, 1, 0, 5, 0, 0, 0, 32, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 241, 0, 0, 0, 74, 0, 0, 0, + 13, 1, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 240, 0, 0, 0, 3, 0, 1, 0, - 252, 0, 0, 0, 2, 0, 1, 0, + 12, 1, 0, 0, 3, 0, 1, 0, + 24, 1, 0, 0, 2, 0, 1, 0, 6, 0, 0, 0, 33, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 249, 0, 0, 0, 146, 0, 0, 0, + 21, 1, 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 252, 0, 0, 0, 3, 0, 1, 0, - 8, 1, 0, 0, 2, 0, 1, 0, + 24, 1, 0, 0, 3, 0, 1, 0, + 36, 1, 0, 0, 2, 0, 1, 0, 7, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 5, 1, 0, 0, 50, 0, 0, 0, + 33, 1, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1, 0, 0, 3, 0, 1, 0, - 12, 1, 0, 0, 2, 0, 1, 0, + 28, 1, 0, 0, 3, 0, 1, 0, + 40, 1, 0, 0, 2, 0, 1, 0, + 8, 0, 0, 0, 5, 0, 0, 0, + 0, 0, 1, 0, 8, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 37, 1, 0, 0, 130, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 36, 1, 0, 0, 3, 0, 1, 0, + 48, 1, 0, 0, 2, 0, 1, 0, 99, 101, 108, 108, 86, 97, 108, 78, 117, 109, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, @@ -1346,6 +1439,15 @@ static const ::capnp::_::AlignedData<144> b_92ad78f56de3d76a = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 114, 100, 101, 114, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 110, 117, 109, 101, 114, 97, 116, + 105, 111, 110, 78, 97, 109, 101, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -1359,11 +1461,147 @@ static const ::capnp::_::AlignedData<144> b_92ad78f56de3d76a = { static const ::capnp::_::RawSchema* const d_92ad78f56de3d76a[] = { &s_bc4583f733eac4f5, }; -static const uint16_t m_92ad78f56de3d76a[] = {0, 4, 6, 3, 1, 5, 7, 2}; -static const uint16_t i_92ad78f56de3d76a[] = {0, 1, 2, 3, 4, 5, 6, 7}; +static const uint16_t m_92ad78f56de3d76a[] = {0, 8, 4, 6, 3, 1, 5, 7, 2}; +static const uint16_t i_92ad78f56de3d76a[] = {0, 1, 2, 3, 4, 5, 6, 7, 8}; const ::capnp::_::RawSchema s_92ad78f56de3d76a = { - 0x92ad78f56de3d76a, b_92ad78f56de3d76a.words, 144, d_92ad78f56de3d76a, m_92ad78f56de3d76a, - 1, 8, i_92ad78f56de3d76a, nullptr, nullptr, { &s_92ad78f56de3d76a, nullptr, nullptr, 0, 0, nullptr } + 0x92ad78f56de3d76a, b_92ad78f56de3d76a.words, 160, d_92ad78f56de3d76a, m_92ad78f56de3d76a, + 1, 9, i_92ad78f56de3d76a, nullptr, nullptr, { &s_92ad78f56de3d76a, nullptr, nullptr, 0, 0, nullptr } +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<125> b_d00b2f19cc21b9b4 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 180, 185, 33, 204, 25, 47, 11, 208, + 18, 0, 0, 0, 1, 0, 1, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 5, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 242, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 29, 0, 0, 0, 143, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 69, 110, 117, 109, 101, 114, + 97, 116, 105, 111, 110, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 28, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 181, 0, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 176, 0, 0, 0, 3, 0, 1, 0, + 188, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 185, 0, 0, 0, 74, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 184, 0, 0, 0, 3, 0, 1, 0, + 196, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 193, 0, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 188, 0, 0, 0, 3, 0, 1, 0, + 200, 0, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 197, 0, 0, 0, 90, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 196, 0, 0, 0, 3, 0, 1, 0, + 208, 0, 0, 0, 2, 0, 1, 0, + 4, 0, 0, 0, 32, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 205, 0, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 200, 0, 0, 0, 3, 0, 1, 0, + 212, 0, 0, 0, 2, 0, 1, 0, + 5, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 5, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 209, 0, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 204, 0, 0, 0, 3, 0, 1, 0, + 216, 0, 0, 0, 2, 0, 1, 0, + 6, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 6, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 213, 0, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 208, 0, 0, 0, 3, 0, 1, 0, + 220, 0, 0, 0, 2, 0, 1, 0, + 110, 97, 109, 101, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 112, 97, 116, 104, 78, 97, 109, 101, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 121, 112, 101, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 101, 108, 108, 86, 97, 108, 78, + 117, 109, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 111, 114, 100, 101, 114, 101, 100, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 100, 97, 116, 97, 0, 0, 0, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 111, 102, 102, 115, 101, 116, 115, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; +::capnp::word const* const bp_d00b2f19cc21b9b4 = b_d00b2f19cc21b9b4.words; +#if !CAPNP_LITE +static const uint16_t m_d00b2f19cc21b9b4[] = {3, 5, 0, 6, 4, 1, 2}; +static const uint16_t i_d00b2f19cc21b9b4[] = {0, 1, 2, 3, 4, 5, 6}; +const ::capnp::_::RawSchema s_d00b2f19cc21b9b4 = { + 0xd00b2f19cc21b9b4, b_d00b2f19cc21b9b4.words, 125, nullptr, m_d00b2f19cc21b9b4, + 0, 7, i_d00b2f19cc21b9b4, nullptr, nullptr, { &s_d00b2f19cc21b9b4, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE static const ::capnp::_::AlignedData<143> b_d20a578112fa92a2 = { @@ -4271,17 +4509,17 @@ const ::capnp::_::RawSchema s_cbe1e7c13508aa2c = { 1, 4, i_cbe1e7c13508aa2c, nullptr, nullptr, { &s_cbe1e7c13508aa2c, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<65> b_dac6a7f675c57409 = { +static const ::capnp::_::AlignedData<81> b_dac6a7f675c57409 = { { 0, 0, 0, 0, 5, 0, 6, 0, 9, 116, 197, 117, 246, 167, 198, 218, - 18, 0, 0, 0, 1, 0, 0, 0, + 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 18, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 0, 0, 0, 175, 0, 0, 0, + 33, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, @@ -4290,28 +4528,35 @@ static const ::capnp::_::AlignedData<65> b_dac6a7f675c57409 = { 105, 111, 110, 67, 108, 97, 117, 115, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, - 12, 0, 0, 0, 3, 0, 4, 0, + 16, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 69, 0, 0, 0, 82, 0, 0, 0, + 97, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 68, 0, 0, 0, 3, 0, 1, 0, - 80, 0, 0, 0, 2, 0, 1, 0, + 96, 0, 0, 0, 3, 0, 1, 0, + 108, 0, 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 77, 0, 0, 0, 50, 0, 0, 0, + 105, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 72, 0, 0, 0, 3, 0, 1, 0, - 84, 0, 0, 0, 2, 0, 1, 0, + 100, 0, 0, 0, 3, 0, 1, 0, + 112, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 81, 0, 0, 0, 26, 0, 0, 0, + 109, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 76, 0, 0, 0, 3, 0, 1, 0, - 88, 0, 0, 0, 2, 0, 1, 0, + 104, 0, 0, 0, 3, 0, 1, 0, + 116, 0, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 113, 0, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 112, 0, 0, 0, 3, 0, 1, 0, + 124, 0, 0, 0, 2, 0, 1, 0, 102, 105, 101, 108, 100, 78, 97, 109, 101, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, @@ -4335,19 +4580,28 @@ static const ::capnp::_::AlignedData<65> b_dac6a7f675c57409 = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 117, 115, 101, 69, 110, 117, 109, 101, + 114, 97, 116, 105, 111, 110, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, } }; ::capnp::word const* const bp_dac6a7f675c57409 = b_dac6a7f675c57409.words; #if !CAPNP_LITE -static const uint16_t m_dac6a7f675c57409[] = {0, 2, 1}; -static const uint16_t i_dac6a7f675c57409[] = {0, 1, 2}; +static const uint16_t m_dac6a7f675c57409[] = {0, 2, 3, 1}; +static const uint16_t i_dac6a7f675c57409[] = {0, 1, 2, 3}; const ::capnp::_::RawSchema s_dac6a7f675c57409 = { - 0xdac6a7f675c57409, b_dac6a7f675c57409.words, 65, nullptr, m_dac6a7f675c57409, - 0, 3, i_dac6a7f675c57409, nullptr, nullptr, { &s_dac6a7f675c57409, nullptr, nullptr, 0, 0, nullptr } + 0xdac6a7f675c57409, b_dac6a7f675c57409.words, 81, nullptr, m_dac6a7f675c57409, + 0, 4, i_dac6a7f675c57409, nullptr, nullptr, { &s_dac6a7f675c57409, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<116> b_afc739d5c01e6496 = { +static const ::capnp::_::AlignedData<132> b_afc739d5c01e6496 = { { 0, 0, 0, 0, 5, 0, 6, 0, 150, 100, 30, 192, 213, 57, 199, 175, 18, 0, 0, 0, 1, 0, 1, 0, @@ -4357,7 +4611,7 @@ static const ::capnp::_::AlignedData<116> b_afc739d5c01e6496 = { 21, 0, 0, 0, 210, 0, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 0, 0, 0, 87, 1, 0, 0, + 29, 0, 0, 0, 143, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, @@ -4365,49 +4619,56 @@ static const ::capnp::_::AlignedData<116> b_afc739d5c01e6496 = { 112, 58, 65, 83, 84, 78, 111, 100, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, - 24, 0, 0, 0, 3, 0, 4, 0, + 28, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 153, 0, 0, 0, 106, 0, 0, 0, + 181, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 152, 0, 0, 0, 3, 0, 1, 0, - 164, 0, 0, 0, 2, 0, 1, 0, + 180, 0, 0, 0, 3, 0, 1, 0, + 192, 0, 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 161, 0, 0, 0, 82, 0, 0, 0, + 189, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 160, 0, 0, 0, 3, 0, 1, 0, - 172, 0, 0, 0, 2, 0, 1, 0, + 188, 0, 0, 0, 3, 0, 1, 0, + 200, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 169, 0, 0, 0, 50, 0, 0, 0, + 197, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 164, 0, 0, 0, 3, 0, 1, 0, - 176, 0, 0, 0, 2, 0, 1, 0, + 192, 0, 0, 0, 3, 0, 1, 0, + 204, 0, 0, 0, 2, 0, 1, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 173, 0, 0, 0, 26, 0, 0, 0, + 201, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 168, 0, 0, 0, 3, 0, 1, 0, - 180, 0, 0, 0, 2, 0, 1, 0, + 196, 0, 0, 0, 3, 0, 1, 0, + 208, 0, 0, 0, 2, 0, 1, 0, 4, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 177, 0, 0, 0, 74, 0, 0, 0, + 205, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 176, 0, 0, 0, 3, 0, 1, 0, - 204, 0, 0, 0, 2, 0, 1, 0, + 204, 0, 0, 0, 3, 0, 1, 0, + 232, 0, 0, 0, 2, 0, 1, 0, 5, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 201, 0, 0, 0, 114, 0, 0, 0, + 229, 0, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 200, 0, 0, 0, 3, 0, 1, 0, - 212, 0, 0, 0, 2, 0, 1, 0, + 228, 0, 0, 0, 3, 0, 1, 0, + 240, 0, 0, 0, 2, 0, 1, 0, + 6, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 6, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 237, 0, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 236, 0, 0, 0, 3, 0, 1, 0, + 248, 0, 0, 0, 2, 0, 1, 0, 105, 115, 69, 120, 112, 114, 101, 115, 115, 105, 111, 110, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, @@ -4462,6 +4723,15 @@ static const ::capnp::_::AlignedData<116> b_afc739d5c01e6496 = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 117, 115, 101, 69, 110, 117, 109, 101, + 114, 97, 116, 105, 111, 110, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, } }; @@ -4470,11 +4740,11 @@ static const ::capnp::_::AlignedData<116> b_afc739d5c01e6496 = { static const ::capnp::_::RawSchema* const d_afc739d5c01e6496[] = { &s_afc739d5c01e6496, }; -static const uint16_t m_afc739d5c01e6496[] = {4, 5, 1, 0, 3, 2}; -static const uint16_t i_afc739d5c01e6496[] = {0, 1, 2, 3, 4, 5}; +static const uint16_t m_afc739d5c01e6496[] = {4, 5, 1, 0, 3, 6, 2}; +static const uint16_t i_afc739d5c01e6496[] = {0, 1, 2, 3, 4, 5, 6}; const ::capnp::_::RawSchema s_afc739d5c01e6496 = { - 0xafc739d5c01e6496, b_afc739d5c01e6496.words, 116, d_afc739d5c01e6496, m_afc739d5c01e6496, - 1, 6, i_afc739d5c01e6496, nullptr, nullptr, { &s_afc739d5c01e6496, nullptr, nullptr, 0, 0, nullptr } + 0xafc739d5c01e6496, b_afc739d5c01e6496.words, 132, d_afc739d5c01e6496, m_afc739d5c01e6496, + 1, 7, i_afc739d5c01e6496, nullptr, nullptr, { &s_afc739d5c01e6496, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE static const ::capnp::_::AlignedData<73> b_eaf57cb9871fc06f = { @@ -9010,6 +9280,14 @@ constexpr ::capnp::Kind Attribute::_capnpPrivate::kind; constexpr ::capnp::_::RawSchema const* Attribute::_capnpPrivate::schema; #endif // !CAPNP_LITE +// Enumeration +constexpr uint16_t Enumeration::_capnpPrivate::dataWordSize; +constexpr uint16_t Enumeration::_capnpPrivate::pointerCount; +#if !CAPNP_LITE +constexpr ::capnp::Kind Enumeration::_capnpPrivate::kind; +constexpr ::capnp::_::RawSchema const* Enumeration::_capnpPrivate::schema; +#endif // !CAPNP_LITE + // AttributeBufferHeader constexpr uint16_t AttributeBufferHeader::_capnpPrivate::dataWordSize; constexpr uint16_t AttributeBufferHeader::_capnpPrivate::pointerCount; diff --git a/tiledb/sm/serialization/win32/tiledb-rest.capnp.h b/tiledb/sm/serialization/win32/tiledb-rest.capnp.h index 260cbaa58fb9..05ccbc0a6b37 100644 --- a/tiledb/sm/serialization/win32/tiledb-rest.capnp.h +++ b/tiledb/sm/serialization/win32/tiledb-rest.capnp.h @@ -23,6 +23,7 @@ CAPNP_DECLARE_SCHEMA(d71de32f98e296fe); CAPNP_DECLARE_SCHEMA(ceff8d62d10cd1de); CAPNP_DECLARE_SCHEMA(a1b81d67548230d4); CAPNP_DECLARE_SCHEMA(92ad78f56de3d76a); +CAPNP_DECLARE_SCHEMA(d00b2f19cc21b9b4); CAPNP_DECLARE_SCHEMA(d20a578112fa92a2); CAPNP_DECLARE_SCHEMA(95e26a84d32d8223); CAPNP_DECLARE_SCHEMA(a2a652536db09fa0); @@ -196,7 +197,7 @@ struct ArraySchema { class Pipeline; struct _capnpPrivate { - CAPNP_DECLARE_STRUCT_HEADER(d71de32f98e296fe, 2, 13) + CAPNP_DECLARE_STRUCT_HEADER(d71de32f98e296fe, 2, 15) #if !CAPNP_LITE static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; @@ -230,7 +231,7 @@ struct ArraySchemaEvolution { class Pipeline; struct _capnpPrivate { - CAPNP_DECLARE_STRUCT_HEADER(a1b81d67548230d4, 0, 3) + CAPNP_DECLARE_STRUCT_HEADER(a1b81d67548230d4, 0, 5) #if !CAPNP_LITE static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; @@ -247,7 +248,24 @@ struct Attribute { class Pipeline; struct _capnpPrivate { - CAPNP_DECLARE_STRUCT_HEADER(92ad78f56de3d76a, 1, 5) + CAPNP_DECLARE_STRUCT_HEADER(92ad78f56de3d76a, 1, 6) +#if !CAPNP_LITE + static constexpr ::capnp::_::RawBrandedSchema const* brand() { + return &schema->defaultBrand; + } +#endif // !CAPNP_LITE + }; +}; + +struct Enumeration { + Enumeration() = delete; + + class Reader; + class Builder; + class Pipeline; + + struct _capnpPrivate { + CAPNP_DECLARE_STRUCT_HEADER(d00b2f19cc21b9b4, 1, 5) #if !CAPNP_LITE static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; @@ -820,7 +838,7 @@ struct ConditionClause { class Pipeline; struct _capnpPrivate { - CAPNP_DECLARE_STRUCT_HEADER(dac6a7f675c57409, 0, 3) + CAPNP_DECLARE_STRUCT_HEADER(dac6a7f675c57409, 1, 3) #if !CAPNP_LITE static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; @@ -2441,6 +2459,18 @@ class ArraySchema::Reader { ::capnp::Kind::STRUCT>::Reader getDimensionLabels() const; + inline bool hasEnumerations() const; + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader + getEnumerations() const; + + inline bool hasEnumerationPathMap() const; + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>::Reader + getEnumerationPathMap() const; + private: ::capnp::_::StructReader _reader; template @@ -2640,6 +2670,48 @@ class ArraySchema::Builder { ::capnp::Kind::STRUCT>> disownDimensionLabels(); + inline bool hasEnumerations(); + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder + getEnumerations(); + inline void setEnumerations(::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader value); + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder + initEnumerations(unsigned int size); + inline void adoptEnumerations( + ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>&& value); + inline ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>> + disownEnumerations(); + + inline bool hasEnumerationPathMap(); + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>::Builder + getEnumerationPathMap(); + inline void setEnumerationPathMap(::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>::Reader value); + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>::Builder + initEnumerationPathMap(unsigned int size); + inline void adoptEnumerationPathMap( + ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>>&& value); + inline ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>> + disownEnumerationPathMap(); + private: ::capnp::_::StructBuilder _builder; template @@ -2883,6 +2955,16 @@ class ArraySchemaEvolution::Reader { inline ::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>::Reader getTimestampRange() const; + inline bool hasEnumerationsToAdd() const; + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader + getEnumerationsToAdd() const; + + inline bool hasEnumerationsToDrop() const; + inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Reader + getEnumerationsToDrop() const; + private: ::capnp::_::StructReader _reader; template @@ -2972,6 +3054,43 @@ class ArraySchemaEvolution::Builder { inline ::capnp::Orphan<::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>> disownTimestampRange(); + inline bool hasEnumerationsToAdd(); + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder + getEnumerationsToAdd(); + inline void setEnumerationsToAdd( + ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader value); + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder + initEnumerationsToAdd(unsigned int size); + inline void adoptEnumerationsToAdd( + ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>&& value); + inline ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>> + disownEnumerationsToAdd(); + + inline bool hasEnumerationsToDrop(); + inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Builder + getEnumerationsToDrop(); + inline void setEnumerationsToDrop( + ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Reader value); + inline void setEnumerationsToDrop( + ::kj::ArrayPtr value); + inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Builder + initEnumerationsToDrop(unsigned int size); + inline void adoptEnumerationsToDrop( + ::capnp::Orphan<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>&& + value); + inline ::capnp::Orphan<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>> + disownEnumerationsToDrop(); + private: ::capnp::_::StructBuilder _builder; template @@ -3042,6 +3161,9 @@ class Attribute::Reader { inline bool hasOrder() const; inline ::capnp::Text::Reader getOrder() const; + inline bool hasEnumerationName() const; + inline ::capnp::Text::Reader getEnumerationName() const; + private: ::capnp::_::StructReader _reader; template @@ -3131,6 +3253,13 @@ class Attribute::Builder { inline void adoptOrder(::capnp::Orphan<::capnp::Text>&& value); inline ::capnp::Orphan<::capnp::Text> disownOrder(); + inline bool hasEnumerationName(); + inline ::capnp::Text::Builder getEnumerationName(); + inline void setEnumerationName(::capnp::Text::Reader value); + inline ::capnp::Text::Builder initEnumerationName(unsigned int size); + inline void adoptEnumerationName(::capnp::Orphan<::capnp::Text>&& value); + inline ::capnp::Orphan<::capnp::Text> disownEnumerationName(); + private: ::capnp::_::StructBuilder _builder; template @@ -3163,6 +3292,153 @@ class Attribute::Pipeline { }; #endif // !CAPNP_LITE +class Enumeration::Reader { + public: + typedef Enumeration Reads; + + Reader() = default; + inline explicit Reader(::capnp::_::StructReader base) + : _reader(base) { + } + + inline ::capnp::MessageSize totalSize() const { + return _reader.totalSize().asPublic(); + } + +#if !CAPNP_LITE + inline ::kj::StringTree toString() const { + return ::capnp::_::structString(_reader, *_capnpPrivate::brand()); + } +#endif // !CAPNP_LITE + + inline bool hasName() const; + inline ::capnp::Text::Reader getName() const; + + inline bool hasPathName() const; + inline ::capnp::Text::Reader getPathName() const; + + inline bool hasType() const; + inline ::capnp::Text::Reader getType() const; + + inline ::uint32_t getCellValNum() const; + + inline bool getOrdered() const; + + inline bool hasData() const; + inline ::capnp::Data::Reader getData() const; + + inline bool hasOffsets() const; + inline ::capnp::Data::Reader getOffsets() const; + + private: + ::capnp::_::StructReader _reader; + template + friend struct ::capnp::ToDynamic_; + template + friend struct ::capnp::_::PointerHelpers; + template + friend struct ::capnp::List; + friend class ::capnp::MessageBuilder; + friend class ::capnp::Orphanage; +}; + +class Enumeration::Builder { + public: + typedef Enumeration Builds; + + Builder() = delete; // Deleted to discourage incorrect usage. + // You can explicitly initialize to nullptr instead. + inline Builder(decltype(nullptr)) { + } + inline explicit Builder(::capnp::_::StructBuilder base) + : _builder(base) { + } + inline operator Reader() const { + return Reader(_builder.asReader()); + } + inline Reader asReader() const { + return *this; + } + + inline ::capnp::MessageSize totalSize() const { + return asReader().totalSize(); + } +#if !CAPNP_LITE + inline ::kj::StringTree toString() const { + return asReader().toString(); + } +#endif // !CAPNP_LITE + + inline bool hasName(); + inline ::capnp::Text::Builder getName(); + inline void setName(::capnp::Text::Reader value); + inline ::capnp::Text::Builder initName(unsigned int size); + inline void adoptName(::capnp::Orphan<::capnp::Text>&& value); + inline ::capnp::Orphan<::capnp::Text> disownName(); + + inline bool hasPathName(); + inline ::capnp::Text::Builder getPathName(); + inline void setPathName(::capnp::Text::Reader value); + inline ::capnp::Text::Builder initPathName(unsigned int size); + inline void adoptPathName(::capnp::Orphan<::capnp::Text>&& value); + inline ::capnp::Orphan<::capnp::Text> disownPathName(); + + inline bool hasType(); + inline ::capnp::Text::Builder getType(); + inline void setType(::capnp::Text::Reader value); + inline ::capnp::Text::Builder initType(unsigned int size); + inline void adoptType(::capnp::Orphan<::capnp::Text>&& value); + inline ::capnp::Orphan<::capnp::Text> disownType(); + + inline ::uint32_t getCellValNum(); + inline void setCellValNum(::uint32_t value); + + inline bool getOrdered(); + inline void setOrdered(bool value); + + inline bool hasData(); + inline ::capnp::Data::Builder getData(); + inline void setData(::capnp::Data::Reader value); + inline ::capnp::Data::Builder initData(unsigned int size); + inline void adoptData(::capnp::Orphan<::capnp::Data>&& value); + inline ::capnp::Orphan<::capnp::Data> disownData(); + + inline bool hasOffsets(); + inline ::capnp::Data::Builder getOffsets(); + inline void setOffsets(::capnp::Data::Reader value); + inline ::capnp::Data::Builder initOffsets(unsigned int size); + inline void adoptOffsets(::capnp::Orphan<::capnp::Data>&& value); + inline ::capnp::Orphan<::capnp::Data> disownOffsets(); + + private: + ::capnp::_::StructBuilder _builder; + template + friend struct ::capnp::ToDynamic_; + friend class ::capnp::Orphanage; + template + friend struct ::capnp::_::PointerHelpers; +}; + +#if !CAPNP_LITE +class Enumeration::Pipeline { + public: + typedef Enumeration Pipelines; + + inline Pipeline(decltype(nullptr)) + : _typeless(nullptr) { + } + inline explicit Pipeline(::capnp::AnyPointer::Pipeline&& typeless) + : _typeless(kj::mv(typeless)) { + } + + private: + ::capnp::AnyPointer::Pipeline _typeless; + friend class ::capnp::PipelineHook; + template + friend struct ::capnp::ToDynamic_; +}; +#endif // !CAPNP_LITE + class AttributeBufferHeader::Reader { public: typedef AttributeBufferHeader Reads; @@ -7193,6 +7469,8 @@ class ConditionClause::Reader { inline bool hasOp() const; inline ::capnp::Text::Reader getOp() const; + inline bool getUseEnumeration() const; + private: ::capnp::_::StructReader _reader; template @@ -7253,6 +7531,9 @@ class ConditionClause::Builder { inline void adoptOp(::capnp::Orphan<::capnp::Text>&& value); inline ::capnp::Orphan<::capnp::Text> disownOp(); + inline bool getUseEnumeration(); + inline void setUseEnumeration(bool value); + private: ::capnp::_::StructBuilder _builder; template @@ -7321,6 +7602,8 @@ class ASTNode::Reader { inline bool hasCombinationOp() const; inline ::capnp::Text::Reader getCombinationOp() const; + inline bool getUseEnumeration() const; + private: ::capnp::_::StructReader _reader; template @@ -7411,6 +7694,9 @@ class ASTNode::Builder { inline void adoptCombinationOp(::capnp::Orphan<::capnp::Text>&& value); inline ::capnp::Orphan<::capnp::Text> disownCombinationOp(); + inline bool getUseEnumeration(); + inline void setUseEnumeration(bool value); + private: ::capnp::_::StructBuilder _builder; template @@ -15475,54 +15761,199 @@ ArraySchema::Builder::disownDimensionLabels() { ::capnp::POINTERS)); } -inline ::uint32_t DimensionLabel::Reader::getDimensionId() const { - return _reader.getDataField<::uint32_t>( - ::capnp::bounded<0>() * ::capnp::ELEMENTS); -} - -inline ::uint32_t DimensionLabel::Builder::getDimensionId() { - return _builder.getDataField<::uint32_t>( - ::capnp::bounded<0>() * ::capnp::ELEMENTS); -} -inline void DimensionLabel::Builder::setDimensionId(::uint32_t value) { - _builder.setDataField<::uint32_t>( - ::capnp::bounded<0>() * ::capnp::ELEMENTS, value); -} - -inline bool DimensionLabel::Reader::hasName() const { - return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) +inline bool ArraySchema::Reader::hasEnumerations() const { + return !_reader.getPointerField(::capnp::bounded<13>() * ::capnp::POINTERS) .isNull(); } -inline bool DimensionLabel::Builder::hasName() { - return !_builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) +inline bool ArraySchema::Builder::hasEnumerations() { + return !_builder.getPointerField(::capnp::bounded<13>() * ::capnp::POINTERS) .isNull(); } -inline ::capnp::Text::Reader DimensionLabel::Reader::getName() const { - return ::capnp::_::PointerHelpers<::capnp::Text>::get( - _reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); -} -inline ::capnp::Text::Builder DimensionLabel::Builder::getName() { - return ::capnp::_::PointerHelpers<::capnp::Text>::get( - _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader +ArraySchema::Reader::getEnumerations() const { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>::get(_reader + .getPointerField( + ::capnp::bounded<13>() * + ::capnp::POINTERS)); } -inline void DimensionLabel::Builder::setName(::capnp::Text::Reader value) { - ::capnp::_::PointerHelpers<::capnp::Text>::set( - _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), - value); +inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder +ArraySchema::Builder::getEnumerations() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>::get(_builder + .getPointerField( + ::capnp::bounded<13>() * + ::capnp::POINTERS)); } -inline ::capnp::Text::Builder DimensionLabel::Builder::initName( - unsigned int size) { - return ::capnp::_::PointerHelpers<::capnp::Text>::init( - _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), - size); +inline void ArraySchema::Builder::setEnumerations( + ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>:: + set(_builder.getPointerField(::capnp::bounded<13>() * ::capnp::POINTERS), + value); } -inline void DimensionLabel::Builder::adoptName( - ::capnp::Orphan<::capnp::Text>&& value) { - ::capnp::_::PointerHelpers<::capnp::Text>::adopt( - _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), - kj::mv(value)); +inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder +ArraySchema::Builder::initEnumerations(unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>:: + init( + _builder.getPointerField(::capnp::bounded<13>() * ::capnp::POINTERS), + size); } -inline ::capnp::Orphan<::capnp::Text> DimensionLabel::Builder::disownName() { +inline void ArraySchema::Builder::adoptEnumerations( + ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>&& value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>:: + adopt( + _builder.getPointerField(::capnp::bounded<13>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>> +ArraySchema::Builder::disownEnumerations() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>::disown(_builder + .getPointerField( + ::capnp::bounded<13>() * + ::capnp::POINTERS)); +} + +inline bool ArraySchema::Reader::hasEnumerationPathMap() const { + return !_reader.getPointerField(::capnp::bounded<14>() * ::capnp::POINTERS) + .isNull(); +} +inline bool ArraySchema::Builder::hasEnumerationPathMap() { + return !_builder.getPointerField(::capnp::bounded<14>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp:: + List<::tiledb::sm::serialization::capnp::KV, ::capnp::Kind::STRUCT>::Reader + ArraySchema::Reader::getEnumerationPathMap() const { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>>::get(_reader + .getPointerField( + ::capnp::bounded<14>() * + ::capnp::POINTERS)); +} +inline ::capnp:: + List<::tiledb::sm::serialization::capnp::KV, ::capnp::Kind::STRUCT>::Builder + ArraySchema::Builder::getEnumerationPathMap() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>>::get(_builder + .getPointerField( + ::capnp::bounded<14>() * + ::capnp::POINTERS)); +} +inline void ArraySchema::Builder::setEnumerationPathMap( + ::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>::Reader value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>>:: + set(_builder.getPointerField(::capnp::bounded<14>() * ::capnp::POINTERS), + value); +} +inline ::capnp:: + List<::tiledb::sm::serialization::capnp::KV, ::capnp::Kind::STRUCT>::Builder + ArraySchema::Builder::initEnumerationPathMap(unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>>:: + init( + _builder.getPointerField(::capnp::bounded<14>() * ::capnp::POINTERS), + size); +} +inline void ArraySchema::Builder::adoptEnumerationPathMap( + ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>>&& value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>>:: + adopt( + _builder.getPointerField(::capnp::bounded<14>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>> +ArraySchema::Builder::disownEnumerationPathMap() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>>::disown(_builder + .getPointerField( + ::capnp::bounded<14>() * + ::capnp::POINTERS)); +} + +inline ::uint32_t DimensionLabel::Reader::getDimensionId() const { + return _reader.getDataField<::uint32_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS); +} + +inline ::uint32_t DimensionLabel::Builder::getDimensionId() { + return _builder.getDataField<::uint32_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS); +} +inline void DimensionLabel::Builder::setDimensionId(::uint32_t value) { + _builder.setDataField<::uint32_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS, value); +} + +inline bool DimensionLabel::Reader::hasName() const { + return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) + .isNull(); +} +inline bool DimensionLabel::Builder::hasName() { + return !_builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::Text::Reader DimensionLabel::Reader::getName() const { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder DimensionLabel::Builder::getName() { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline void DimensionLabel::Builder::setName(::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers<::capnp::Text>::set( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + value); +} +inline ::capnp::Text::Builder DimensionLabel::Builder::initName( + unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::Text>::init( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + size); +} +inline void DimensionLabel::Builder::adoptName( + ::capnp::Orphan<::capnp::Text>&& value) { + ::capnp::_::PointerHelpers<::capnp::Text>::adopt( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::Text> DimensionLabel::Builder::disownName() { return ::capnp::_::PointerHelpers<::capnp::Text>::disown( _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); } @@ -15962,6 +16393,136 @@ ArraySchemaEvolution::Builder::disownTimestampRange() { _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS)); } +inline bool ArraySchemaEvolution::Reader::hasEnumerationsToAdd() const { + return !_reader.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS) + .isNull(); +} +inline bool ArraySchemaEvolution::Builder::hasEnumerationsToAdd() { + return !_builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader +ArraySchemaEvolution::Reader::getEnumerationsToAdd() const { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>::get(_reader + .getPointerField( + ::capnp::bounded<3>() * + ::capnp::POINTERS)); +} +inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder +ArraySchemaEvolution::Builder::getEnumerationsToAdd() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>::get(_builder + .getPointerField( + ::capnp::bounded<3>() * + ::capnp::POINTERS)); +} +inline void ArraySchemaEvolution::Builder::setEnumerationsToAdd( + ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>:: + set(_builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS), + value); +} +inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder +ArraySchemaEvolution::Builder::initEnumerationsToAdd(unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>:: + init( + _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS), + size); +} +inline void ArraySchemaEvolution::Builder::adoptEnumerationsToAdd( + ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>&& value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>:: + adopt( + _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>> +ArraySchemaEvolution::Builder::disownEnumerationsToAdd() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>::disown(_builder + .getPointerField( + ::capnp::bounded<3>() * + ::capnp::POINTERS)); +} + +inline bool ArraySchemaEvolution::Reader::hasEnumerationsToDrop() const { + return !_reader.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS) + .isNull(); +} +inline bool ArraySchemaEvolution::Builder::hasEnumerationsToDrop() { + return !_builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Reader +ArraySchemaEvolution::Reader::getEnumerationsToDrop() const { + return ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::get( + _reader.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS)); +} +inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Builder +ArraySchemaEvolution::Builder::getEnumerationsToDrop() { + return ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::get( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS)); +} +inline void ArraySchemaEvolution::Builder::setEnumerationsToDrop( + ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Reader value) { + ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::set( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS), + value); +} +inline void ArraySchemaEvolution::Builder::setEnumerationsToDrop( + ::kj::ArrayPtr value) { + ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::set( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS), + value); +} +inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Builder +ArraySchemaEvolution::Builder::initEnumerationsToDrop(unsigned int size) { + return ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::init( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS), + size); +} +inline void ArraySchemaEvolution::Builder::adoptEnumerationsToDrop( + ::capnp::Orphan<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>&& + value) { + ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::adopt( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>> +ArraySchemaEvolution::Builder::disownEnumerationsToDrop() { + return ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::disown( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS)); +} + inline ::uint32_t Attribute::Reader::getCellValNum() const { return _reader.getDataField<::uint32_t>( ::capnp::bounded<0>() * ::capnp::ELEMENTS); @@ -16207,6 +16768,263 @@ inline ::capnp::Orphan<::capnp::Text> Attribute::Builder::disownOrder() { _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS)); } +inline bool Attribute::Reader::hasEnumerationName() const { + return !_reader.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS) + .isNull(); +} +inline bool Attribute::Builder::hasEnumerationName() { + return !_builder.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::Text::Reader Attribute::Reader::getEnumerationName() const { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _reader.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder Attribute::Builder::getEnumerationName() { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _builder.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS)); +} +inline void Attribute::Builder::setEnumerationName( + ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers<::capnp::Text>::set( + _builder.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS), + value); +} +inline ::capnp::Text::Builder Attribute::Builder::initEnumerationName( + unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::Text>::init( + _builder.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS), + size); +} +inline void Attribute::Builder::adoptEnumerationName( + ::capnp::Orphan<::capnp::Text>&& value) { + ::capnp::_::PointerHelpers<::capnp::Text>::adopt( + _builder.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::Text> +Attribute::Builder::disownEnumerationName() { + return ::capnp::_::PointerHelpers<::capnp::Text>::disown( + _builder.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS)); +} + +inline bool Enumeration::Reader::hasName() const { + return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) + .isNull(); +} +inline bool Enumeration::Builder::hasName() { + return !_builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::Text::Reader Enumeration::Reader::getName() const { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder Enumeration::Builder::getName() { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline void Enumeration::Builder::setName(::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers<::capnp::Text>::set( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + value); +} +inline ::capnp::Text::Builder Enumeration::Builder::initName( + unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::Text>::init( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + size); +} +inline void Enumeration::Builder::adoptName( + ::capnp::Orphan<::capnp::Text>&& value) { + ::capnp::_::PointerHelpers<::capnp::Text>::adopt( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::Text> Enumeration::Builder::disownName() { + return ::capnp::_::PointerHelpers<::capnp::Text>::disown( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} + +inline bool Enumeration::Reader::hasPathName() const { + return !_reader.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS) + .isNull(); +} +inline bool Enumeration::Builder::hasPathName() { + return !_builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::Text::Reader Enumeration::Reader::getPathName() const { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _reader.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder Enumeration::Builder::getPathName() { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); +} +inline void Enumeration::Builder::setPathName(::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers<::capnp::Text>::set( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), + value); +} +inline ::capnp::Text::Builder Enumeration::Builder::initPathName( + unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::Text>::init( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), + size); +} +inline void Enumeration::Builder::adoptPathName( + ::capnp::Orphan<::capnp::Text>&& value) { + ::capnp::_::PointerHelpers<::capnp::Text>::adopt( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::Text> Enumeration::Builder::disownPathName() { + return ::capnp::_::PointerHelpers<::capnp::Text>::disown( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); +} + +inline bool Enumeration::Reader::hasType() const { + return !_reader.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS) + .isNull(); +} +inline bool Enumeration::Builder::hasType() { + return !_builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::Text::Reader Enumeration::Reader::getType() const { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _reader.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder Enumeration::Builder::getType() { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS)); +} +inline void Enumeration::Builder::setType(::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers<::capnp::Text>::set( + _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS), + value); +} +inline ::capnp::Text::Builder Enumeration::Builder::initType( + unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::Text>::init( + _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS), + size); +} +inline void Enumeration::Builder::adoptType( + ::capnp::Orphan<::capnp::Text>&& value) { + ::capnp::_::PointerHelpers<::capnp::Text>::adopt( + _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::Text> Enumeration::Builder::disownType() { + return ::capnp::_::PointerHelpers<::capnp::Text>::disown( + _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS)); +} + +inline ::uint32_t Enumeration::Reader::getCellValNum() const { + return _reader.getDataField<::uint32_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS); +} + +inline ::uint32_t Enumeration::Builder::getCellValNum() { + return _builder.getDataField<::uint32_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS); +} +inline void Enumeration::Builder::setCellValNum(::uint32_t value) { + _builder.setDataField<::uint32_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS, value); +} + +inline bool Enumeration::Reader::getOrdered() const { + return _reader.getDataField(::capnp::bounded<32>() * ::capnp::ELEMENTS); +} + +inline bool Enumeration::Builder::getOrdered() { + return _builder.getDataField( + ::capnp::bounded<32>() * ::capnp::ELEMENTS); +} +inline void Enumeration::Builder::setOrdered(bool value) { + _builder.setDataField( + ::capnp::bounded<32>() * ::capnp::ELEMENTS, value); +} + +inline bool Enumeration::Reader::hasData() const { + return !_reader.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS) + .isNull(); +} +inline bool Enumeration::Builder::hasData() { + return !_builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::Data::Reader Enumeration::Reader::getData() const { + return ::capnp::_::PointerHelpers<::capnp::Data>::get( + _reader.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS)); +} +inline ::capnp::Data::Builder Enumeration::Builder::getData() { + return ::capnp::_::PointerHelpers<::capnp::Data>::get( + _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS)); +} +inline void Enumeration::Builder::setData(::capnp::Data::Reader value) { + ::capnp::_::PointerHelpers<::capnp::Data>::set( + _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS), + value); +} +inline ::capnp::Data::Builder Enumeration::Builder::initData( + unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::Data>::init( + _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS), + size); +} +inline void Enumeration::Builder::adoptData( + ::capnp::Orphan<::capnp::Data>&& value) { + ::capnp::_::PointerHelpers<::capnp::Data>::adopt( + _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::Data> Enumeration::Builder::disownData() { + return ::capnp::_::PointerHelpers<::capnp::Data>::disown( + _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS)); +} + +inline bool Enumeration::Reader::hasOffsets() const { + return !_reader.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS) + .isNull(); +} +inline bool Enumeration::Builder::hasOffsets() { + return !_builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::Data::Reader Enumeration::Reader::getOffsets() const { + return ::capnp::_::PointerHelpers<::capnp::Data>::get( + _reader.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS)); +} +inline ::capnp::Data::Builder Enumeration::Builder::getOffsets() { + return ::capnp::_::PointerHelpers<::capnp::Data>::get( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS)); +} +inline void Enumeration::Builder::setOffsets(::capnp::Data::Reader value) { + ::capnp::_::PointerHelpers<::capnp::Data>::set( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS), + value); +} +inline ::capnp::Data::Builder Enumeration::Builder::initOffsets( + unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::Data>::init( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS), + size); +} +inline void Enumeration::Builder::adoptOffsets( + ::capnp::Orphan<::capnp::Data>&& value) { + ::capnp::_::PointerHelpers<::capnp::Data>::adopt( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::Data> Enumeration::Builder::disownOffsets() { + return ::capnp::_::PointerHelpers<::capnp::Data>::disown( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS)); +} + inline bool AttributeBufferHeader::Reader::hasName() const { return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) .isNull(); @@ -20752,6 +21570,17 @@ inline ::capnp::Orphan<::capnp::Text> ConditionClause::Builder::disownOp() { _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS)); } +inline bool ConditionClause::Reader::getUseEnumeration() const { + return _reader.getDataField(::capnp::bounded<0>() * ::capnp::ELEMENTS); +} + +inline bool ConditionClause::Builder::getUseEnumeration() { + return _builder.getDataField(::capnp::bounded<0>() * ::capnp::ELEMENTS); +} +inline void ConditionClause::Builder::setUseEnumeration(bool value) { + _builder.setDataField(::capnp::bounded<0>() * ::capnp::ELEMENTS, value); +} + inline bool ASTNode::Reader::getIsExpression() const { return _reader.getDataField(::capnp::bounded<0>() * ::capnp::ELEMENTS); } @@ -20986,6 +21815,17 @@ inline ::capnp::Orphan<::capnp::Text> ASTNode::Builder::disownCombinationOp() { _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS)); } +inline bool ASTNode::Reader::getUseEnumeration() const { + return _reader.getDataField(::capnp::bounded<1>() * ::capnp::ELEMENTS); +} + +inline bool ASTNode::Builder::getUseEnumeration() { + return _builder.getDataField(::capnp::bounded<1>() * ::capnp::ELEMENTS); +} +inline void ASTNode::Builder::setUseEnumeration(bool value) { + _builder.setDataField(::capnp::bounded<1>() * ::capnp::ELEMENTS, value); +} + inline bool Condition::Reader::hasClauses() const { return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) .isNull(); diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index 5b89809704a1..369ecb4a8047 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -46,6 +46,7 @@ #include "tiledb/sm/array/array_directory.h" #include "tiledb/sm/array_schema/array_schema.h" #include "tiledb/sm/array_schema/array_schema_evolution.h" +#include "tiledb/sm/array_schema/enumeration.h" #include "tiledb/sm/consolidator/consolidator.h" #include "tiledb/sm/consolidator/fragment_consolidator.h" #include "tiledb/sm/enums/array_type.h" @@ -630,6 +631,11 @@ Status StorageManagerCanonical::array_create( array_uri.join_path(constants::array_schema_dir_name); RETURN_NOT_OK(vfs()->create_dir(array_schema_dir_uri)); + // Create the enumerations directory inside the array schema directory + URI array_enumerations_uri = + array_schema_dir_uri.join_path(constants::array_enumerations_dir_name); + RETURN_NOT_OK(vfs()->create_dir(array_enumerations_uri)); + // Create commit directory URI array_commit_uri = array_uri.join_path(constants::array_commits_dir_name); RETURN_NOT_OK(vfs()->create_dir(array_commit_uri)); @@ -736,11 +742,9 @@ Status StorageManager::array_evolve_schema( auto&& array_schema = array_dir.load_array_schema_latest(encryption_key); // Evolve schema - auto&& [st1, array_schema_evolved] = - schema_evolution->evolve_schema(array_schema); - RETURN_NOT_OK(st1); + auto array_schema_evolved = schema_evolution->evolve_schema(array_schema); - Status st = store_array_schema(array_schema_evolved.value(), encryption_key); + Status st = store_array_schema(array_schema_evolved, encryption_key); if (!st.ok()) { logger_->status_no_return_value(st); return logger_->status(Status_StorageManagerError( @@ -1692,11 +1696,46 @@ Status StorageManagerCanonical::store_array_schema( URI array_schema_dir_uri = array_schema->array_uri().join_path(constants::array_schema_dir_name); RETURN_NOT_OK(vfs()->is_dir(array_schema_dir_uri, &schema_dir_exists)); + if (!schema_dir_exists) RETURN_NOT_OK(vfs()->create_dir(array_schema_dir_uri)); RETURN_NOT_OK(store_data_to_generic_tile(tile, schema_uri, encryption_key)); + // Create the `__enumerations` directory under `__schema` if it doesn't + // exist. This might happen if someone tries to add an enumeration to an + // array created before version 19. + bool enumerations_dir_exists = false; + URI array_enumerations_dir_uri = + array_schema_dir_uri.join_path(constants::array_enumerations_dir_name); + RETURN_NOT_OK( + vfs()->is_dir(array_enumerations_dir_uri, &enumerations_dir_exists)); + + if (!enumerations_dir_exists) { + RETURN_NOT_OK(vfs()->create_dir(array_enumerations_dir_uri)); + } + + // Serialize all enumerations into the `__enumerations` directory + for (auto& enmr_name : array_schema->get_loaded_enumeration_names()) { + auto enmr = array_schema->get_enumeration(enmr_name); + if (enmr == nullptr) { + return logger_->status(Status_StorageManagerError( + "Error serializing enumeration; Loaded enumeration is null")); + } + + SizeComputationSerializer enumeration_size_serializer; + enmr->serialize(enumeration_size_serializer); + + WriterTile tile{ + WriterTile::from_generic(enumeration_size_serializer.size())}; + Serializer serializer(tile.data(), tile.size()); + enmr->serialize(serializer); + + auto abs_enmr_uri = array_enumerations_dir_uri.join_path(enmr->path_name()); + RETURN_NOT_OK( + store_data_to_generic_tile(tile, abs_enmr_uri, encryption_key)); + } + return Status::Ok(); } diff --git a/tiledb/sm/tile/CMakeLists.txt b/tiledb/sm/tile/CMakeLists.txt index c23f54bec7ff..f52ea816f9b0 100644 --- a/tiledb/sm/tile/CMakeLists.txt +++ b/tiledb/sm/tile/CMakeLists.txt @@ -44,7 +44,6 @@ commence(object_library generic_tile_io) baseline buffer constants - context_resources tiledb_crypto filter_pipeline tile @@ -52,4 +51,10 @@ commence(object_library generic_tile_io) ) conclude(object_library) +# This is linked outside the object_library scope because ContextResources +# is recompiled as part of the capi_context_stub. Including context_resources +# here like this prevents many headaches revolving around duplicate symbols +# when linking executables. +target_link_libraries(compile_generic_tile_io PRIVATE context_resources) + add_test_subdirectory() diff --git a/tiledb/storage_format/uri/CMakeLists.txt b/tiledb/storage_format/uri/CMakeLists.txt index 6923c26548d7..4dbd4dd3f319 100644 --- a/tiledb/storage_format/uri/CMakeLists.txt +++ b/tiledb/storage_format/uri/CMakeLists.txt @@ -30,6 +30,6 @@ include(object_library) # `uri_format` object library # commence(object_library uri_format) - this_target_sources(parse_uri.cc) + this_target_sources(parse_uri.cc generate_uri.cc) this_target_object_libraries(baseline time uuid vfs) conclude(object_library)