From 18ac027a10d9f02b5dafc74d7d0604f030050732 Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Thu, 6 Apr 2023 11:17:53 -0500 Subject: [PATCH] Add Enumerated Data Types This PR adds the Enumerated data types. Enumerated data types work by adding an Enumeration to the ArraySchema, setting an enumeration name on an attribute, and then adding the attribute to the ArraySchema. An Enumeration object contains a short list of options and a vector of values. An attribute that has an enumeration name set must have an integral type that is wide enough to index all of the enumerated values. Changes to the values of an enumeration (any of adding, renaming, or removing) can be accomplished via ArraySchemaEvolution. --- CMakeLists.txt | 6 +- PJD_TODO.md | 9 + examples/cpp_api/enumerations.cc | 160 ++ format_spec/FORMAT_SPEC.md | 2 +- format_spec/enumeration.md | 27 + test/CMakeLists.txt | 4 + test/src/unit-capi-array_schema.cc | 5 +- test/src/unit-capi-enumerations.cc | 119 + test/src/unit-cppapi-deletes.cc | 36 +- test/src/unit-cppapi-enumerations.cc | 501 +++++ test/src/unit-cppapi-schema-evolution.cc | 35 + test/src/unit-enum-helpers.cc | 63 + test/src/unit-enumerations.cc | 1995 +++++++++++++++++ tiledb/CMakeLists.txt | 21 +- tiledb/api/c_api/CMakeLists.txt | 3 + .../api/c_api/dimension_label/CMakeLists.txt | 1 + .../c_api/dimension_label/test/CMakeLists.txt | 1 + tiledb/api/c_api/enumeration/CMakeLists.txt | 40 + .../api/c_api/enumeration/enumeration_api.cc | 241 ++ .../enumeration_api_experimental.h | 281 +++ .../enumeration/enumeration_api_internal.h | 139 ++ .../api/c_api/enumeration/test/CMakeLists.txt | 32 + .../compile_capi_enumeration_stub_main.cc | 39 + .../enumeration/test/unit_capi_enumeration.cc | 471 ++++ tiledb/sm/array/array.cc | 42 + tiledb/sm/array/array.h | 25 + tiledb/sm/array/array_directory.cc | 51 +- tiledb/sm/array/array_directory.h | 23 + tiledb/sm/array/test/CMakeLists.txt | 2 +- tiledb/sm/array_schema/CMakeLists.txt | 25 +- tiledb/sm/array_schema/array_schema.cc | 291 ++- tiledb/sm/array_schema/array_schema.h | 127 ++ .../sm/array_schema/array_schema_evolution.cc | 161 +- .../sm/array_schema/array_schema_evolution.h | 67 +- tiledb/sm/array_schema/attribute.cc | 46 +- tiledb/sm/array_schema/attribute.h | 15 +- tiledb/sm/array_schema/enumeration.cc | 260 +++ tiledb/sm/array_schema/enumeration.h | 388 ++++ .../test/compile_enumeration_main.cc | 39 + tiledb/sm/c_api/tiledb.cc | 245 +- tiledb/sm/c_api/tiledb_experimental.h | 231 ++ tiledb/sm/cpp_api/array_experimental.h | 77 + tiledb/sm/cpp_api/array_schema_evolution.h | 44 +- tiledb/sm/cpp_api/array_schema_experimental.h | 32 + tiledb/sm/cpp_api/attribute_experimental.h | 94 + tiledb/sm/cpp_api/deleter.h | 4 + tiledb/sm/cpp_api/enumeration_experimental.h | 369 +++ .../sm/cpp_api/query_condition_experimental.h | 57 + tiledb/sm/cpp_api/tiledb_experimental | 4 + tiledb/sm/enums/datatype.h | 48 +- .../sm/filter/bit_width_reduction_filter.cc | 6 +- tiledb/sm/filter/positive_delta_filter.cc | 4 +- tiledb/sm/misc/constants.cc | 17 +- tiledb/sm/misc/constants.h | 13 +- tiledb/sm/misc/integral_type_casts.h | 144 ++ tiledb/sm/misc/test/CMakeLists.txt | 9 +- .../sm/misc/test/unit_integral_type_casts.cc | 511 +++++ tiledb/sm/misc/utils.cc | 1 + tiledb/sm/query/ast/CMakeLists.txt | 2 +- tiledb/sm/query/ast/query_ast.cc | 92 + tiledb/sm/query/ast/query_ast.h | 146 +- tiledb/sm/query/query.cc | 31 + tiledb/sm/query/query_condition.cc | 24 + tiledb/sm/query/query_condition.h | 29 + tiledb/sm/serialization/array_schema.cc | 129 +- tiledb/sm/serialization/array_schema.h | 19 + .../serialization/array_schema_evolution.cc | 86 +- .../serialization/posix/tiledb-rest.capnp.c++ | 566 +++-- .../serialization/posix/tiledb-rest.capnp.h | 928 +++++++- tiledb/sm/serialization/query.cc | 13 +- tiledb/sm/serialization/tiledb-rest.capnp | 45 + .../serialization/win32/tiledb-rest.capnp.c++ | 566 +++-- .../serialization/win32/tiledb-rest.capnp.h | 928 +++++++- tiledb/sm/storage_manager/storage_manager.cc | 47 +- tiledb/sm/tile/CMakeLists.txt | 7 +- tiledb/storage_format/uri/CMakeLists.txt | 2 +- 76 files changed, 10835 insertions(+), 528 deletions(-) create mode 100644 PJD_TODO.md create mode 100644 examples/cpp_api/enumerations.cc create mode 100644 format_spec/enumeration.md create mode 100644 test/src/unit-capi-enumerations.cc create mode 100644 test/src/unit-cppapi-enumerations.cc create mode 100644 test/src/unit-enum-helpers.cc create mode 100644 test/src/unit-enumerations.cc create mode 100644 tiledb/api/c_api/enumeration/CMakeLists.txt create mode 100644 tiledb/api/c_api/enumeration/enumeration_api.cc create mode 100644 tiledb/api/c_api/enumeration/enumeration_api_experimental.h create mode 100644 tiledb/api/c_api/enumeration/enumeration_api_internal.h create mode 100644 tiledb/api/c_api/enumeration/test/CMakeLists.txt create mode 100644 tiledb/api/c_api/enumeration/test/compile_capi_enumeration_stub_main.cc create mode 100644 tiledb/api/c_api/enumeration/test/unit_capi_enumeration.cc create mode 100644 tiledb/sm/array_schema/enumeration.cc create mode 100644 tiledb/sm/array_schema/enumeration.h create mode 100644 tiledb/sm/array_schema/test/compile_enumeration_main.cc create mode 100644 tiledb/sm/cpp_api/array_experimental.h create mode 100644 tiledb/sm/cpp_api/attribute_experimental.h create mode 100644 tiledb/sm/cpp_api/enumeration_experimental.h create mode 100644 tiledb/sm/cpp_api/query_condition_experimental.h create mode 100644 tiledb/sm/misc/integral_type_casts.h create mode 100644 tiledb/sm/misc/test/unit_integral_type_casts.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index 635ae3690ceb..38455849f61d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -227,7 +227,10 @@ else() "$<$:-O3>" "$<$:-g3;-ggdb3;-gdwarf-3>") if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") - add_compile_options("$<$:-fstandalone-debug>") + add_compile_options( + "$<$:-fstandalone-debug>" + "$<$:-fprofile-instr-generate;-fcoverage-mapping>") + add_link_options("$<$:--coverage;-fprofile-instr-generate;-fcoverage-mapping>") endif() # Disable newer Apple Clang warnings about unqualified calls to std::move @@ -344,6 +347,7 @@ list(APPEND TILEDB_C_API_RELATIVE_HEADERS "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/datatype/datatype_api_external.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/dimension/dimension_api_external.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/dimension_label/dimension_label_api_external.h" + "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/enumeration/enumeration_api_experimental.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/error/error_api_external.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/filesystem/filesystem_api_enum.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/filesystem/filesystem_api_external.h" diff --git a/PJD_TODO.md b/PJD_TODO.md new file mode 100644 index 000000000000..d67f3852a2c2 --- /dev/null +++ b/PJD_TODO.md @@ -0,0 +1,9 @@ +Chores Left +=== + +* NEED HELP + - New REST API endpoint for loading one or more enumerations + - Update get_array_schema_from_rest API endpoint to have a "?include_enumerations=true|false" QS parameter. + +* Miscellany + * Document enumeration format changes in the storage format docs diff --git a/examples/cpp_api/enumerations.cc b/examples/cpp_api/enumerations.cc new file mode 100644 index 000000000000..ea439d181c81 --- /dev/null +++ b/examples/cpp_api/enumerations.cc @@ -0,0 +1,160 @@ +/** + * @file enumerations.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * When run, this program will create a simple 2D sparse array with an + * enumeration and then use a query condition to select data based on the + * enumerations values. + */ + +#include +#include +#include + +using namespace tiledb; + +// Name of array. +std::string array_name("enumerations_example_array"); + +void create_array() { + Context ctx; + + // First some standard boiler plate for creating an array. Nothing here + // is important or required for Enumerations. + ArraySchema schema(ctx, TILEDB_SPARSE); + schema.set_order({{TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR}}); + + auto dim1 = Dimension::create(ctx, "rows", {{1, 4}}, 4); + auto dim2 = Dimension::create(ctx, "cols", {{1, 4}}, 4); + + Domain dom(ctx); + dom.add_dimensions(dim1, dim2); + schema.set_domain(dom); + + // The most basic enumeration only requires a name and a vector of values + // to use as lookups. Enumeration values can be any supported TileDB type + // although they are most commonly strings. + std::vector values = {"red", "green", "blue"}; + auto enmr = Enumeration::create(ctx, "colors", values); + + // To use an enumeration with an attribute, we just set the enumeration + // name on the attribute before adding it to the schema. Attributes that + // use an enumeration are required to have an integral type that is wide + // enough to index the entire enumeration. For instance, an enumeration with + // 256 values can fit in a uint8_t type, but at 257 values, the attribute + // would require a type of int16_t at a minimum. + auto attr = Attribute::create(ctx, "attr"); + AttributeExperimental::set_enumeration_name(ctx, attr, "colors"); + + // The enumeration must be added to the schema before any attribute that + // references the enumeration so that the requirements of the attribute + // can be accurately checked. + ArraySchemaExperimental::add_enumeration(ctx, schema, enmr); + + // Finally, we add the attribute as per normal. + schema.add_attribute(attr); + + Array::create(array_name, schema); +} + +void write_array() { + Context ctx; + + std::vector row_data = {1, 2, 2}; + std::vector col_data = {1, 4, 3}; + + // Attribute data for an enumeration is just numeric indices into the + // list of enumeration values. + std::vector attr_data = {2, 1, 1}; + + // Open the array for writing and create the query. + Array array(ctx, array_name, TILEDB_WRITE); + Query query(ctx, array); + query.set_layout(TILEDB_UNORDERED) + .set_data_buffer("rows", row_data) + .set_data_buffer("cols", col_data) + .set_data_buffer("attr", attr_data); + + // Write and close the array + query.submit(); + array.close(); +} + +void read_array() { + Context ctx; + + // This is all standard boiler plate for reading from an array. The + // section below will demonstrate using a QueryCondition to select + // rows based on the enumeration. + Array array(ctx, array_name, TILEDB_READ); + Subarray subarray(ctx, array); + subarray.add_range(0, 1, 4).add_range(1, 1, 4); + + std::vector row_data(16); + std::vector col_data(16); + std::vector attr_data(16); + + Query query(ctx, array, TILEDB_READ); + query.set_subarray(subarray) + .set_layout(TILEDB_ROW_MAJOR) + .set_data_buffer("rows", row_data) + .set_data_buffer("cols", col_data) + .set_data_buffer("attr", attr_data); + + // Query conditions apply against the enumeration's values instead of the + // integral data. Thus, we can select values here using color names instead + // of the integer indices. + QueryCondition qc(ctx); + qc.init("attr", "green", 5, TILEDB_EQ); + query.set_condition(qc); + + // Finally, submit and display the query results + query.submit(); + array.close(); + + // Print out the results. + auto result_num = (int)query.result_buffer_elements()["attr"].second; + for (int i = 0; i < result_num; i++) { + int r = row_data[i]; + int c = col_data[i]; + int a = attr_data[i]; + std::cout << "Cell (" << r << ", " << c << ") has attr " << a << "\n"; + } +} + +int main() { + Context ctx; + + if (Object::object(ctx, array_name).type() != Object::Type::Array) { + create_array(); + write_array(); + } + + read_array(); + return 0; +} diff --git a/format_spec/FORMAT_SPEC.md b/format_spec/FORMAT_SPEC.md index 42e16b15a986..ee0db16d584e 100644 --- a/format_spec/FORMAT_SPEC.md +++ b/format_spec/FORMAT_SPEC.md @@ -4,7 +4,7 @@ title: Format Specification **Notes:** -* The current TileDB format version number is **18** (`uint32_t`). +* The current TileDB format version number is **20** (`uint32_t`). * Data written by TileDB and referenced in this document is **little-endian** with the following exceptions: diff --git a/format_spec/enumeration.md b/format_spec/enumeration.md new file mode 100644 index 000000000000..a334a6632340 --- /dev/null +++ b/format_spec/enumeration.md @@ -0,0 +1,27 @@ +--- +title: Enumerations +--- + +## Main Structure + +``` +my_array # array folder + | ... + |_ schema # ArraySchema directory named `__schema` + |_ enumerations # Enumeration directory named `__enumerations` + |_ enumeration # enumeration data with names `__uuid_v` + + +Enumeration data is stored in a subdirectory of the [array schema][./array_schema.md] +directory. Enumerations are stored using [Generic Tiles][./generic_tile.md]. + +| **Field** | **Type** | **Description** | +| :--- | :--- | :--- | +| Version number | `uint32_t` | Enumerations version number | +| Datatype | `uint8_t` | The datatype of the enumeration values | +| Cell Val Num | `uint32_t` | The cell val num of the enumeration values | +| Ordered | `bool` | Whether the enumeration values should be considered ordered | +| Data Size | `uint64_t` | The number of bytes used to store the values | +| Data | `uint8_t` * Data Size | The data for the enumeration values | +| Offsets Size | `uint64_t` | The number of bytes used to store offsets if cell_var_num is TILEDB_VAR_NUM | +| Offsets | `uint8_t` * Offsets Size | The offsets data for the enumeration if cell_var_num is TILEDB_VAR_NUM | diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index c8fe3287492b..b58b15a78694 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -134,6 +134,7 @@ set(TILEDB_UNIT_TEST_SOURCES src/unit-capi-dense_neg.cc src/unit-capi-dense_vector.cc src/unit-capi-enum_values.cc + src/unit-capi-enumerations.cc src/unit-capi-error.cc src/unit-capi-filestore.cc src/unit-capi-fill_values.cc @@ -172,6 +173,8 @@ set(TILEDB_UNIT_TEST_SOURCES src/unit-dimension.cc src/unit-duplicates.cc src/unit-empty-var-length.cc + src/unit-enumerations.cc + src/unit-enum-helpers.cc src/unit-filter-buffer.cc src/unit-filter-pipeline.cc src/unit-global-order.cc @@ -213,6 +216,7 @@ if (TILEDB_CPP_API) src/unit-cppapi-deletes.cc src/unit-cppapi-dense-qc-coords-mode.cc src/unit-cppapi-time.cc + src/unit-cppapi-enumerations.cc src/unit-cppapi-fill_values.cc src/unit-cppapi-filter.cc src/unit-cppapi-float-scaling-filter.cc diff --git a/test/src/unit-capi-array_schema.cc b/test/src/unit-capi-array_schema.cc index 7a16a681a0ef..043e8b8bc471 100644 --- a/test/src/unit-capi-array_schema.cc +++ b/test/src/unit-capi-array_schema.cc @@ -959,7 +959,10 @@ int ArraySchemaFx::get_schema_file_struct(const char* path, void* data) { int rc = tiledb_vfs_is_dir(ctx, vfs, path, &is_dir); CHECK(rc == TILEDB_OK); - data_struct->path = path; + if (!is_dir) { + data_struct->path = path; + } + return 1; } diff --git a/test/src/unit-capi-enumerations.cc b/test/src/unit-capi-enumerations.cc new file mode 100644 index 000000000000..58add24ea2f0 --- /dev/null +++ b/test/src/unit-capi-enumerations.cc @@ -0,0 +1,119 @@ +/** + * @file unit-capi-enumerations.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * Tests for the various Enumerations C API errors. + */ + +#include +#include "tiledb/sm/c_api/tiledb.h" +#include "tiledb/sm/c_api/tiledb_experimental.h" + +#include + +TEST_CASE( + "C API: Test invalid attribute for tiledb_attribute_set_enumeration_name", + "[enumeration][capi][error]") { + tiledb_ctx_t* ctx; + + auto rc = tiledb_ctx_alloc(nullptr, &ctx); + CHECK(rc == TILEDB_OK); + + rc = tiledb_attribute_set_enumeration_name(ctx, nullptr, "enmr_name"); + REQUIRE(rc == TILEDB_ERR); +} + +TEST_CASE( + "C API: Test invalid attribute for tiledb_attribute_get_enumeration_name", + "[enumeration][capi][error]") { + tiledb_ctx_t* ctx; + + auto rc = tiledb_ctx_alloc(nullptr, &ctx); + CHECK(rc == TILEDB_OK); + + tiledb_string_t* name; + rc = tiledb_attribute_get_enumeration_name(ctx, nullptr, &name); + REQUIRE(rc == TILEDB_ERR); +} + +TEST_CASE( + "C API: Test invalid array schema for tiledb_array_schema_add_enumeration", + "[enumeration][capi][error]") { + tiledb_ctx_t* ctx; + + auto rc = tiledb_ctx_alloc(nullptr, &ctx); + CHECK(rc == TILEDB_OK); + + tiledb_enumeration_t* enmr; + uint32_t values[5] = {1, 2, 3, 4, 5}; + rc = tiledb_enumeration_alloc( + ctx, + "an_enumeration", + TILEDB_UINT32, + 1, + 0, + values, + sizeof(uint32_t) * 5, + nullptr, + 0, + &enmr); + REQUIRE(rc == TILEDB_OK); + + rc = tiledb_array_schema_add_enumeration(ctx, nullptr, enmr); + REQUIRE(rc == TILEDB_ERR); +} + +TEST_CASE( + "C API: Test invalid array for tiledb_array_get_enumeration", + "[enumeration][capi][error]") { + tiledb_ctx_t* ctx; + + auto rc = tiledb_ctx_alloc(nullptr, &ctx); + CHECK(rc == TILEDB_OK); + + tiledb_enumeration_t* enmr; + rc = tiledb_array_get_enumeration(ctx, nullptr, "an_enumeration", &enmr); + REQUIRE(rc == TILEDB_ERR); +} + +TEST_CASE( + "C API: Test invalid enumeration name for tiledb_array_get_enumeration", + "[enumeration][capi][error]") { + tiledb_ctx_t* ctx; + + auto rc = tiledb_ctx_alloc(nullptr, &ctx); + CHECK(rc == TILEDB_OK); + + tiledb_array_t* array; + rc = tiledb_array_alloc(ctx, "array_uri", &array); + REQUIRE(rc == TILEDB_OK); + + tiledb_enumeration_t* enmr; + rc = tiledb_array_get_enumeration(ctx, array, nullptr, &enmr); + REQUIRE(rc == TILEDB_ERR); +} diff --git a/test/src/unit-cppapi-deletes.cc b/test/src/unit-cppapi-deletes.cc index a27b83a7c418..b61292dfc0d2 100644 --- a/test/src/unit-cppapi-deletes.cc +++ b/test/src/unit-cppapi-deletes.cc @@ -111,6 +111,7 @@ struct DeletesFx { bool is_array(const std::string& array_name); void validate_array_dir_after_delete(const std::string& path); void validate_group_dir_after_delete(const std::string& path); + std::vector list_schemas(const std::string& array_name); }; DeletesFx::DeletesFx() @@ -507,6 +508,29 @@ void DeletesFx::validate_group_dir_after_delete(const std::string& path) { REQUIRE(!vfs_.is_dir(path + tiledb::sm::constants::group_metadata_dir_name)); } +std::vector DeletesFx::list_schemas( + const std::string& array_name) { + auto& enum_dir = tiledb::sm::constants::array_enumerations_dir_name; + auto schemas = + vfs_.ls(array_name + tiledb::sm::constants::array_schema_dir_name); + + auto it = schemas.begin(); + while (it != schemas.end()) { + if ((*it).size() < enum_dir.size()) { + continue; + } + if ((*it).substr((*it).size() - enum_dir.size()) == enum_dir) { + break; + } + ++it; + } + if (it != schemas.end()) { + schemas.erase(it); + } + + return schemas; +} + TEST_CASE_METHOD( DeletesFx, "CPP API: Test writing delete condition", @@ -1968,8 +1992,7 @@ TEST_CASE_METHOD( // Check write CHECK(tiledb::test::num_commits(SPARSE_ARRAY_NAME) == 4); CHECK(tiledb::test::num_fragments(SPARSE_ARRAY_NAME) == 4); - auto schemas = - vfs_.ls(array_name + tiledb::sm::constants::array_schema_dir_name); + auto schemas = list_schemas(array_name); CHECK(schemas.size() == 1); auto meta = vfs_.ls(array_name + tiledb::sm::constants::array_metadata_dir_name); @@ -2040,8 +2063,7 @@ TEST_CASE_METHOD( vfs_.touch(extraneous_file_path); // Check write - auto schemas = - vfs_.ls(array_name + tiledb::sm::constants::array_schema_dir_name); + auto schemas = list_schemas(array_name); CHECK(schemas.size() == 1); auto uris = vfs_.ls(array_name); bool ok_exists = false; @@ -2259,11 +2281,9 @@ TEST_CASE_METHOD( auto group_meta_dir = vfs_.ls(GROUP_NAME + tiledb::sm::constants::group_metadata_dir_name); CHECK(group_meta_dir.size() == 1); - auto array_schema = - vfs_.ls(array_path + tiledb::sm::constants::array_schema_dir_name); + auto array_schema = list_schemas(array_path); CHECK(array_schema.size() == 1); - auto array2_schema = - vfs_.ls(array2_path + tiledb::sm::constants::array_schema_dir_name); + auto array2_schema = list_schemas(array2_path); CHECK(array2_schema.size() == 1); // Recursively delete group in modify exclusive mode diff --git a/test/src/unit-cppapi-enumerations.cc b/test/src/unit-cppapi-enumerations.cc new file mode 100644 index 000000000000..677e3715f72c --- /dev/null +++ b/test/src/unit-cppapi-enumerations.cc @@ -0,0 +1,501 @@ +/** + * @file unit-cppapi-enumeration.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * Tests the C++ API for enumeration related functions. + */ + +#include + +#include +#include "tiledb/sm/array_schema/array_schema.h" +#include "tiledb/sm/c_api/tiledb_struct_def.h" +#include "tiledb/sm/cpp_api/tiledb" +#include "tiledb/sm/cpp_api/tiledb_experimental" + +using namespace tiledb; + +struct CPPEnumerationFx { + CPPEnumerationFx(); + ~CPPEnumerationFx(); + + template + void check_dump(const T& val); + + void create_array(); + void rm_array(); + + std::string uri_; + Context ctx_; + VFS vfs_; +}; + +const static std::string enmr_name = "an_enumeration"; +const static std::string dump_name = "enumeration_dump_test.txt"; + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Enumeration API - Create Fixed Size", + "[enumeration][create][fixed-size]") { + std::vector values = {1, 2, 3, 4, 5}; + auto enmr = Enumeration::create(ctx_, enmr_name, values); + REQUIRE(enmr.ptr() != nullptr); + REQUIRE(enmr.name() == enmr_name); + REQUIRE(enmr.type() == TILEDB_INT32); + REQUIRE(enmr.cell_val_num() == 1); + REQUIRE(enmr.ordered() == false); + + auto data = enmr.as_vector(); + REQUIRE(data == values); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Enumeration API - Create Variable Size", + "[enumeration][create][var-size]") { + std::vector values = {"fee", "fi", "fo", "fum"}; + auto enmr = Enumeration::create(ctx_, enmr_name, values); + REQUIRE(enmr.ptr() != nullptr); + REQUIRE(enmr.name() == enmr_name); + REQUIRE(enmr.type() == TILEDB_STRING_ASCII); + REQUIRE(enmr.cell_val_num() == TILEDB_VAR_NUM); + REQUIRE(enmr.ordered() == false); + + auto data = enmr.as_vector(); + REQUIRE(data == values); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Enumeration API - Create Ordered", + "[enumeration][create][var-size]") { + std::vector values = {1, 2, 3, 4, 5}; + auto enmr = Enumeration::create(ctx_, enmr_name, values, true); + REQUIRE(enmr.ptr() != nullptr); + REQUIRE(enmr.name() == enmr_name); + REQUIRE(enmr.type() == TILEDB_INT32); + REQUIRE(enmr.cell_val_num() == 1); + REQUIRE(enmr.ordered() == true); + + auto data = enmr.as_vector(); + REQUIRE(data == values); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Enumeration API - Dump Basic", + "[enumeration][dump][basic]") { + std::vector values = {1, 2, 3, 4, 5}; + auto enmr = Enumeration::create(ctx_, enmr_name, values, true); + check_dump(enmr); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Add Enumeration to Schema", + "[enumeration][add-attribute]") { + ArraySchema schema(ctx_, TILEDB_DENSE); + + auto dim = Dimension::create(ctx_, "dim", {{-100, 100}}); + auto dom = Domain(ctx_); + dom.add_dimension(dim); + schema.set_domain(dom); + + std::vector values = {"fred", "wilma", "barney", "pebbles"}; + auto enmr = Enumeration::create(ctx_, enmr_name, values); + ArraySchemaExperimental::add_enumeration(ctx_, schema, enmr); + + auto attr = Attribute::create(ctx_, "attr"); + AttributeExperimental::set_enumeration_name(ctx_, attr, enmr_name); + schema.add_attribute(attr); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Load Schema from URI", + "[enumeration][add-attribute]") { + create_array(); + auto schema = ArraySchemaExperimental::load_with_enumerations(ctx_, uri_); + auto enmr_names = + schema.ptr().get()->array_schema_->get_loaded_enumeration_names(); + REQUIRE(enmr_names.size() > 0); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Load Schema from URI - REMOTE NOT SUPPORTED YET", + "[enumeration][add-attribute][fixme]") { + std::string uri = "tiledb://namespace/array_name"; + REQUIRE_THROWS(ArraySchemaExperimental::load_with_enumerations(ctx_, uri)); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Schema Dump With Enumeration", + "[enumeration][array-schema][dump]") { + ArraySchema schema(ctx_, TILEDB_DENSE); + + auto dim = Dimension::create(ctx_, "dim", {{-100, 100}}); + auto dom = Domain(ctx_); + dom.add_dimension(dim); + schema.set_domain(dom); + + std::vector values = {"fred", "wilma", "barney", "pebbles"}; + auto enmr = Enumeration::create(ctx_, enmr_name, values); + ArraySchemaExperimental::add_enumeration(ctx_, schema, enmr); + + auto attr = Attribute::create(ctx_, "attr"); + AttributeExperimental::set_enumeration_name(ctx_, attr, enmr_name); + schema.add_attribute(attr); + + check_dump(schema); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Enumerations From Disk - Array::get_enumeration", + "[enumeration][array-get-enumeration]") { + create_array(); + auto array = Array(ctx_, uri_, TILEDB_READ); + auto enmr = ArrayExperimental::get_enumeration(ctx_, array, "an_enumeration"); + REQUIRE(enmr.ptr() != nullptr); + REQUIRE(enmr.name() == "an_enumeration"); + REQUIRE(enmr.type() == TILEDB_STRING_ASCII); + REQUIRE(enmr.cell_val_num() == TILEDB_VAR_NUM); + REQUIRE(enmr.ordered() == false); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Enumerations From Disk - Attribute::get_enumeration_name", + "[enumeration][attr-get-enumeration-name]") { + create_array(); + auto schema = Array::load_schema(ctx_, uri_); + + auto attr1 = schema.attribute("attr1"); + auto enmr_name1 = AttributeExperimental::get_enumeration_name(ctx_, attr1); + REQUIRE(enmr_name1.has_value() == true); + + auto attr2 = schema.attribute("attr2"); + auto enmr_name2 = AttributeExperimental::get_enumeration_name(ctx_, attr2); + REQUIRE(enmr_name2.has_value() == false); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Array::load_all_enumerations", + "[enumeration][array-load-all-enumerations]") { + create_array(); + auto array = Array(ctx_, uri_, TILEDB_READ); + REQUIRE_NOTHROW(ArrayExperimental::load_all_enumerations(ctx_, array)); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "C API: Array load_all_enumerations - Check nullptr", + "[enumeration][array-load-all-enumerations]") { + auto rc = tiledb_array_load_all_enumerations(ctx_.ptr().get(), nullptr, 0); + REQUIRE(rc != TILEDB_OK); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: ArraySchemaEvolution - Add Enumeration", + "[enumeration][array-schema-evolution][add-enumeration]") { + ArraySchemaEvolution ase(ctx_); + std::vector values = {"fred", "wilma", "barney", "pebbles"}; + auto enmr = Enumeration::create(ctx_, enmr_name, values); + CHECK_NOTHROW(ase.add_enumeration(enmr)); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "C API: ArraySchemaEvolution - Add Enumeration - Check nullptr", + "[enumeration][array-schema-evolution][error]") { + auto rc = tiledb_array_schema_evolution_add_enumeration( + ctx_.ptr().get(), nullptr, nullptr); + REQUIRE(rc != TILEDB_OK); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: ArraySchemaEvolution - Drop Enumeration", + "[enumeration][array-schema-evolution][drop-enumeration]") { + ArraySchemaEvolution ase(ctx_); + CHECK_NOTHROW(ase.drop_enumeration("an_enumeration_name")); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "C API: ArraySchemaEvolution - Drop Enumeration - Check nullptr", + "[enumeration][array-schema-evolution][drop-enumeration]") { + auto rc = tiledb_array_schema_evolution_drop_enumeration( + ctx_.ptr().get(), nullptr, "foo"); + REQUIRE(rc != TILEDB_OK); + + ArraySchemaEvolution ase(ctx_); + rc = tiledb_array_schema_evolution_drop_enumeration( + ctx_.ptr().get(), ase.ptr().get(), nullptr); + REQUIRE(rc != TILEDB_OK); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Enumeration Query - Basic", + "[enumeration][query][basic]") { + // Basic smoke test. Check that a simple query condition applied against + // an array returns sane results. + create_array(); + + // Check attr1 == "fred" when attr1 is an enumeration. + QueryCondition qc(ctx_); + qc.init("attr1", "fred", 4, TILEDB_EQ); + + // Execute the query condition against the array + std::vector dim(5); + std::vector attr1(5); + + auto array = Array(ctx_, uri_, TILEDB_READ); + Query query(ctx_, array); + query.add_range("dim", 1, 5) + .set_layout(TILEDB_ROW_MAJOR) + .set_data_buffer("dim", dim) + .set_data_buffer("attr1", attr1) + .set_condition(qc); + REQUIRE(query.submit() == Query::Status::COMPLETE); + query.finalize(); + + // attr1 == "fred" in position 0 and position 4. + std::vector dim_expect = {1, 2, 3, 4, 5}; + std::vector attr1_expect = {0, INT32_MIN, INT32_MIN, INT32_MIN, 0}; + + REQUIRE(dim == dim_expect); + REQUIRE(attr1 == attr1_expect); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Enumeration Query - Negation", + "[enumeration][query][negation]") { + // Another basic query test, the only twist here is that we're checking + // that query condition negation works as expected. + create_array(); + + // Create a query condition attr1 == "fred" and then negate it so that + // we can verify rewriting a negated query works. + QueryCondition qc1(ctx_); + qc1.init("attr1", "fred", 4, TILEDB_EQ); + auto qc2 = qc1.negate(); + + // Execute a read query against the created array + std::vector dim(5); + std::vector attr1(5); + + auto array = Array(ctx_, uri_, TILEDB_READ); + Query query(ctx_, array); + query.add_range("dim", 1, 5) + .set_layout(TILEDB_ROW_MAJOR) + .set_data_buffer("dim", dim) + .set_data_buffer("attr1", attr1) + .set_condition(qc2); + REQUIRE(query.submit() == Query::Status::COMPLETE); + query.finalize(); + + // attr1 == "fred" in positions 0 and 4 so those values should not match + // and return the default fill values. + std::vector dim_expect = {1, 2, 3, 4, 5}; + std::vector attr1_expect = {INT32_MIN, 1, 2, 1, INT32_MIN}; + + REQUIRE(dim == dim_expect); + REQUIRE(attr1 == attr1_expect); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Enumeration Query - Combination", + "[enumeration][query][combination]") { + // Same test as before except using multi-condition query condtions + create_array(); + + // Create query condition: attr1 == "fred" OR attr2 = 3.0f + QueryCondition qc1(ctx_); + qc1.init("attr1", "fred", 4, TILEDB_EQ); + + QueryCondition qc2(ctx_); + float val = 3.0f; + qc2.init("attr2", &val, sizeof(float), TILEDB_EQ); + + QueryCondition qc3 = qc1.combine(qc2, TILEDB_OR); + + // Execute a query with the query condition + std::vector dim(5); + std::vector attr1(5); + std::vector attr2(5); + + auto array = Array(ctx_, uri_, TILEDB_READ); + Query query(ctx_, array); + query.add_range("dim", 1, 5) + .set_layout(TILEDB_ROW_MAJOR) + .set_data_buffer("dim", dim) + .set_data_buffer("attr1", attr1) + .set_data_buffer("attr2", attr2) + .set_condition(qc3); + REQUIRE(query.submit() == Query::Status::COMPLETE); + query.finalize(); + + // Check the results match the expected results. attr1 == "fred" in + // positions 0 and 4, while attr2 == 3.0f in position 2. + std::vector dim_expect = {1, 2, 3, 4, 5}; + std::vector attr1_expect = {0, INT32_MIN, 2, INT32_MIN, 0}; + std::vector attr2_expect = { + 1.0f, std::nanf(""), 3.0f, std::nanf(""), 5.0f}; + + REQUIRE(dim == dim_expect); + REQUIRE(attr1 == attr1_expect); + + // NaN != NaN so we have to loop over the attr2 results to special case + // `isnan(v1) == isnan(v2)` when NaN is encountered the expect vector. + for (size_t i = 0; i < attr2_expect.size(); i++) { + if (std::isnan(attr2_expect[i])) { + REQUIRE(std::isnan(attr2[i])); + } else { + REQUIRE(attr2[i] == attr2_expect[i]); + } + } +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Enumeration Query - Set Use Enumeration", + "[enumeration][query][set-use-enumeration]") { + QueryCondition qc(ctx_); + qc.init("attr1", "fred", 4, TILEDB_EQ); + REQUIRE_NOTHROW( + QueryConditionExperimental::set_use_enumeration(ctx_, qc, true)); + REQUIRE_NOTHROW( + QueryConditionExperimental::set_use_enumeration(ctx_, qc, false)); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "C API: Enumeration Query - Check nullptr", + "[enumeration][query][check-nullptr]") { + auto rc = + tiledb_query_condition_set_use_enumeration(ctx_.ptr().get(), nullptr, 0); + REQUIRE(rc != TILEDB_OK); +} + +CPPEnumerationFx::CPPEnumerationFx() + : uri_("enumeration_test_array") + , vfs_(ctx_) { + rm_array(); +} + +CPPEnumerationFx::~CPPEnumerationFx() { + rm_array(); +} + +template +void CPPEnumerationFx::check_dump(const T& val) { + FILE* handle = fopen(dump_name.c_str(), "w"); + REQUIRE(handle != nullptr); + val.dump(handle); + fclose(handle); + + std::stringstream ss; + + // Scoped in an anonymous block to ensure that rstream closes before + // we attempt to delete the file for cleanup. + { + std::ifstream rstream(dump_name); + if (rstream.is_open()) { + ss << rstream.rdbuf(); + } + } + + auto data = ss.str(); + auto iter = data.find("Enumeration"); + REQUIRE(iter != std::string::npos); + + vfs_.remove_file(dump_name); +} + +void CPPEnumerationFx::create_array() { + // Create a simple array for testing. This ends up with just five elements in + // the array. dim is an int32_t dimension, attr1 is an enumeration with string + // values and int32_t attribute values. attr2 is a float attribute. + // + // The array data is summarized as below, however, pay attention to the fact + // that attr1 is storing integral index values instead of the raw string data. + // + // dim = {1, 2, 3, 4, 5} + // attr1 = {"fred", "wilma", "barney", "wilma", "fred"} + // attr2 = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f} + ArraySchema schema(ctx_, TILEDB_DENSE); + + auto dim = Dimension::create(ctx_, "dim", {{-100, 100}}); + auto dom = Domain(ctx_); + dom.add_dimension(dim); + schema.set_domain(dom); + + // The list of string values in the attr1 enumeration + std::vector values = {"fred", "wilma", "barney", "pebbles"}; + auto enmr = Enumeration::create(ctx_, "an_enumeration", values); + ArraySchemaExperimental::add_enumeration(ctx_, schema, enmr); + + auto attr1 = Attribute::create(ctx_, "attr1"); + AttributeExperimental::set_enumeration_name(ctx_, attr1, "an_enumeration"); + schema.add_attribute(attr1); + + auto attr2 = Attribute::create(ctx_, "attr2"); + schema.add_attribute(attr2); + + Array::create(uri_, schema); + + // Attribute data + std::vector attr1_values = {0, 1, 2, 1, 0}; + std::vector attr2_values = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f}; + + Array array(ctx_, uri_, TILEDB_WRITE); + Subarray subarray(ctx_, array); + subarray.set_subarray({1, 5}); + Query query(ctx_, array); + query.set_subarray(subarray) + .set_layout(TILEDB_ROW_MAJOR) + .set_data_buffer("attr1", attr1_values) + .set_data_buffer("attr2", attr2_values); + CHECK_NOTHROW(query.submit()); + query.finalize(); + array.close(); +} + +void CPPEnumerationFx::rm_array() { + if (vfs_.is_dir(uri_)) { + vfs_.remove_dir(uri_); + } +} diff --git a/test/src/unit-cppapi-schema-evolution.cc b/test/src/unit-cppapi-schema-evolution.cc index 73d847725c6a..a291bfc488e5 100644 --- a/test/src/unit-cppapi-schema-evolution.cc +++ b/test/src/unit-cppapi-schema-evolution.cc @@ -31,8 +31,15 @@ */ #include +#include "tiledb/sm/array_schema/array_schema.h" +#include "tiledb/sm/array_schema/array_schema_evolution.h" +#include "tiledb/sm/array_schema/attribute.h" +#include "tiledb/sm/array_schema/dimension.h" +#include "tiledb/sm/array_schema/domain.h" #include "tiledb/sm/cpp_api/tiledb" #include "tiledb/sm/cpp_api/tiledb_experimental" +#include "tiledb/sm/enums/array_type.h" +#include "tiledb/sm/enums/datatype.h" #include "tiledb/sm/misc/constants.h" #include @@ -776,3 +783,31 @@ TEST_CASE( vfs.remove_dir(array_uri); } } + +TEST_CASE( + "SchemaEvolution Error Handling Tests", + "[cppapi][schema][evolution][errors]") { + auto ase = make_shared(HERE()); + REQUIRE_THROWS(ase->evolve_schema(nullptr)); + REQUIRE_THROWS(ase->add_attribute(nullptr)); + + auto attr = make_shared( + HERE(), "attr", tiledb::sm::Datatype::STRING_ASCII); + ase->add_attribute(attr.get()); + REQUIRE_THROWS(ase->add_attribute(attr.get())); + + ase->set_timestamp_range(std::make_pair(1, 1)); + + auto schema = make_shared( + HERE(), tiledb::sm::ArrayType::SPARSE); + auto dim = make_shared( + HERE(), "dim1", tiledb::sm::Datatype::INT32); + int range[2] = {0, 1000}; + throw_if_not_ok(dim->set_domain(range)); + + auto dom = make_shared(HERE()); + throw_if_not_ok(dom->add_dimension(dim)); + throw_if_not_ok(schema->set_domain(dom)); + + CHECK_NOTHROW(ase->evolve_schema(schema)); +} diff --git a/test/src/unit-enum-helpers.cc b/test/src/unit-enum-helpers.cc new file mode 100644 index 000000000000..555b8f993707 --- /dev/null +++ b/test/src/unit-enum-helpers.cc @@ -0,0 +1,63 @@ +/** + * @file unit-enum-helpers.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * Tests for enum conversions. + */ + +#include + +#include "test/support/tdb_catch.h" +#include "tiledb/sm/enums/datatype.h" + +using namespace tiledb::sm; + +TEST_CASE( + "Test datatype_max_integral_value", + "[enums][datatype][max-integral-value]") { + std::vector> test_data = { + {Datatype::BOOL, 255}, + {Datatype::INT8, 127}, + {Datatype::INT16, 32767}, + {Datatype::INT32, 2147483647}, + {Datatype::INT64, 9223372036854775807}, + + {Datatype::UINT8, 255}, + {Datatype::UINT16, 65535}, + {Datatype::UINT32, 4294967295}, + {Datatype::UINT64, 18446744073709551615u}}; + + for (auto& [dtype, max_size] : test_data) { + std::cerr << "DTYPE: " << datatype_str(dtype) << std::endl; + REQUIRE(datatype_max_integral_value(dtype) == max_size); + } + + REQUIRE_THROWS(datatype_max_integral_value(Datatype::BLOB)); + REQUIRE_THROWS(datatype_max_integral_value(Datatype::FLOAT64)); + REQUIRE_THROWS(datatype_max_integral_value(Datatype::STRING_ASCII)); +} diff --git a/test/src/unit-enumerations.cc b/test/src/unit-enumerations.cc new file mode 100644 index 000000000000..062e321efb50 --- /dev/null +++ b/test/src/unit-enumerations.cc @@ -0,0 +1,1995 @@ +/** + * @file unit-enumerations.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * Tests the C++ API for enumeration related functions. + */ + +#include + +#include "test/support/tdb_catch.h" +#include "tiledb/sm/array/array.h" +#include "tiledb/sm/array/array_directory.h" +#include "tiledb/sm/array_schema/array_schema.h" +#include "tiledb/sm/array_schema/array_schema_evolution.h" +#include "tiledb/sm/array_schema/attribute.h" +#include "tiledb/sm/array_schema/dimension.h" +#include "tiledb/sm/array_schema/domain.h" +#include "tiledb/sm/array_schema/enumeration.h" +#include "tiledb/sm/buffer/buffer.h" +#include "tiledb/sm/buffer/buffer_list.h" +#include "tiledb/sm/config/config.h" +#include "tiledb/sm/enums/array_type.h" +#include "tiledb/sm/enums/encryption_type.h" +#include "tiledb/sm/enums/layout.h" +#include "tiledb/sm/query/query.h" +#include "tiledb/sm/query/query_condition.h" +#include "tiledb/sm/storage_manager/context.h" + +#ifdef TILEDB_SERIALIZATION +#include "tiledb/sm/enums/serialization_type.h" +#include "tiledb/sm/serialization/array_schema.h" +#include "tiledb/sm/serialization/array_schema_evolution.h" +#include "tiledb/sm/serialization/query.h" +#endif + +using namespace tiledb::sm; + +#define ENUM_NAME_PTR "an_enumeration" +const static std::string default_enmr_name = ENUM_NAME_PTR; + +struct EnumerationFx { + EnumerationFx(); + ~EnumerationFx(); + + template + shared_ptr create_enumeration( + const std::vector& values, + bool ordered = false, + Datatype type = static_cast(255), + std::string enmr_name = default_enmr_name); + + template + void check_enumeration( + shared_ptr enmr, + const std::string& name, + const std::vector& values, + Datatype data_type, + uint32_t cell_val_num, + bool ordered); + + template + void check_storage_serialization(const std::vector& values); + + template + void check_storage_deserialization(const std::vector& values); + + storage_size_t calculate_serialized_size(shared_ptr enmr); + WriterTile serialize_to_tile(shared_ptr enmr); + + template + std::vector as_vector(shared_ptr enmr); + + shared_ptr create_schema(); + + void create_array(); + shared_ptr get_array(QueryType type); + shared_ptr get_array_directory(); + shared_ptr get_array_schema_latest(bool load_enmrs = false); + + // Serialization helpers + ArraySchema ser_des_array_schema( + shared_ptr schema, + bool client_side, + SerializationType stype); + + shared_ptr ser_des_array_schema_evolution( + ArraySchemaEvolution* ase, bool client_side, SerializationType stype); + + void ser_des_query( + Query* q_in, Query* q_out, bool client_side, SerializationType stype); + + template + bool vec_cmp(std::vector v1, std::vector v2); + + void flatten_buffer_list(BufferList& blist, Buffer& buf); + + void rm_array(); + + URI uri_; + Config cfg_; + Context ctx_; + EncryptionKey enc_key_; +}; + +template +QueryCondition create_qc( + const char* field_name, T condition_value, const QueryConditionOp& op); + +/* ********************************* */ +/* Testing Enumeration */ +/* ********************************* */ + +TEST_CASE_METHOD( + EnumerationFx, + "Basic Fixed Size Enumeration Creation", + "[enumeration][basic][fixed]") { + std::vector values = {1, 2, 3, 4, 5}; + auto enmr = create_enumeration(values); + check_enumeration( + enmr, default_enmr_name, values, Datatype::UINT32, 1, false); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Basic Variable Size Enumeration Creation", + "[enumeration][basic][fixed]") { + std::vector values = {"foo", "bar", "baz", "bingo", "bango"}; + auto enmr = create_enumeration(values); + check_enumeration( + enmr, + default_enmr_name, + values, + Datatype::STRING_ASCII, + constants::var_num, + false); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Basic Variable Size With Empty Value Enumeration Creation", + "[enumeration][basic][fixed]") { + std::vector values = {"foo", "bar", "", "bingo", "bango"}; + auto enmr = create_enumeration(values); + check_enumeration( + enmr, + default_enmr_name, + values, + Datatype::STRING_ASCII, + constants::var_num, + false); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation with Ordered", + "[enumeration][basic][var-size][ordered]") { + std::vector values = {"foo", "bar", "baz", "bingo", "bango"}; + auto enmr = create_enumeration(values, true); + check_enumeration( + enmr, + default_enmr_name, + values, + Datatype::STRING_ASCII, + constants::var_num, + true); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation with Datatype", + "[enumeration][basic][var-size][custom-datatype]") { + std::vector values = {"foo", "bar", "baz", "bingo", "bango"}; + auto enmr = create_enumeration(values, false, Datatype::STRING_UTF8); + check_enumeration( + enmr, + default_enmr_name, + values, + Datatype::STRING_UTF8, + constants::var_num, + false); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation with Multi-Cell Val Num", + "[enumeration][basic][fixed][multi-cell-val-num]") { + std::vector values = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + auto enmr = Enumeration::create( + default_enmr_name, + Datatype::INT32, + 2, + false, + values.data(), + values.size() * sizeof(int), + nullptr, + 0); + check_enumeration(enmr, default_enmr_name, values, Datatype::INT32, 2, false); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Invalid empty name - std::string", + "[enumeration][error][invalid-name]") { + std::vector values = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + REQUIRE_THROWS(Enumeration::create( + std::string(), + Datatype::INT32, + 2, + false, + values.data(), + values.size() * sizeof(int), + nullptr, + 0)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Invalid empty name - char*", + "[enumeration][error][invalid-name]") { + std::vector values = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + REQUIRE_THROWS(Enumeration::create( + "", + Datatype::INT32, + 2, + false, + values.data(), + values.size() * sizeof(int), + nullptr, + 0)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Invalid name with slash", + "[enumeration][error][invalid-name]") { + std::vector values = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + REQUIRE_THROWS(Enumeration::create( + "an/enumeration", + Datatype::INT32, + 2, + false, + values.data(), + values.size() * sizeof(int), + nullptr, + 0)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Invalid cell val num", + "[enumeration][error][invalid-cell-val-num]") { + std::vector values = {1, 2, 3}; + REQUIRE_THROWS(Enumeration::create( + default_enmr_name, + Datatype::INT32, + 0, + false, + values.data(), + values.size() * sizeof(int), + nullptr, + 0)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - No data pointer", + "[enumeration][error][data-nullptr]") { + std::vector values = {1, 2, 3}; + REQUIRE_THROWS(Enumeration::create( + default_enmr_name, + Datatype::INT32, + 1, + false, + nullptr, + values.size() * sizeof(int), + nullptr, + 0)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Zero data size", + "[enumeration][error][data-zero-size]") { + std::vector values = {1, 2, 3}; + REQUIRE_THROWS(Enumeration::create( + default_enmr_name, + Datatype::INT32, + 1, + false, + values.data(), + 0, + nullptr, + 0)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - No offsets pointer", + "[enumeration][error][offsets-nullptr]") { + auto data = "foobarbazbam"; + std::vector offsets = {0, 3, 6, 9}; + REQUIRE_THROWS(Enumeration::create( + default_enmr_name, + Datatype::STRING_ASCII, + constants::var_num, + false, + data, + strlen(data), + nullptr, + offsets.size() * sizeof(uint64_t))); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - No offsets size", + "[enumeration][error][offsets-zero-size]") { + auto data = "foobarbazbam"; + std::vector offsets = {0, 3, 6, 9}; + REQUIRE_THROWS(Enumeration::create( + default_enmr_name, + Datatype::STRING_ASCII, + constants::var_num, + false, + data, + strlen(data), + offsets.data(), + 0)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Offsets not required, pointer provided", + "[enumeration][error][offsets-not-required]") { + std::vector values = {0, 1, 2, 3, 4}; + std::vector offsets = {0, 3, 6, 9}; + REQUIRE_THROWS(Enumeration::create( + default_enmr_name, + Datatype::INT32, + 1, + false, + values.data(), + values.size() * sizeof(int), + offsets.data(), + 0)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Offsets not required, size provided", + "[enumeration][error][offsets-not-required]") { + std::vector values = {0, 1, 2, 3, 4}; + REQUIRE_THROWS(Enumeration::create( + default_enmr_name, + Datatype::INT32, + 1, + false, + values.data(), + values.size() * sizeof(int), + nullptr, + 100)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Invalid offsests size provided", + "[enumeration][error][offsets-invalid-size]") { + auto data = "foobarbazbam"; + std::vector offsets = {0, 3, 6, 9}; + // Passing 3 for the offsets size is incorrect because the offsets size has + // to be a multiple of `sizeof(uint64_t)` + REQUIRE_THROWS(Enumeration::create( + default_enmr_name, + Datatype::STRING_ASCII, + constants::var_num, + false, + data, + strlen(data), + offsets.data(), + 3)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Offsets to data beyond provided data size", + "[enumeration][error][invalid-offset-data]") { + auto data = "foobarbazbam"; + std::vector offsets = {0, 3, 6, 100}; + // The last offset is larger than data_size + REQUIRE_THROWS(Enumeration::create( + default_enmr_name, + Datatype::STRING_ASCII, + constants::var_num, + false, + data, + strlen(data), + offsets.data(), + offsets.size() * sizeof(uint64_t))); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Invalid data size", + "[enumeration][error][invalid-data-size]") { + std::vector values = {1, 2, 3, 4, 5}; + // Passing 3 for the data size is invalid as its not a multiple of + // sizeof(int) + REQUIRE_THROWS(Enumeration::create( + default_enmr_name, + Datatype::INT32, + 1, + false, + values.data(), + 3, + nullptr, + 0)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Repeated fixed sized values", + "[enumeration][error][repeated-values]") { + std::vector values = {1, 2, 3, 3, 4, 5}; + REQUIRE_THROWS(create_enumeration(values)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Repeated variable sized values", + "[enumeration][error][repeated-values]") { + std::vector values = {"foo", "bar", "bang", "bar"}; + REQUIRE_THROWS(create_enumeration(values)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Repeated empty variable sized values", + "[enumeration][error][repeated-values]") { + std::vector values = {"foo", "", "bang", ""}; + REQUIRE_THROWS(create_enumeration(values)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Serialization - Fixed Size", + "[enumeration][serialization][fixed-size]") { + std::vector values = {1, 2, 3, 4, 5}; + check_storage_serialization(values); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Serialization - Variable Size", + "[enumeration][serialization][var-size]") { + std::vector values = {"foo", "bar", "baz", "bam", "cap"}; + check_storage_serialization(values); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Deserialization - Fixed Size", + "[enumeration][deserialization][fixed-size]") { + std::vector values = {1, 2, 3, 4, 5}; + check_storage_deserialization(values); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Deserialization - Variable Size", + "[enumeration][deserialization][var-size]") { + std::vector values = {"foo", "bar", "baz", "bam", "cap"}; + check_storage_deserialization(values); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Deserialization Error - Invalid version", + "[enumeration][deserialization][error][invalid-version]") { + std::vector values = {"foo", "bar", "baz", "bam", "cap"}; + auto enmr = create_enumeration(values); + auto tile = serialize_to_tile(enmr); + + REQUIRE(tile.size() > 4); + auto data = tile.data(); + memset(data, 1, 4); + + Deserializer deserializer(tile.data(), tile.size()); + REQUIRE_THROWS(Enumeration::deserialize(deserializer)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration index_of - Fixed Size", + "[enumeration][index-of][fixed-size]") { + std::vector values = {1, 2, 3, 4, 5}; + auto enmr = create_enumeration(values); + + for (uint64_t i = 0; i < values.size(); i++) { + int tmp = values[i]; + UntypedDatumView udv(&tmp, sizeof(int)); + REQUIRE(enmr->index_of(udv) == i); + } +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration index_of - Fixed Size Missing", + "[enumeration][index-of][fixed-size]") { + std::vector values = {1, 2, 3, 4, 5}; + auto enmr = create_enumeration(values); + + int zero = 0; + UntypedDatumView udv_zero(&zero, sizeof(int)); + REQUIRE(enmr->index_of(udv_zero) == constants::enumeration_missing_value); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration index_of - Variable Size", + "[enumeration][index-of][var-size]") { + std::vector values = {"foo", "bar", "baz", "bang", "ohai"}; + auto enmr = create_enumeration(values); + + for (uint64_t i = 0; i < values.size(); i++) { + UntypedDatumView udv(values[i].data(), values[i].size()); + REQUIRE(enmr->index_of(udv) == i); + } +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration index_of - Variable Size Missing", + "[enumeration][index-of][var-size]") { + std::vector values = {"foo", "bar", "baz", "bang", "ohai"}; + auto enmr = create_enumeration(values); + + UntypedDatumView udv_empty("", 0); + REQUIRE(enmr->index_of(udv_empty) == constants::enumeration_missing_value); +} + +/* ********************************* */ +/* Testing Attribute */ +/* ********************************* */ + +TEST_CASE_METHOD( + EnumerationFx, + "Attribute set_enumeration_name - Error - Empty Name", + "[enumeration][attribute][error]") { + auto attr = make_shared(HERE(), "foo", Datatype::INT8); + REQUIRE_THROWS(attr->set_enumeration_name("")); +} + +/* ********************************* */ +/* Testing Array */ +/* ********************************* */ + +TEST_CASE_METHOD( + EnumerationFx, + "Array - Get Enumeration", + "[enumeration][array][get-enumeration]") { + create_array(); + auto array = get_array(QueryType::READ); + auto enmr = array->get_enumeration("test_enmr"); + REQUIRE(enmr != nullptr); + + std::vector values = {"ant", "bat", "cat", "dog", "emu"}; + check_enumeration( + enmr, + "test_enmr", + values, + Datatype::STRING_ASCII, + constants::var_num, + false); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Array - Get Enumeration Repeated", + "[enumeration][array][get-enumeration]") { + create_array(); + auto array = get_array(QueryType::READ); + auto enmr1 = array->get_enumeration("test_enmr"); + auto enmr2 = array->get_enumeration("test_enmr"); + REQUIRE(enmr1 == enmr2); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Array - Get Enumeration Error - REMOTE NOT YET SUPPORTED", + "[enumeration][array][error][get-remote]") { + std::string uri_str = "tiledb://namespace/array_name"; + auto array = make_shared(HERE(), URI(uri_str), ctx_.storage_manager()); + REQUIRE_THROWS(array->get_enumeration("something_here")); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Array - Get Enumeration Error - Not Open", + "[enumeration][array][error][not-open]") { + auto array = make_shared(HERE(), uri_, ctx_.storage_manager()); + REQUIRE_THROWS(array->get_enumeration("foo")); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Array - Get Non-existent Enumeration", + "[enumeration][array][unknown-enumeration]") { + create_array(); + auto array = get_array(QueryType::READ); + REQUIRE_THROWS(array->get_enumeration("foo")); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Array - Load All Enumerations - Latest Only", + "[enumeration][array][load-all-enumerations][latest-only]") { + create_array(); + auto array = get_array(QueryType::READ); + auto schema = array->array_schema_latest_ptr(); + REQUIRE(schema->is_enumeration_loaded("test_enmr") == false); + CHECK_NOTHROW(array->load_all_enumerations()); + REQUIRE(schema->is_enumeration_loaded("test_enmr") == true); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Array - Load All Enumerations - Repeated", + "[enumeration][array][load-all-enumerations][repeated]") { + create_array(); + auto array = get_array(QueryType::READ); + auto schema = array->array_schema_latest_ptr(); + + REQUIRE(schema->is_enumeration_loaded("test_enmr") == false); + + CHECK_NOTHROW(array->load_all_enumerations()); + REQUIRE(schema->is_enumeration_loaded("test_enmr") == true); + + CHECK_NOTHROW(array->load_all_enumerations()); + REQUIRE(schema->is_enumeration_loaded("test_enmr") == true); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Array - Load All Enumerations", + "[enumeration][array][load-all-enumerations]") { + create_array(); + auto array = get_array(QueryType::READ); + auto schema = array->array_schema_latest_ptr(); + REQUIRE(schema->is_enumeration_loaded("test_enmr") == false); + CHECK_NOTHROW(array->load_all_enumerations(false)); + REQUIRE(schema->is_enumeration_loaded("test_enmr") == true); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Array - Load All Enumerations Error - REMOTE NOT YET SUPPORTED", + "[enumeration][array][error][get-remote]") { + std::string uri_str = "tiledb://namespace/array_name"; + auto array = make_shared(HERE(), URI(uri_str), ctx_.storage_manager()); + REQUIRE_THROWS(array->load_all_enumerations()); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Array - Load All Enumerations Error - Not Open", + "[enumeration][array][error][not-open]") { + auto array = make_shared(HERE(), uri_, ctx_.storage_manager()); + REQUIRE_THROWS(array->load_all_enumerations()); +} + +/* ********************************* */ +/* Testing ArrayDirectory */ +/* ********************************* */ + +TEST_CASE_METHOD( + EnumerationFx, + "ArrayDirectory - Load Enumeration", + "[enumeration][array-directory][load-enumeration]") { + create_array(); + + auto schema = get_array_schema_latest(); + auto ad = get_array_directory(); + auto enmr_name = schema->attribute("attr1")->get_enumeration_name(); + REQUIRE(enmr_name.has_value()); + + auto enmr = ad->load_enumeration(schema, enmr_name.value(), enc_key_); + std::vector values = {"ant", "bat", "cat", "dog", "emu"}; + check_enumeration( + enmr, + "test_enmr", + values, + Datatype::STRING_ASCII, + constants::var_num, + false); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArrayDirectory - Load Enumeration Error", + "[enumeration][array-directory][error]") { + create_array(); + + auto schema = get_array_schema_latest(); + auto ad = get_array_directory(); + + // Check that this function throws an exception when attempting to load + // an unknown enumeration + REQUIRE_THROWS(ad->load_enumeration(schema, "not_an_enumeration", enc_key_)); +} + +/* ********************************* */ +/* Testing ArraySchema */ +/* ********************************* */ + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Add Enumeration - Enumeration nullptr Error", + "[enumeration][array-schema][error]") { + auto schema = make_shared(HERE()); + REQUIRE_THROWS(schema->add_enumeration(nullptr)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Add Basic Enumeration", + "[enumeration][array-schema][basic]") { + auto schema = make_shared(HERE()); + + std::vector values = {1, 2, 3, 4, 5}; + auto enmr = create_enumeration(values); + schema->add_enumeration(enmr); + + auto attr = make_shared(HERE(), "foo", Datatype::INT8); + attr->set_enumeration_name(enmr->name()); + CHECK_NOTHROW(schema->add_attribute(attr)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Get Enumeration", + "[enumeration][array-schema][get]") { + auto schema = make_shared(HERE(), ArrayType::DENSE); + + std::vector values = {1, 2, 3, 4, 5}; + auto enmr1 = create_enumeration(values); + schema->add_enumeration(enmr1); + + auto enmr2 = schema->get_enumeration(default_enmr_name); + check_enumeration(enmr2, enmr1->name(), values, Datatype::INT32, 1, false); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Get Missing Enumeration Error", + "[enumeration][array-schema][error]") { + auto schema = make_shared(HERE(), ArrayType::SPARSE); + REQUIRE_THROWS(schema->get_enumeration("not_an_enumeration")); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Add Enumeration with Existing Enumeration of same Name", + "[enumeration][array-schema][eror]") { + auto schema = make_shared(HERE(), ArrayType::SPARSE); + std::vector values = {1, 2, 3, 4, 5}; + auto enmr = create_enumeration(values); + + schema->add_enumeration(enmr); + REQUIRE_THROWS(schema->add_enumeration(enmr)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Add Attribute with Missing Enumeration Error", + "[enumeration][array-schema][eror]") { + auto schema = make_shared(HERE(), ArrayType::SPARSE); + auto attr = make_shared(HERE(), "an_attr", Datatype::INT32); + attr->set_enumeration_name("not_an_enumeration"); + REQUIRE(!schema->add_attribute(attr).ok()); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Get All Enumeration Names Empty", + "[enumeration][array-schema][get-all][empty]") { + auto schema = make_shared(HERE(), ArrayType::DENSE); + auto enmr_names = schema->get_enumeration_names(); + REQUIRE(enmr_names.size() == 0); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Get All Enumeration Names", + "[enumeration][array-schema][get-all]") { + auto schema = make_shared(HERE(), ArrayType::DENSE); + + std::vector values = {1.0f, 1.1f, 1.2f, 1.3f, 1.4f}; + auto enmr1 = create_enumeration(values); + schema->add_enumeration(enmr1); + + auto enmr_names = schema->get_enumeration_names(); + REQUIRE(enmr_names.size() == 1); + + auto enmr2 = schema->get_enumeration(enmr_names[0]); + REQUIRE(enmr2 == enmr1); + check_enumeration( + enmr2, default_enmr_name, values, Datatype::FLOAT32, 1, false); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Attribute with Invalid Datatype", + "[enumeration][array-schema][error][bad-attr-datatype]") { + auto schema = make_shared(HERE(), ArrayType::DENSE); + + std::vector values = {1, 2, 3, 4, 5}; + auto enmr = create_enumeration(values); + schema->add_enumeration(enmr); + + auto attr = make_shared(HERE(), "ohai", Datatype::FLOAT32); + attr->set_enumeration_name(default_enmr_name); + REQUIRE(schema->add_attribute(attr).ok() == false); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Attribute with Invalid Cell Val Num", + "[enumeration][array-schema][error][bad-attr-cell-val-num]") { + auto schema = make_shared(HERE(), ArrayType::DENSE); + + std::vector values = {1, 2, 3, 4, 5}; + auto enmr = create_enumeration(values); + schema->add_enumeration(enmr); + + auto attr = make_shared( + HERE(), "ohai", Datatype::INT32, 2, DataOrder::UNORDERED_DATA); + attr->set_enumeration_name(default_enmr_name); + REQUIRE(schema->add_attribute(attr).ok() == false); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Store nullptr Enumeration Error", + "[enumeration][array-schema][error][store-nullptr-enumeration]") { + auto schema = make_shared(HERE(), ArrayType::DENSE); + REQUIRE_THROWS(schema->store_enumeration(nullptr)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Store Enumeration Error", + "[enumeration][array-schema][error][store-unknown-enumeration]") { + auto schema = make_shared(HERE(), ArrayType::DENSE); + std::vector values = {1, 2, 3, 4, 5}; + auto enmr = + create_enumeration(values, false, Datatype::INT32, "unknown_enmr"); + REQUIRE_THROWS(schema->store_enumeration(enmr)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Store Enumeration Error - Already Loaded", + "[enumeration][array-schema][error][store-loaded-enumeration]") { + auto schema = make_shared(HERE(), ArrayType::DENSE); + + std::vector values = {0, 1, 2, 100000000}; + auto enmr = create_enumeration(values); + schema->add_enumeration(enmr); + + // This is an error because there's already an enumeration stored in the + // ArraySchema::enumeration_map_. When deserializing schemas from disk the + // entries in ArraySchema::enumeration_map_ are (std::string, nullptr). + REQUIRE_THROWS(schema->store_enumeration(enmr)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Attribute Get Enumeration Name From Attribute", + "[enumeration][array-schema][has-enumeration]") { + auto schema = make_shared(HERE(), ArrayType::SPARSE); + + std::vector values = {"a", "spot", "of", "tea", "perhaps?"}; + auto enmr = create_enumeration(values); + schema->add_enumeration(enmr); + + REQUIRE(schema->get_enumeration(enmr->name()) == enmr); + REQUIRE(schema->get_enumeration_path_name(enmr->name()) == enmr->path_name()); + + auto attr = make_shared(HERE(), "ohai", Datatype::INT64); + attr->set_enumeration_name(default_enmr_name); + throw_if_not_ok(schema->add_attribute(attr)); + + REQUIRE(schema->attribute("ohai")->get_enumeration_name().has_value()); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Schema Copy Constructor", + "[enumeration][array-schema][copy-ctor]") { + auto schema = create_schema(); + + // Check that the schema is valid and that we can copy it using the + // copy constructor. + CHECK_NOTHROW(schema->check()); + CHECK_NOTHROW(make_shared(HERE(), *(schema.get()))); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Mismatched path name error", + "[enumeration][array-schema][error]") { + create_array(); + auto schema = get_array_schema_latest(); + + // Creating a new Enumeration will give it a different UUID path name. + std::vector values = {1, 2, 3, 4, 5}; + auto enmr = create_enumeration(values, false, Datatype::INT32, "test_enmr"); + + // Storing an enumeration with a known name but different path is an error + REQUIRE_THROWS(schema->store_enumeration(enmr)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Drop Enumeration - Empty Name", + "[enumeration][array-schema][error]") { + create_array(); + auto schema = get_array_schema_latest(); + REQUIRE_THROWS(schema->drop_enumeration("")); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Drop Enumeration - Unknown Enumeration Name", + "[enumeration][array-schema][error]") { + create_array(); + auto schema = get_array_schema_latest(); + REQUIRE_THROWS(schema->drop_enumeration("not_an_enumeration")); +} + +/* ********************************* */ +/* Testing ArraySchemaEvolution */ +/* ********************************* */ + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Simple No Enumeration", + "[enumeration][array-schema-evolution][simple]") { + create_array(); + auto orig_schema = get_array_schema_latest(true); + auto ase = make_shared(HERE()); + auto attr3 = make_shared(HERE(), "attr3", Datatype::UINT32); + ase->add_attribute(attr3.get()); + CHECK_NOTHROW(ase->evolve_schema(orig_schema)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Simple With Enumeration", + "[enumeration][array-schema-evolution][simple]") { + create_array(); + auto orig_schema = get_array_schema_latest(); + auto ase = make_shared(HERE()); + + std::vector values{0, 1, 2, 3, 4, 1000}; + auto enmr = create_enumeration(values); + ase->add_enumeration(enmr); + + auto attr3 = make_shared(HERE(), "attr3", Datatype::UINT32); + attr3->set_enumeration_name(default_enmr_name); + ase->add_attribute(attr3.get()); + + ase->drop_attribute("attr2"); + + CHECK_NOTHROW(ase->evolve_schema(orig_schema)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Drop Attribute After Add", + "[enumeration][array-schema-evolution][drop-add]") { + create_array(); + auto orig_schema = get_array_schema_latest(); + auto ase = make_shared(HERE()); + + std::vector values{0, 1, 2, 3, 4, 1000}; + auto enmr = create_enumeration(values); + ase->add_enumeration(enmr); + + auto attr3 = make_shared(HERE(), "attr3", Datatype::UINT32); + attr3->set_enumeration_name(default_enmr_name); + ase->add_attribute(attr3.get()); + + CHECK_NOTHROW(ase->drop_attribute("attr3")); + CHECK_NOTHROW(ase->evolve_schema(orig_schema)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Enumeration Attribute Names to Add", + "[enumeration][array-schema-evolution][simple]") { + create_array(); + auto orig_schema = get_array_schema_latest(); + + auto ase = make_shared(HERE()); + + std::vector values{0, 1, 2, 3, 4, 1000}; + auto enmr = create_enumeration(values); + ase->add_enumeration(enmr); + + auto enmr_names = ase->enumeration_names_to_add(); + REQUIRE(enmr_names.size() == 1); + REQUIRE(enmr_names[0] == enmr->name()); + + CHECK_NOTHROW(ase->evolve_schema(orig_schema)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Enumeration to Add", + "[enumeration][array-schema-evolution][enmr-to-add]") { + create_array(); + auto orig_schema = get_array_schema_latest(); + auto ase = make_shared(HERE()); + + std::vector values{0, 1, 2, 3, 4, 1000}; + auto enmr1 = create_enumeration(values); + ase->add_enumeration(enmr1); + + auto enmr2 = ase->enumeration_to_add(enmr1->name()); + REQUIRE(enmr2 == enmr1); + + CHECK_NOTHROW(ase->evolve_schema(orig_schema)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Drop Enumeration", + "[enumeration][array-schema-evolution][enmr-to-add]") { + create_array(); + auto orig_schema = get_array_schema_latest(); + auto ase1 = make_shared(HERE()); + + std::vector values{0, 1, 2, 3, 4, 1000}; + auto enmr1 = create_enumeration(values, false, Datatype::UINT64, "enmr"); + ase1->add_enumeration(enmr1); + + auto new_schema = ase1->evolve_schema(orig_schema); + + auto ase2 = make_shared(HERE()); + ase2->drop_enumeration("enmr"); + + CHECK_NOTHROW(ase2->evolve_schema(new_schema)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Drop Enumeration", + "[enumeration][array-schema-evolution][enmr-to-drop]") { + auto ase = make_shared(HERE()); + CHECK_NOTHROW(ase->drop_enumeration("test_enmr")); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Drop Enumeration Repeated", + "[enumeration][array-schema-evolution][enmr-to-drop-repeated]") { + auto ase = make_shared(HERE()); + CHECK_NOTHROW(ase->drop_enumeration("test_enmr")); + CHECK_NOTHROW(ase->drop_enumeration("test_enmr")); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Drop Enumeration After Add", + "[enumeration][array-schema-evolution][enmr-add-drop]") { + auto ase = make_shared(HERE()); + + std::vector values{0, 1, 2, 3, 4, 1000}; + auto enmr = create_enumeration(values, false, Datatype::UINT64, "enmr"); + ase->add_enumeration(enmr); + REQUIRE(ase->enumeration_names_to_add().size() == 1); + CHECK_NOTHROW(ase->drop_enumeration("enmr")); + REQUIRE(ase->enumeration_names_to_add().size() == 0); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Enumeration to Add - nullptr", + "[enumeration][array-schema-evolution][enmr-nullptr]") { + create_array(); + auto ase = make_shared(HERE()); + REQUIRE_THROWS(ase->add_enumeration(nullptr)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Enumeration to Add - Already Added", + "[enumeration][array-schema-evolution][enmr-already-added]") { + create_array(); + auto ase = make_shared(HERE()); + + std::vector values{0, 1, 2, 3, 4, 1000}; + auto enmr1 = create_enumeration(values, false, Datatype::UINT64, "enmr"); + auto enmr2 = create_enumeration(values, false, Datatype::UINT64, "enmr"); + ase->add_enumeration(enmr1); + REQUIRE_THROWS(ase->add_enumeration(enmr2)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Enumeration to Add - Missing Name", + "[enumeration][array-schema-evolution][missing-name]") { + create_array(); + auto ase = make_shared(HERE()); + REQUIRE(ase->enumeration_to_add("foo") == nullptr); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Drop enumeration while still in use.", + "[enumeration][array-schema-evolution][enmr-still-in-use]") { + create_array(); + auto orig_schema = get_array_schema_latest(); + auto ase = make_shared(HERE()); + ase->drop_enumeration("test_enmr"); + + REQUIRE_THROWS(ase->evolve_schema(orig_schema)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Enumeration not Loaded", + "[enumeration][array-schema-evolution][enmr-not-loaded]") { + create_array(); + + auto attr3 = make_shared(HERE(), "attr3", Datatype::UINT32); + attr3->set_enumeration_name("test_enmr"); + + auto ase = make_shared(HERE()); + ase->add_attribute(attr3.get()); + + auto orig_schema = get_array_schema_latest(); + REQUIRE_THROWS(ase->evolve_schema(orig_schema)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Enumeration to large for signed attribute type", + "[enumeration][array-schema-evolution][enmr-to-large]") { + create_array(); + + std::vector values; + values.resize(129); + std::iota(values.begin(), values.end(), 0); + auto enmr = create_enumeration(values, false, Datatype::INT32, "big_enmr"); + + auto attr3 = make_shared(HERE(), "attr3", Datatype::INT8); + attr3->set_enumeration_name("big_enmr"); + + auto ase = make_shared(HERE()); + ase->add_enumeration(enmr); + ase->add_attribute(attr3.get()); + + auto orig_schema = get_array_schema_latest(); + REQUIRE_THROWS(ase->evolve_schema(orig_schema)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Enumeration to large for unsigned attribute type", + "[enumeration][array-schema-evolution][enmr-to-large]") { + create_array(); + + std::vector values; + values.resize(257); + std::iota(values.begin(), values.end(), 0); + auto enmr = create_enumeration(values, false, Datatype::INT32, "big_enmr"); + + auto attr3 = make_shared(HERE(), "attr3", Datatype::UINT8); + attr3->set_enumeration_name("big_enmr"); + + auto ase = make_shared(HERE()); + ase->add_enumeration(enmr); + ase->add_attribute(attr3.get()); + + auto orig_schema = get_array_schema_latest(); + REQUIRE_THROWS(ase->evolve_schema(orig_schema)); +} + +/* ********************************* */ +/* Testing QueryCondition */ +/* ********************************* */ + +TEST_CASE_METHOD( + EnumerationFx, + "QueryCondition - Rewrite Enumeration Value", + "[enumeration][query-condition][rewrite-enumeration-value]") { + create_array(); + auto array = get_array(QueryType::READ); + auto schema = array->array_schema_latest_ptr(); + + // This is normally invoked by the query class when not being tested. It's + // required here so that the enumeration's data is loaded. + array->get_enumeration("test_enmr"); + + // Assert that the enumerations were loaded + auto enmr_names = schema->get_loaded_enumeration_names(); + REQUIRE(enmr_names.size() == 1); + REQUIRE(enmr_names[0] == "test_enmr"); + + // Create two copies of the same query condition for assertions + auto qc1 = create_qc("attr1", std::string("cat"), QueryConditionOp::EQ); + auto qc2 = qc1; + + qc2.rewrite_enumeration_conditions(*(schema.get())); + + // Assert that the rewritten tree matches in the right places while also + // different to verify the assertion of having been rewritten. + auto& tree1 = qc1.ast(); + auto& tree2 = qc2.ast(); + + REQUIRE(tree1->is_expr() == false); + REQUIRE(tree1->get_field_name() == "attr1"); + + REQUIRE(tree2->is_expr() == tree1->is_expr()); + REQUIRE(tree2->get_field_name() == tree1->get_field_name()); + + auto udv1 = tree1->get_condition_value_view(); + auto udv2 = tree2->get_condition_value_view(); + REQUIRE(udv2.size() != udv1.size()); + REQUIRE(udv2.content() != udv1.content()); + REQUIRE(udv2.value_as() == 2); +} + +TEST_CASE_METHOD( + EnumerationFx, + "QueryCondition - Skip enumeration rewrite", + "[enumeration][query-condition][skip-rewrite]") { + create_array(); + auto schema = get_array_schema_latest(); + + // Almost exactly the same test as before, except this time we call + // `set_use_enumeration(false)` before rewriting and assert that the + // resulting rewritten query tree matches exactly since no enumeration + // rewriting has taken place. + auto qc1 = create_qc("attr1", (int)2, QueryConditionOp::EQ); + qc1.set_use_enumeration(false); + auto qc2 = qc1; + + qc2.rewrite_enumeration_conditions(*(schema.get())); + + auto& tree1 = qc1.ast(); + auto& tree2 = qc2.ast(); + + // Check that both trees match exactly + REQUIRE(tree1->is_expr() == false); + REQUIRE(tree1->get_field_name() == "attr1"); + + REQUIRE(tree2->is_expr() == tree1->is_expr()); + REQUIRE(tree2->get_field_name() == tree1->get_field_name()); + + auto udv1 = tree1->get_condition_value_view(); + auto udv2 = tree1->get_condition_value_view(); + REQUIRE(udv2.size() == udv1.size()); + REQUIRE(udv2.content() == udv1.content()); + REQUIRE(udv2.value_as() == 2); +} + +TEST_CASE_METHOD( + EnumerationFx, + "QueryCondition - Rewrite - No failure on unknown attribute", + "[enumeration][query-condition]") { + create_array(); + auto schema = get_array_schema_latest(); + + auto qc1 = create_qc("not_an_attr", (int)2, QueryConditionOp::EQ); + qc1.rewrite_enumeration_conditions(*(schema.get())); +} + +TEST_CASE_METHOD( + EnumerationFx, + "QueryCondition - Rewrite - Enumeration Not Loaded", + "[enumeration][query-condition][error]") { + create_array(); + auto schema = get_array_schema_latest(); + + auto qc1 = create_qc("attr1", (int)2, QueryConditionOp::EQ); + REQUIRE_THROWS(qc1.rewrite_enumeration_conditions(*(schema.get()))); +} + +TEST_CASE_METHOD( + EnumerationFx, + "QueryCondition - Rewrite - Inequality on Unordered Enumeration", + "[enumeration][query-condition][error]") { + // If an enumeration isn't marked as ordered, then it should throw an error + // when attempting to use an inequality operator on the attribute. + create_array(); + auto array = get_array(QueryType::READ); + auto schema = get_array_schema_latest(); + + // This is normally invoked by the query class when not being tested. + array->get_enumeration("test_enmr"); + + auto qc1 = create_qc("attr1", (int)2, QueryConditionOp::LT); + REQUIRE_THROWS(qc1.rewrite_enumeration_conditions(*(schema.get()))); +} + +TEST_CASE_METHOD( + EnumerationFx, + "QueryCondition - Rewrite Empty QC - Coverage", + "[enumeration][query-condition][coverage]") { + // Check that qc.rewrite_enumeration_conditions doesn't throw on an empty QC + create_array(); + auto schema = get_array_schema_latest(); + + QueryCondition qc; + CHECK_NOTHROW(qc.rewrite_enumeration_conditions(*(schema.get()))); +} + +TEST_CASE_METHOD( + EnumerationFx, + "QueryCondition - use_enumeration - Check Accessor for Coverage", + "[enumeration][query-condition][coverage]") { + auto qc1 = create_qc("attr1", (int)2, QueryConditionOp::LT); + auto& node = qc1.ast(); + + REQUIRE(node->use_enumeration() == true); + qc1.set_use_enumeration(false); + REQUIRE(node->use_enumeration() == false); +} + +TEST_CASE_METHOD( + EnumerationFx, + "QueryCondition - set_use_enumeration - Affects Children", + "[enumeration][query-condition][set_use_enumeration]") { + // Check that set_use_enumeration is applied to the entire tree if applied + // to an expression node. + auto qc1 = create_qc("attr1", (int)2, QueryConditionOp::EQ); + auto qc2 = create_qc("attr2", 3.0f, QueryConditionOp::LT); + QueryCondition qc3; + throw_if_not_ok(qc1.combine(qc2, QueryConditionCombinationOp::AND, &qc3)); + + auto& tree1 = qc3.ast(); + for (auto& child : tree1->get_children()) { + REQUIRE(child->use_enumeration() == true); + } + + qc3.set_use_enumeration(false); + + auto& tree2 = qc3.ast(); + for (auto& child : tree2->get_children()) { + REQUIRE(child->use_enumeration() == false); + } +} + +TEST_CASE_METHOD( + EnumerationFx, + "QueryCondition - use_enumeration - Error on ASTNodeExpr", + "[enumeration][query-condition][error]") { + // Check that an ASTNodeExpr throws correctly when calling + // `use_enumeration()`. + auto qc1 = create_qc("attr1", (int)2, QueryConditionOp::EQ); + auto qc2 = create_qc("attr2", 3.0f, QueryConditionOp::LT); + QueryCondition qc3; + throw_if_not_ok(qc1.combine(qc2, QueryConditionCombinationOp::AND, &qc3)); + auto& node = qc3.ast(); + + REQUIRE_THROWS(node->use_enumeration()); +} + +/* ********************************* */ +/* Testing Cap'N Proto Serialization */ +/* ********************************* */ + +#ifdef TILEDB_SERIALIZATION + +TEST_CASE_METHOD( + EnumerationFx, + "Cap'N Proto - Basic New ArraySchema Serialization", + "[enumeration][capnp][basic][initialized-in-ram") { + auto client_side = GENERATE(true, false); + auto ser_type = GENERATE(SerializationType::CAPNP, SerializationType::JSON); + + auto schema1 = create_schema(); + auto schema2 = ser_des_array_schema(schema1, client_side, ser_type); + + auto all_names1 = schema1->get_enumeration_names(); + auto all_names2 = schema2.get_enumeration_names(); + REQUIRE(vec_cmp(all_names1, all_names2)); + + auto loaded_names1 = schema1->get_loaded_enumeration_names(); + auto loaded_names2 = schema2.get_loaded_enumeration_names(); + REQUIRE(vec_cmp(loaded_names1, loaded_names2)); + + // This is a new schema in RAM, so the loaded names should be the same + // as all names. + REQUIRE(vec_cmp(all_names1, loaded_names1)); + REQUIRE(vec_cmp(all_names2, loaded_names2)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Cap'N Proto - Basic Exisitng ArraySchema Serialization", + "[enumeration][capnp][basic][deserialized-from-disk]") { + create_array(); + + auto client_side = GENERATE(true, false); + auto ser_type = GENERATE(SerializationType::CAPNP, SerializationType::JSON); + + auto schema1 = get_array_schema_latest(); + auto schema2 = ser_des_array_schema(schema1, client_side, ser_type); + + auto all_names1 = schema1->get_enumeration_names(); + auto all_names2 = schema2.get_enumeration_names(); + REQUIRE(vec_cmp(all_names1, all_names2)); + + // This schema was deserialized from disk without any enumerations loaded + // so both of these should be empty. + auto loaded_names1 = schema1->get_loaded_enumeration_names(); + auto loaded_names2 = schema2.get_loaded_enumeration_names(); + + REQUIRE(loaded_names1.empty()); + REQUIRE(loaded_names2.empty()); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Cap'N Proto - Basic ArraySchemaEvolution Serialization", + "[enumeration][capnp][basic][array-schema-evolution]") { + auto client_side = GENERATE(true, false); + auto ser_type = GENERATE(SerializationType::CAPNP, SerializationType::JSON); + + std::vector values1 = {1, 2, 3, 4, 5}; + auto enmr1 = create_enumeration(values1, false, Datatype::INT32, "enmr1"); + + std::vector values2 = {1.0, 2.0, 3.0, 4.0, 5.0}; + auto enmr2 = create_enumeration(values2, true, Datatype::FLOAT64, "enmr2"); + + auto attr = make_shared(HERE(), "ohai", Datatype::INT64); + attr->set_enumeration_name("enmr2"); + + ArraySchemaEvolution ase1; + ase1.add_attribute(attr.get()); + ase1.add_enumeration(enmr1); + ase1.add_enumeration(enmr2); + ase1.drop_attribute("some_attr"); + ase1.drop_enumeration("an_enumeration"); + + auto ase2 = ser_des_array_schema_evolution(&ase1, client_side, ser_type); + + auto enmrs_to_add1 = ase1.enumeration_names_to_add(); + auto enmrs_to_add2 = ase2->enumeration_names_to_add(); + REQUIRE(vec_cmp(enmrs_to_add1, enmrs_to_add2)); + + for (auto& name : enmrs_to_add1) { + REQUIRE(ase1.enumeration_to_add(name) != nullptr); + REQUIRE(ase2->enumeration_to_add(name) != nullptr); + REQUIRE(ase1.enumeration_to_add(name) != ase2->enumeration_to_add(name)); + } +} + +TEST_CASE_METHOD( + EnumerationFx, + "Cap'N Proto - Basic Backwards Compatible Query Serialization", + "[enumeration][capnp][basic][query-condition-old]") { + auto client_side = GENERATE(true, false); + + // Query does not support serialization to JSON + auto ser_type = SerializationType::CAPNP; + + create_array(); + auto array = get_array(QueryType::READ); + + // This is normally invoked by the query class when not being tested. + array->get_enumeration("test_enmr"); + + auto qc1 = create_qc("attr1", (int)2, QueryConditionOp::EQ); + qc1.set_use_enumeration(false); + + Query q1(ctx_.storage_manager(), array); + throw_if_not_ok(q1.set_condition(qc1)); + + Query q2(ctx_.storage_manager(), array); + ser_des_query(&q1, &q2, client_side, ser_type); + + auto qc2 = q2.condition(); + REQUIRE(qc2.has_value()); + + auto& node = qc2.value().ast(); + REQUIRE(node->use_enumeration() == false); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Cap'N Proto - Basic New Query Serialization", + "[enumeration][capnp][basic][query-condition-new]") { + auto client_side = GENERATE(true, false); + + // Query does not support serialization to JSON + auto ser_type = SerializationType::CAPNP; + + create_array(); + auto array = get_array(QueryType::READ); + + // This is normally invoked by the query class when not being tested. + array->get_enumeration("test_enmr"); + + auto qc1 = create_qc("attr1", (int)2, QueryConditionOp::EQ); + qc1.set_use_enumeration(false); + + auto qc2 = create_qc("attr2", std::string("foo"), QueryConditionOp::NE); + QueryCondition qc3; + + throw_if_not_ok(qc1.combine(qc2, QueryConditionCombinationOp::OR, &qc3)); + + Query q1(ctx_.storage_manager(), array); + throw_if_not_ok(q1.set_condition(qc3)); + + Query q2(ctx_.storage_manager(), array); + ser_des_query(&q1, &q2, client_side, ser_type); + + auto qc4 = q2.condition(); + REQUIRE(qc4.has_value()); + + auto& node1 = qc4.value().ast()->get_children()[0]; + auto& node2 = qc4.value().ast()->get_children()[1]; + REQUIRE(node1->use_enumeration() == false); + REQUIRE(node2->use_enumeration() == true); +} + +#endif // ifdef TILEDB_SERIALIZATIONs + +/* ********************************* */ +/* VERIFY SUPPORT CODE */ +/* ********************************* */ + +TEST_CASE_METHOD( + EnumerationFx, + "Check EnumerationFx::vec_cmp", + "[enumeration][test-code][verify]") { + std::vector v1 = {1, 2, 3, 4, 5}; + std::vector v2 = {5, 3, 4, 2, 1}; + + REQUIRE(vec_cmp(v1, v2)); + + std::vector v3 = {}; + REQUIRE(!vec_cmp(v1, v3)); + + std::vector v4 = {1, 2}; + REQUIRE(!vec_cmp(v1, v4)); + + std::vector v5 = {3, 4, 5, 6, 7}; + REQUIRE(!vec_cmp(v1, v5)); +} + +/* ********************************* */ +/* TEST SUPPORT CODE */ +/* ********************************* */ + +struct TypeParams { + TypeParams(Datatype type, uint32_t cell_val_num) + : type_(type) + , cell_val_num_(cell_val_num) { + } + + template + static TypeParams get(const std::vector>&) { + return TypeParams(Datatype::STRING_ASCII, constants::var_num); + } + + static TypeParams get(const std::vector&) { + return TypeParams(Datatype::INT32, 1); + } + + static TypeParams get(const std::vector&) { + return TypeParams(Datatype::UINT32, 1); + } + + static TypeParams get(const std::vector&) { + return TypeParams(Datatype::UINT64, 1); + } + + static TypeParams get(const std::vector&) { + return TypeParams(Datatype::FLOAT32, 1); + } + + static TypeParams get(const std::vector&) { + return TypeParams(Datatype::FLOAT64, 1); + } + + Datatype type_; + uint32_t cell_val_num_; +}; + +EnumerationFx::EnumerationFx() + : uri_("enumeration_test_array") + , ctx_(cfg_) { + rm_array(); + throw_if_not_ok(enc_key_.set_key(EncryptionType::NO_ENCRYPTION, nullptr, 0)); +} + +EnumerationFx::~EnumerationFx() { + rm_array(); +} + +template +shared_ptr EnumerationFx::create_enumeration( + const std::vector& values, + bool ordered, + Datatype type, + std::string name) { + TypeParams tp = TypeParams::get(values); + + if (type != static_cast(255)) { + tp.type_ = type; + } + + if constexpr (std::is_pod_v) { + return Enumeration::create( + name, + tp.type_, + tp.cell_val_num_, + ordered, + values.data(), + values.size() * sizeof(T), + nullptr, + 0); + } else { + uint64_t total_size = 0; + for (auto v : values) { + total_size += v.size(); + } + + std::vector data(total_size, 0); + std::vector offsets; + offsets.reserve(values.size()); + uint64_t curr_offset = 0; + + for (auto v : values) { + std::memcpy(data.data() + curr_offset, v.data(), v.size()); + offsets.push_back(curr_offset); + curr_offset += v.size(); + } + + return Enumeration::create( + name, + tp.type_, + tp.cell_val_num_, + ordered, + data.data(), + total_size, + offsets.data(), + offsets.size() * sizeof(uint64_t)); + } +} + +template +void EnumerationFx::check_enumeration( + shared_ptr enmr, + const std::string& name, + const std::vector& values, + Datatype data_type, + uint32_t cell_val_num, + bool ordered) { + REQUIRE(enmr->name() == name); + REQUIRE(!enmr->path_name().empty()); + REQUIRE(enmr->type() == data_type); + REQUIRE(enmr->cell_val_num() == cell_val_num); + REQUIRE(enmr->ordered() == ordered); + + std::vector data = as_vector(enmr); + REQUIRE(data == values); +} + +template +void EnumerationFx::check_storage_serialization(const std::vector& values) { + auto enmr = create_enumeration(values); + auto tile = serialize_to_tile(enmr); + REQUIRE(tile.size() == calculate_serialized_size(enmr)); +} + +template +void EnumerationFx::check_storage_deserialization( + const std::vector& values) { + auto enmr = create_enumeration(values); + auto tile = serialize_to_tile(enmr); + + Deserializer deserializer(tile.data(), tile.size()); + auto deserialized = Enumeration::deserialize(deserializer); + + REQUIRE(deserialized->name() == enmr->name()); + REQUIRE(deserialized->path_name().empty() == false); + REQUIRE(deserialized->type() == enmr->type()); + REQUIRE(deserialized->cell_val_num() == enmr->cell_val_num()); + REQUIRE(deserialized->ordered() == enmr->ordered()); + REQUIRE(deserialized->cell_size() == enmr->cell_size()); + REQUIRE(deserialized->var_size() == enmr->var_size()); + + auto orig_dspan = enmr->data(); + auto des_dspan = deserialized->data(); + REQUIRE(des_dspan.size() == orig_dspan.size()); + REQUIRE(memcmp(des_dspan.data(), orig_dspan.data(), orig_dspan.size()) == 0); + + if (enmr->var_size()) { + auto orig_ospan = enmr->offsets(); + auto des_ospan = deserialized->offsets(); + REQUIRE(orig_ospan.size() == des_ospan.size()); + REQUIRE( + memcmp(des_ospan.data(), orig_ospan.data(), orig_ospan.size()) == 0); + } +} + +storage_size_t EnumerationFx::calculate_serialized_size( + shared_ptr enmr) { + // Size is the sum of the following sizes: + storage_size_t num_bytes = 0; + + // uint32_t - version + num_bytes += sizeof(uint32_t); + + // uint32_t - name length + num_bytes += sizeof(uint32_t); + + // name.size() bytes + num_bytes += enmr->name().size(); + + // uint32_t - path_name length; + num_bytes += sizeof(uint32_t); + + // path_name.size() bytes + num_bytes += enmr->path_name().size(); + + // uint8_t - data type + num_bytes += sizeof(uint8_t); + + // uint32_t - cell_val_num + num_bytes += sizeof(uint32_t); + + // bool - ordered + num_bytes += sizeof(bool); + + // uint64_t - data.size() + // data.size() bytes + auto dspan = enmr->data(); + num_bytes += sizeof(uint64_t); + num_bytes += dspan.size(); + + // if var_sized: + if (enmr->var_size()) { + auto ospan = enmr->offsets(); + num_bytes += sizeof(uint64_t); + num_bytes += ospan.size(); + } + + return num_bytes; +} + +WriterTile EnumerationFx::serialize_to_tile( + shared_ptr enmr) { + SizeComputationSerializer size_serializer; + enmr->serialize(size_serializer); + + WriterTile tile{WriterTile::from_generic(size_serializer.size())}; + Serializer serializer(tile.data(), tile.size()); + enmr->serialize(serializer); + + return tile; +} + +template +std::vector EnumerationFx::as_vector(shared_ptr enmr) { + std::vector ret; + + if constexpr (std::is_pod_v) { + auto dspan = enmr->data(); + + const T* elems = reinterpret_cast(dspan.data()); + size_t count = dspan.size() / sizeof(T); + + ret.reserve(count); + for (size_t i = 0; i < count; i++) { + ret.push_back(elems[i]); + } + } else { + auto dspan = enmr->data(); + auto ospan = enmr->offsets(); + + auto str_data = reinterpret_cast(dspan.data()); + auto elems = reinterpret_cast(ospan.data()); + size_t count = ospan.size() / sizeof(uint64_t); + + ret.reserve(count); + for (size_t i = 0; i < count; i++) { + uint64_t len; + if (i + 1 < count) { + len = elems[i + 1] - elems[i]; + } else { + len = dspan.size() - elems[i]; + } + ret.emplace_back(str_data + elems[i], len); + } + } + + return ret; +} + +shared_ptr EnumerationFx::create_schema() { + // Create a schema to serialize + auto schema = make_shared(HERE(), ArrayType::SPARSE); + + auto dim = make_shared(HERE(), "dim1", Datatype::INT32); + int range[2] = {0, 1000}; + throw_if_not_ok(dim->set_domain(range)); + + auto dom = make_shared(HERE()); + throw_if_not_ok(dom->add_dimension(dim)); + throw_if_not_ok(schema->set_domain(dom)); + + std::vector values = {"ant", "bat", "cat", "dog", "emu"}; + auto enmr = + create_enumeration(values, false, Datatype::STRING_ASCII, "test_enmr"); + schema->add_enumeration(enmr); + + auto attr1 = make_shared(HERE(), "attr1", Datatype::INT32); + attr1->set_enumeration_name("test_enmr"); + throw_if_not_ok(schema->add_attribute(attr1)); + + auto attr2 = make_shared(HERE(), "attr2", Datatype::STRING_ASCII); + throw_if_not_ok(schema->add_attribute(attr2)); + + return schema; +} + +void EnumerationFx::create_array() { + auto schema = create_schema(); + throw_if_not_ok(ctx_.storage_manager()->array_create(uri_, schema, enc_key_)); +} + +shared_ptr EnumerationFx::get_array(QueryType type) { + auto array = make_shared(HERE(), uri_, ctx_.storage_manager()); + throw_if_not_ok(array->open(type, EncryptionType::NO_ENCRYPTION, nullptr, 0)); + return array; +} + +shared_ptr EnumerationFx::get_array_directory() { + return make_shared( + HERE(), ctx_.resources(), uri_, 0, UINT64_MAX, ArrayDirectoryMode::READ); +} + +shared_ptr EnumerationFx::get_array_schema_latest( + bool load_enmrs) { + auto array_dir = get_array_directory(); + auto schema = array_dir->load_array_schema_latest(enc_key_); + if (load_enmrs) { + array_dir->load_all_enumerations(schema, enc_key_); + } + return schema; +} + +#ifdef TILEDB_SERIALIZATION + +ArraySchema EnumerationFx::ser_des_array_schema( + shared_ptr schema, + bool client_side, + SerializationType stype) { + Buffer buf; + throw_if_not_ok(serialization::array_schema_serialize( + *(schema.get()), stype, &buf, client_side)); + return serialization::array_schema_deserialize(stype, buf); +} + +shared_ptr EnumerationFx::ser_des_array_schema_evolution( + ArraySchemaEvolution* ase, bool client_side, SerializationType stype) { + Buffer buf; + throw_if_not_ok(serialization::array_schema_evolution_serialize( + ase, stype, &buf, client_side)); + + ArraySchemaEvolution* ret; + throw_if_not_ok( + serialization::array_schema_evolution_deserialize(&ret, stype, buf)); + + return shared_ptr(ret); +} + +void EnumerationFx::ser_des_query( + Query* q_in, Query* q_out, bool client_side, SerializationType stype) { + Buffer buf; + BufferList blist; + + throw_if_not_ok( + serialization::query_serialize(q_in, stype, client_side, &blist)); + + flatten_buffer_list(blist, buf); + + throw_if_not_ok(serialization::query_deserialize( + buf, + stype, + client_side, + nullptr, + q_out, + &(ctx_.resources().compute_tp()))); +} + +#else // No TILEDB_SERIALIZATION + +ArraySchema EnumerationFx::ser_des_array_schema( + shared_ptr, bool, SerializationType) { + throw std::logic_error("Serialization not enabled."); +} + +shared_ptr EnumerationFx::ser_des_array_schema_evolution( + ArraySchemaEvolution*, bool, SerializationType) { + throw std::logic_error("Serialization not enabled."); +} + +void EnumerationFx::ser_des_query(Query*, Query*, bool, SerializationType) { + throw std::logic_error("Serialization not enabled."); +} + +#endif // TILEDB_SERIALIZATION + +template +bool EnumerationFx::vec_cmp(std::vector v1, std::vector v2) { + std::sort(v1.begin(), v1.end()); + std::sort(v2.begin(), v2.end()); + + if (v1.size() != v2.size()) { + return false; + } + + for (size_t i = 0; i < v1.size(); i++) { + if (v1[i] != v2[i]) { + return false; + } + } + + return true; +} + +void EnumerationFx::flatten_buffer_list(BufferList& blist, Buffer& buf) { + const auto nbytes = blist.total_size(); + throw_if_not_ok(buf.realloc(nbytes)); + + blist.reset_offset(); + throw_if_not_ok(blist.read(buf.data(), nbytes)); + buf.set_size(nbytes); +} + +void EnumerationFx::rm_array() { + bool is_dir; + throw_if_not_ok(ctx_.resources().vfs().is_dir(uri_, &is_dir)); + if (is_dir) { + throw_if_not_ok(ctx_.resources().vfs().remove_dir(uri_)); + } +} + +template +QueryCondition create_qc( + const char* field_name, T condition_value, const QueryConditionOp& op) { + QueryCondition ret; + + if constexpr (std::is_pod_v) { + throw_if_not_ok(ret.init(field_name, &condition_value, sizeof(T), op)); + } else { + throw_if_not_ok(ret.init( + field_name, condition_value.data(), condition_value.size(), op)); + } + + return ret; +} diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index 5e52ceecd5a5..f14e06168c78 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -85,10 +85,12 @@ if (TILEDB_CPP_API) ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/tiledb_experimental ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/array_deprecated.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/array.h + ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/array_experimental.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/array_schema.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/array_schema_evolution.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/array_schema_experimental.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/attribute.h + ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/attribute_experimental.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/config.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/consolidation_plan_experimental.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/context.h @@ -97,6 +99,7 @@ if (TILEDB_CPP_API) ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/dimension.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/dimension_label_experimental.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/domain.h + ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/enumeration_experimental.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/error.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/exception.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/filter.h @@ -108,6 +111,7 @@ if (TILEDB_CPP_API) ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/object.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/object_iter.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/query.h + ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/query_condition_experimental.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/query_condition.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/query_experimental.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/schema_base.h @@ -148,6 +152,7 @@ set(TILEDB_CORE_SOURCES ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/array_schema/dimension.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/array_schema/dimension_label.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/array_schema/domain.cc + ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/array_schema/enumeration.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/buffer/buffer.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/buffer/buffer_list.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/c_api/api_argument_validator.cc @@ -883,14 +888,22 @@ if(TILEDB_SERIALIZATION) ${CMAKE_CURRENT_SOURCE_DIR}/sm/serialization/${TILEDB_SERIALIZATION_GENERATED_SUBDIR}) # We only need to override the include path and PATH env for EP builds + # note: We run the compiler under `cmake -E env ...` because we need to run capnp executable + # the capnp driver needs to be able to find the plugin executables (eg capnpc-c++). + # For system installed capnp binaries we rely on the user to have their PATH + # include the directory with CapNProto plugin binaries. if(TILEDB_CAPNP_EP_BUILT) - list(APPEND CAPNP_COMPILE_COMMAND "${CMAKE_COMMAND}" -E env PATH="${TILEDB_EP_BASE}/install/bin" "${CAPNP_EXECUTABLE}" compile -I "${TILEDB_EP_BASE}/install/include" -oc++:"${TILEDB_SERIALIZATION_GENERATED_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/sm/serialization/tiledb-rest.capnp" --src-prefix="${CMAKE_CURRENT_SOURCE_DIR}/sm/serialization") + list(APPEND CAPNP_COMPILE_COMMAND "${CMAKE_COMMAND}" -E env PATH="${TILEDB_EP_BASE}/install/bin" "${CAPNP_EXECUTABLE}" compile -I "${TILEDB_EP_BASE}/install/include") + elseif(TILEDB_VCPKG) + set(CAPNP_PLUGIN_DIR $) + list(APPEND CAPNP_COMPILE_COMMAND "${CMAKE_COMMAND}" -E env PATH="${CAPNP_PLUGIN_DIR}" "${CAPNP_EXECUTABLE}" compile "-I${CAPNP_INCLUDE_DIRECTORY}") else() - list(APPEND CAPNP_COMPILE_COMMAND "${CAPNP_EXECUTABLE}" compile -oc++:"${TILEDB_SERIALIZATION_GENERATED_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/sm/serialization/tiledb-rest.capnp" --src-prefix "${CMAKE_CURRENT_SOURCE_DIR}/sm/serialization") + list(APPEND CAPNP_COMPILE_COMMAND "${CAPNP_EXECUTABLE}" compile) endif() - # note: run the compiler under `cmake -E env ...` because we need to run capnp executable - # the capnp driver needs to be able to find the plugin executables (eg capnpc-c++) + # Add the rest of the capnp compile command + list(APPEND CAPNP_COMPILE_COMMAND -oc++:"${TILEDB_SERIALIZATION_GENERATED_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/sm/serialization/tiledb-rest.capnp" --src-prefix="${CMAKE_CURRENT_SOURCE_DIR}/sm/serialization") + add_custom_target( update-serialization ${CMAKE_COMMAND} -E echo "CAPNP_COMPILE_COMMAND: '${CAPNP_COMPILE_COMMAND}'" diff --git a/tiledb/api/c_api/CMakeLists.txt b/tiledb/api/c_api/CMakeLists.txt index 655bc3402249..a1d2cdf7e30d 100644 --- a/tiledb/api/c_api/CMakeLists.txt +++ b/tiledb/api/c_api/CMakeLists.txt @@ -76,6 +76,9 @@ add_subdirectory(data_order) # `dimension_label`: depends on `context`, `datatype`, `data_order` add_subdirectory(dimension_label) +# `enumeration`: depends on `buffer`, `constants`, and `context` +add_subdirectory(enumeration) + # `filesystem`: no dependencies add_subdirectory(filesystem) diff --git a/tiledb/api/c_api/dimension_label/CMakeLists.txt b/tiledb/api/c_api/dimension_label/CMakeLists.txt index 9cfe7b01e707..e8f9d0d214ba 100644 --- a/tiledb/api/c_api/dimension_label/CMakeLists.txt +++ b/tiledb/api/c_api/dimension_label/CMakeLists.txt @@ -46,6 +46,7 @@ target_link_libraries(capi_dimension_label_stub PUBLIC array_schema $(type); + + bool is_ordered = false; + if (ordered != 0) { + is_ordered = true; + } + + try { + *enumeration = tiledb_enumeration_handle_t::make_handle( + std::string(name), + datatype, + cell_val_num, + is_ordered, + data, + data_size, + offsets, + offsets_size); + } catch (...) { + *enumeration = nullptr; + throw; + } + + return TILEDB_OK; +} + +void tiledb_enumeration_free(tiledb_enumeration_t** enumeration) { + ensure_output_pointer_is_valid(enumeration); + ensure_enumeration_is_valid(*enumeration); + tiledb_enumeration_handle_t::break_handle(*enumeration); +} + +capi_return_t tiledb_enumeration_get_name( + tiledb_enumeration_t* enumeration, tiledb_string_t** name) { + ensure_enumeration_is_valid(enumeration); + ensure_output_pointer_is_valid(name); + *name = tiledb_string_handle_t::make_handle(enumeration->name()); + return TILEDB_OK; +} + +capi_return_t tiledb_enumeration_get_type( + tiledb_enumeration_t* enumeration, tiledb_datatype_t* type) { + ensure_enumeration_is_valid(enumeration); + ensure_output_pointer_is_valid(type); + *type = static_cast(enumeration->type()); + return TILEDB_OK; +} + +capi_return_t tiledb_enumeration_get_cell_val_num( + tiledb_enumeration_t* enumeration, uint32_t* cell_val_num) { + ensure_enumeration_is_valid(enumeration); + ensure_output_pointer_is_valid(cell_val_num); + *cell_val_num = enumeration->cell_val_num(); + return TILEDB_OK; +} + +capi_return_t tiledb_enumeration_get_ordered( + tiledb_enumeration_t* enumeration, int* ordered) { + ensure_enumeration_is_valid(enumeration); + ensure_output_pointer_is_valid(ordered); + *ordered = static_cast(enumeration->ordered()); + return TILEDB_OK; +} + +capi_return_t tiledb_enumeration_get_data( + tiledb_enumeration_t* enumeration, const void** data, uint64_t* data_size) { + ensure_enumeration_is_valid(enumeration); + ensure_output_pointer_is_valid(data); + ensure_output_pointer_is_valid(data_size); + auto dspan = enumeration->data(); + *data = dspan.data(); + *data_size = dspan.size(); + return TILEDB_OK; +} + +capi_return_t tiledb_enumeration_get_offsets( + tiledb_enumeration_t* enumeration, + const void** offsets, + uint64_t* offsets_size) { + ensure_enumeration_is_valid(enumeration); + ensure_output_pointer_is_valid(offsets); + ensure_output_pointer_is_valid(offsets_size); + auto ospan = enumeration->offsets(); + *offsets = ospan.data(); + *offsets_size = ospan.size(); + return TILEDB_OK; +} + +capi_return_t tiledb_enumeration_dump( + tiledb_enumeration_t* enumeration, FILE* out) { + ensure_enumeration_is_valid(enumeration); + enumeration->dump(out); + return TILEDB_OK; +} + +} // namespace tiledb::api + +using tiledb::api::api_entry_context; +using tiledb::api::api_entry_void; + +capi_return_t tiledb_enumeration_alloc( + tiledb_ctx_t* ctx, + const char* name, + tiledb_datatype_t type, + uint32_t cell_val_num, + int ordered, + const void* data, + uint64_t data_size, + const void* offsets, + uint64_t offsets_size, + tiledb_enumeration_t** enumeration) noexcept { + return api_entry_context( + ctx, + name, + type, + cell_val_num, + ordered, + data, + data_size, + offsets, + offsets_size, + enumeration); +} + +void tiledb_enumeration_free(tiledb_enumeration_t** enumeration) noexcept { + return api_entry_void(enumeration); +} + +capi_return_t tiledb_enumeration_get_name( + tiledb_ctx_t* ctx, + tiledb_enumeration_t* enumeration, + tiledb_string_t** name) noexcept { + return api_entry_context( + ctx, enumeration, name); +} + +capi_return_t tiledb_enumeration_get_type( + tiledb_ctx_t* ctx, + tiledb_enumeration_t* enumeration, + tiledb_datatype_t* type) noexcept { + return api_entry_context( + ctx, enumeration, type); +} + +capi_return_t tiledb_enumeration_get_cell_val_num( + tiledb_ctx_t* ctx, + tiledb_enumeration_t* enumeration, + uint32_t* cell_val_num) noexcept { + return api_entry_context( + ctx, enumeration, cell_val_num); +} + +capi_return_t tiledb_enumeration_get_ordered( + tiledb_ctx_t* ctx, + tiledb_enumeration_t* enumeration, + int* ordered) noexcept { + return api_entry_context( + ctx, enumeration, ordered); +} + +capi_return_t tiledb_enumeration_get_data( + tiledb_ctx_t* ctx, + tiledb_enumeration_t* enumeration, + const void** data, + uint64_t* data_size) noexcept { + return api_entry_context( + ctx, enumeration, data, data_size); +} + +capi_return_t tiledb_enumeration_get_offsets( + tiledb_ctx_t* ctx, + tiledb_enumeration_t* enumeration, + const void** offsets, + uint64_t* offsets_size) noexcept { + return api_entry_context( + ctx, enumeration, offsets, offsets_size); +} + +capi_return_t tiledb_enumeration_dump( + tiledb_ctx_t* ctx, tiledb_enumeration_t* enumeration, FILE* out) noexcept { + return api_entry_context( + ctx, enumeration, out); +} diff --git a/tiledb/api/c_api/enumeration/enumeration_api_experimental.h b/tiledb/api/c_api/enumeration/enumeration_api_experimental.h new file mode 100644 index 000000000000..46326047661b --- /dev/null +++ b/tiledb/api/c_api/enumeration/enumeration_api_experimental.h @@ -0,0 +1,281 @@ +/** + * @file tiledb/api/c_api/enumeration/enumeration_api_experimental.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares the enumeration section of the C API for TileDB. + */ + +#ifndef TILEDB_CAPI_ENUMERATION_EXPERIMENTAL_H +#define TILEDB_CAPI_ENUMERATION_EXPERIMENTAL_H + +#include + +#include "../api_external_common.h" +#include "../context/context_api_external.h" +#include "../datatype/datatype_api_external.h" +#include "../string/string_api_external.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** A TileDB dimension. */ +typedef struct tiledb_enumeration_handle_t tiledb_enumeration_t; + +/** + * Creates an Enumeration. + * + * **Example:** + * + * @code{.c} + * tiledb_enumeration_t* enumeration; + * void* data = get_data(); + * uint64_t data_size = get_data_size(); + * tiledb_enumeration_alloc( + * ctx, + * TILEDB_INT64, + * cell_val_num, + * FALSE, + * data, + * data_size, + * nullptr, + * 0, + * &enumeration); + * @endcode + * + * @param ctx The TileDB context. + * @param name The name of the enumeration. + * @param type The enumeration type. + * @param cell_val_num The number of values per enumeration value. + * @param ordered Whether this enumeration should be considered as ordered. + * @param data A pointer to the enumeration value data. + * @param data_size The length of the data buffer provided. + * @param offsets A pointer to the offsets buffer if cell_vall_num + * is TILEDB_VAR_NUM. + * @param offsets_size The length of the offsets buffer, zero if no offsets. + * @param enumeration The newly allocated enumeration. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_enumeration_alloc( + tiledb_ctx_t* ctx, + const char* name, + tiledb_datatype_t type, + uint32_t cell_val_num, + int ordered, + const void* data, + uint64_t data_size, + const void* offsets, + uint64_t offsets_size, + tiledb_enumeration_t** enumeration) TILEDB_NOEXCEPT; + +/** + * Destroys a TileDB enumeration, freeing associated memory. + * + * **Example:** + * + * @code{.c} + * tiledb_enumeration_free(&enumeration); + * @endcode + * + * @param enumeration The enumeration to be destroyed. + */ +TILEDB_EXPORT void tiledb_enumeration_free(tiledb_enumeration_t** enumeration) + TILEDB_NOEXCEPT; + +/** + * Return the datatype of the enumeration values + * + * **Example:** + * + * @code{.c} + * tiledb_string_t* name_str; + * tiledb_enumeration_get_type(ctx, enumeration, &name_str); + * const char* name; + * size_t name_len; + * tiledb_string_view(name_str, &name, &name_len); + * @endcode + * + * @param ctx The TileDB context. + * @param enumeration The enumeration. + * @param name The name of the enumeration. + * @return `TILEDB_OK` or `TILEDB_ERR`. + */ +TILEDB_EXPORT capi_return_t tiledb_enumeration_get_name( + tiledb_ctx_t* ctx, + tiledb_enumeration_t* enumeration, + tiledb_string_t** name) TILEDB_NOEXCEPT; + +/** + * Return the datatype of the enumeration values + * + * **Example:** + * + * @code{.c} + * tiledb_datatype_t type; + * tiledb_enumeration_get_type(ctx, enumeration, &type); + * @endcode + * + * @param ctx The TileDB context. + * @param enumeration The enumeration. + * @param type The data type of the enumeration. + * @return `TILEDB_OK` or `TILEDB_ERR`. + */ +TILEDB_EXPORT capi_return_t tiledb_enumeration_get_type( + tiledb_ctx_t* ctx, + tiledb_enumeration_t* enumeration, + tiledb_datatype_t* type) TILEDB_NOEXCEPT; + +/** + * Return the cell value number of the enumeration values + * + * **Example:** + * + * @code{.c} + * uint32_t cell_val_num; + * tiledb_enumeration_get_cell_val_num(ctx, enumeration, &cell_val_num); + * @endcode + * + * @param ctx The TileDB context. + * @param enumeration The enumeration. + * @param type The cell value number of the enumeration. + * @return `TILEDB_OK` or `TILEDB_ERR`. + */ +TILEDB_EXPORT capi_return_t tiledb_enumeration_get_cell_val_num( + tiledb_ctx_t* ctx, + tiledb_enumeration_t* enumeration, + uint32_t* cell_val_num) TILEDB_NOEXCEPT; + +/** + * Return whether the enumeration values should be considered ordered. + * + * **Example:** + * + * @code{.c} + * int ordered; + * tiledb_enumeration_get_ordered(ctx, enumeration, &ordered); + * @endcode + * + * The cell values are considered if the value in ordered after `TILEDB_OK` + * is returned is non-zero. I.e., this is standard `int` as `bool` behavior. + * + * @param ctx The TileDB context. + * @param enumeration The enumeration. + * @param ordered A boolean value indicating whether the values are ordered. + * @return `TILEDB_OK` or `TILEDB_ERR`. + */ +TILEDB_EXPORT capi_return_t tiledb_enumeration_get_ordered( + tiledb_ctx_t* ctx, + tiledb_enumeration_t* enumeration, + int* ordered) TILEDB_NOEXCEPT; + +/** + * Return a pointer to and size of the enumerations underlying value data + * + * **Example:** + * + * @code{.c} + * void* data = NULL; + * uint64_t data_size = 0; + * tiledb_enumeration_get_data(ctx, enumeration, &data, &data_size); + * @endcode + * + * The pointer returned from this function references internal data managed + * by the enumeration. As such, clients should not attempt to free it, or + * access it beyond the lifetime of the enumeration instance. + * + * @param ctx The TileDB context. + * @param enumeration The enumeration. + * @param data The returned pointer to this enumeration value buffer. + * @param data_size The length of the buffer pointed to by data. + * @return `TILEDB_OK` or `TILEDB_ERR`. + */ +TILEDB_EXPORT capi_return_t tiledb_enumeration_get_data( + tiledb_ctx_t* ctx, + tiledb_enumeration_t* enumeration, + const void** data, + uint64_t* data_size) TILEDB_NOEXCEPT; + +/** + * Return a pointer to and size of the enumerations underlying value offsets + * + * **Example:** + * + * @code{.c} + * void* offsets = NULL; + * uint64_t offsets_size = 0; + * tiledb_enumeration_get_offsets(ctx, enumeration, &offsets, &offsets_size); + * @endcode + * + * The pointer returned from this function references internal data managed + * by the enumeration. As such, clients should not attempt to free it, or + * access it beyond the lifetime of the enumeration instance. + * + * If the enumeration values are var sized (i.e., cell_var_num is + * TILEDB_VAR_NUM) the offsets buffer will contain a `uint64_t` value for the + * starting offset of each underlying enumeration value. Note that the number + * of offsets is calculated as `offsets_size / sizeof(uint64_t)`. + * + * @param ctx The TileDB context. + * @param enumeration The enumeration. + * @param data The returned pointer to this enumeration offsets buffer. + * @param data_size The length of the buffer pointed to by offsets. + * @return `TILEDB_OK` or `TILEDB_ERR`. + */ +TILEDB_EXPORT capi_return_t tiledb_enumeration_get_offsets( + tiledb_ctx_t* ctx, + tiledb_enumeration_t* enumeration, + const void** offsets, + uint64_t* offsets_size) TILEDB_NOEXCEPT; + +/** + * Dumps the contents of an Enumeration in ASCII form to some output (e.g., + * file or stdout). + * + * **Example:** + * + * The following prints the enumeration dump to standard output. + * + * @code{.c} + * tiledb_enumeration_dump(ctx, enmr, stdout); + * @endcode + * + * @param ctx The TileDB context. + * @param attr The attribute. + * @param out The output. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error./ + */ +TILEDB_EXPORT capi_return_t tiledb_enumeration_dump( + tiledb_ctx_t* ctx, + tiledb_enumeration_t* enumeration, + FILE* out) TILEDB_NOEXCEPT; + +#ifdef __cplusplus +} +#endif + +#endif // TILEDB_CAPI_ENUMERATION_EXPERIMENTAL_H diff --git a/tiledb/api/c_api/enumeration/enumeration_api_internal.h b/tiledb/api/c_api/enumeration/enumeration_api_internal.h new file mode 100644 index 000000000000..2a984d27e03a --- /dev/null +++ b/tiledb/api/c_api/enumeration/enumeration_api_internal.h @@ -0,0 +1,139 @@ +/** + * @file tiledb/api/c_api/enumeration/enumeration_api_internal.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares the enumeration section of the C API for TileDB. + */ + +#ifndef TILEDB_CAPI_ENUMERATION_INTERNAL_H +#define TILEDB_CAPI_ENUMERATION_INTERNAL_H + +#include "enumeration_api_experimental.h" +#include "tiledb/api/c_api_support/handle/handle.h" +#include "tiledb/common/common.h" +#include "tiledb/sm/array_schema/enumeration.h" + +/** + * Handle `struct` for API enumeration objects. + */ +struct tiledb_enumeration_handle_t + : public tiledb::api::CAPIHandle { + private: + shared_ptr enumeration_; + + public: + /** Type name. */ + static constexpr std::string_view object_type_name{"enumeration"}; + + template + tiledb_enumeration_handle_t(Arg&&... arg) + : enumeration_( + tiledb::sm::Enumeration::create(std::forward(arg)...)) { + } + + /** + * Constructor from `shared_ptr` copies the shared pointer. + */ + explicit tiledb_enumeration_handle_t( + shared_ptr& e) + : enumeration_(e) { + } + + /** + * Copy the underlying enumeration object. + */ + [[nodiscard]] shared_ptr copy() { + return enumeration_; + } + + /** + * Return the name of the enumeration. + */ + [[nodiscard]] inline const std::string& name() const { + return enumeration_->name(); + } + + /** + * Return the data type of the enumeration values. + */ + [[nodiscard]] inline tiledb_datatype_t type() const { + return static_cast(enumeration_->type()); + } + + /** + * Return the cell_val_num of the enumeration values + */ + [[nodiscard]] inline uint32_t cell_val_num() const { + return enumeration_->cell_val_num(); + } + + /** + * Return a bool indicating whether the enumeration values are ordered. + */ + [[nodiscard]] inline bool ordered() const { + return enumeration_->ordered(); + } + + /** + * Return a pointer and size tuple for the underlying data type. + */ + [[nodiscard]] inline const span data() const { + return enumeration_->data(); + } + + /** + * Return a pointer and size tuple for the underlying offsets buffer. + */ + [[nodiscard]] inline const span offsets() const { + return enumeration_->offsets(); + } + + /** + * Dump a representation of the Enumeration to out + * + * @param out Where to display the output, stdout if nullptr + */ + inline void dump(FILE* out) const { + enumeration_->dump(out); + } +}; + +namespace tiledb::api { + +/** + * Returns after successfully validating an error. Throws otherwise. + * + * @param dim A possibly-valid enumeration handle. + */ +inline void ensure_enumeration_is_valid(const tiledb_enumeration_handle_t* e) { + ensure_handle_is_valid(e); +} + +} // namespace tiledb::api + +#endif // TILEDB_CAPI_ENUMERATION_INTERNAL_H diff --git a/tiledb/api/c_api/enumeration/test/CMakeLists.txt b/tiledb/api/c_api/enumeration/test/CMakeLists.txt new file mode 100644 index 000000000000..7ace61d484d8 --- /dev/null +++ b/tiledb/api/c_api/enumeration/test/CMakeLists.txt @@ -0,0 +1,32 @@ +# +# tiledb/api/c_api/enumeration/test/CMakeLists.txt +# +# The MIT License +# +# Copyright (c) 2023 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +include(unit_test) + +commence(unit_test capi_enumeration) + this_target_sources(unit_capi_enumeration.cc) + this_target_object_libraries(capi_enumeration_stub) +conclude(unit_test) diff --git a/tiledb/api/c_api/enumeration/test/compile_capi_enumeration_stub_main.cc b/tiledb/api/c_api/enumeration/test/compile_capi_enumeration_stub_main.cc new file mode 100644 index 000000000000..0b7139553f2a --- /dev/null +++ b/tiledb/api/c_api/enumeration/test/compile_capi_enumeration_stub_main.cc @@ -0,0 +1,39 @@ +/** + * @file tiledb/api/c_api/enumeration/test/compile_capi_enumeration_stub_main.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "../enumeration_api_internal.h" +#include "tiledb/sm/enums/datatype.h" + +int main() { + try { + tiledb_enumeration_handle_t e{ + "fooo", tiledb::sm::Datatype::INT32, 1, 0, nullptr, 0, nullptr, 0}; + } catch (...) { + } + return 0; +} diff --git a/tiledb/api/c_api/enumeration/test/unit_capi_enumeration.cc b/tiledb/api/c_api/enumeration/test/unit_capi_enumeration.cc new file mode 100644 index 000000000000..a185f5e4d7ad --- /dev/null +++ b/tiledb/api/c_api/enumeration/test/unit_capi_enumeration.cc @@ -0,0 +1,471 @@ +/** + * @file tiledb/api/c_api/enumeration/test/unit_capi_enumeration.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#define CATCH_CONFIG_MAIN +#include +#include "../enumeration_api_experimental.h" +#include "tiledb/api/c_api_test_support/testsupport_capi_context.h" +#include "tiledb/sm/misc/constants.h" + +// TILEDB_VAR_NUM is defined in tiledb.h which we can't use without linking +// against the entire libtiledb. Instead we'll just declare our own. +#ifndef TILEDB_VAR_NUM +#define TILEDB_VAR_NUM tiledb::sm::constants::var_num +#endif + +using namespace tiledb::api::test_support; + +struct FixedSizeEnumeration { + FixedSizeEnumeration() { + uint32_t values[5] = {1, 2, 3, 4, 5}; + auto rc = tiledb_enumeration_alloc( + ctx_.context, + "an_enumeration", + TILEDB_UINT32, + 1, + 0, + values, + sizeof(uint32_t) * 5, + nullptr, + 0, + &enumeration_); + REQUIRE(rc == TILEDB_OK); + } + + ~FixedSizeEnumeration() { + tiledb_enumeration_free(&enumeration_); + } + + ordinary_context ctx_; + tiledb_enumeration_t* enumeration_; +}; + +struct VarSizeEnumeration { + VarSizeEnumeration() { + const char* values = "foobarbazbingobango"; + uint64_t offsets[5] = {0, 3, 6, 9, 14}; + auto rc = tiledb_enumeration_alloc( + ctx_.context, + "an_enumeration", + TILEDB_STRING_UTF8, + TILEDB_VAR_NUM, + 0, + values, + strlen(values), + offsets, + 5 * sizeof(uint64_t), + &enumeration_); + REQUIRE(rc == TILEDB_OK); + } + + ~VarSizeEnumeration() { + tiledb_enumeration_free(&enumeration_); + } + + ordinary_context ctx_; + tiledb_enumeration_t* enumeration_; +}; + +TEST_CASE( + "C API: tiledb_enumeration_alloc argument validation", + "[capi][enumeration]") { + ordinary_context ctx{}; + tiledb_enumeration_t* enumeration = nullptr; + + int32_t values[5] = {1, 2, 3, 4, 5}; + const char* data = "foobarbazbingobango"; + uint64_t offsets[5] = {0, 3, 6, 9, 14}; + + SECTION("success - fixed size") { + auto rc = tiledb_enumeration_alloc( + ctx.context, + "an_enumeration", + TILEDB_UINT32, + 1, + 0, + values, + sizeof(uint32_t) * 5, + nullptr, + 0, + &enumeration); + REQUIRE(rc == TILEDB_OK); + tiledb_enumeration_free(&enumeration); + } + + SECTION("success - var size") { + auto rc = tiledb_enumeration_alloc( + ctx.context, + "an_enumeration", + TILEDB_STRING_ASCII, + TILEDB_VAR_NUM, + 0, + (void*)data, + strlen(data), + offsets, + sizeof(uint64_t) * 5, + &enumeration); + REQUIRE(rc == TILEDB_OK); + tiledb_enumeration_free(&enumeration); + } + + SECTION("failure - null context") { + auto rc = tiledb_enumeration_alloc( + nullptr, + "an_enumeration", + TILEDB_UINT32, + 1, + 0, + values, + sizeof(uint32_t) * 5, + nullptr, + 0, + &enumeration); + REQUIRE(rc == TILEDB_INVALID_CONTEXT); + tiledb_enumeration_free(&enumeration); + } + + SECTION("failure - invalid name") { + auto rc = tiledb_enumeration_alloc( + ctx.context, + nullptr, + TILEDB_UINT32, + 1, + 0, + values, + sizeof(uint32_t) * 5, + nullptr, + 0, + &enumeration); + REQUIRE(rc == TILEDB_ERR); + tiledb_enumeration_free(&enumeration); + } + + SECTION("failure - invalid datatype") { + auto rc = tiledb_enumeration_alloc( + ctx.context, + "an_enumeration", + (tiledb_datatype_t)255, + 1, + 0, + values, + sizeof(uint32_t) * 5, + nullptr, + 0, + &enumeration); + REQUIRE(rc == TILEDB_ERR); + tiledb_enumeration_free(&enumeration); + } + + SECTION("failure - data nullptr") { + auto rc = tiledb_enumeration_alloc( + ctx.context, + "an_enumeration", + TILEDB_INT32, + 1, + 0, + nullptr, + sizeof(uint32_t) * 5, + nullptr, + 0, + &enumeration); + REQUIRE(rc == TILEDB_ERR); + tiledb_enumeration_free(&enumeration); + } + + SECTION("failure - data_size == 0") { + auto rc = tiledb_enumeration_alloc( + ctx.context, + "an_enumeration", + TILEDB_INT32, + 1, + 0, + values, + 0, + nullptr, + 0, + &enumeration); + REQUIRE(rc == TILEDB_ERR); + tiledb_enumeration_free(&enumeration); + } + + SECTION("failure - enumeration nullptr") { + auto rc = tiledb_enumeration_alloc( + ctx.context, + "an_enumeration", + TILEDB_INT32, + 1, + 0, + values, + sizeof(uint32_t) * 5, + nullptr, + 0, + nullptr); + REQUIRE(rc == TILEDB_ERR); + tiledb_enumeration_free(&enumeration); + } + + SECTION("failure - offsets nullptr") { + auto rc = tiledb_enumeration_alloc( + ctx.context, + "an_enumeration", + TILEDB_STRING_ASCII, + TILEDB_VAR_NUM, + 0, + (void*)data, + strlen(data), + nullptr, + sizeof(uint64_t) * 5, + &enumeration); + REQUIRE(rc == TILEDB_ERR); + tiledb_enumeration_free(&enumeration); + } + + SECTION("failure - offsets_size == 0") { + auto rc = tiledb_enumeration_alloc( + ctx.context, + "an_enumeration", + TILEDB_STRING_ASCII, + TILEDB_VAR_NUM, + 0, + (void*)data, + strlen(data), + offsets, + 0, + &enumeration); + REQUIRE(rc == TILEDB_ERR); + tiledb_enumeration_free(&enumeration); + } +} + +TEST_CASE( + "C API: tiledb_enumeration_free argument validation", + "[capi][enumeration]") { + REQUIRE_NOTHROW(tiledb_enumeration_free(nullptr)); +} + +TEST_CASE( + "C API: tiledb_enumeration_get_type argument validation", + "[capi][enumeration]") { + FixedSizeEnumeration fe; + VarSizeEnumeration ve; + tiledb_datatype_t dt; + + SECTION("success") { + auto rc = + tiledb_enumeration_get_type(fe.ctx_.context, fe.enumeration_, &dt); + REQUIRE(rc == TILEDB_OK); + REQUIRE(dt == TILEDB_UINT32); + + rc = tiledb_enumeration_get_type(ve.ctx_.context, ve.enumeration_, &dt); + REQUIRE(rc == TILEDB_OK); + REQUIRE(dt == TILEDB_STRING_UTF8); + } + + SECTION("failure - invalid context") { + auto rc = tiledb_enumeration_get_type(nullptr, fe.enumeration_, &dt); + REQUIRE(rc == TILEDB_INVALID_CONTEXT); + } + + SECTION("failure - invalid enumeration") { + auto rc = tiledb_enumeration_get_type(fe.ctx_.context, nullptr, &dt); + REQUIRE(rc == TILEDB_ERR); + } + + SECTION("failure - invalid type pointer") { + auto rc = + tiledb_enumeration_get_type(fe.ctx_.context, fe.enumeration_, nullptr); + REQUIRE(rc == TILEDB_ERR); + } +} + +TEST_CASE( + "C API: tiledb_enumeration_get_cell_val_num argument validation", + "[capi][enumeration]") { + FixedSizeEnumeration fe; + VarSizeEnumeration ve; + uint32_t cvn; + + SECTION("success") { + auto rc = tiledb_enumeration_get_cell_val_num( + fe.ctx_.context, fe.enumeration_, &cvn); + REQUIRE(rc == TILEDB_OK); + REQUIRE(cvn == 1); + + rc = tiledb_enumeration_get_cell_val_num( + ve.ctx_.context, ve.enumeration_, &cvn); + REQUIRE(rc == TILEDB_OK); + REQUIRE(cvn == TILEDB_VAR_NUM); + } + + SECTION("failure - invalid context") { + auto rc = + tiledb_enumeration_get_cell_val_num(nullptr, fe.enumeration_, &cvn); + REQUIRE(rc == TILEDB_INVALID_CONTEXT); + } + + SECTION("failure - invalid enumeration") { + auto rc = + tiledb_enumeration_get_cell_val_num(fe.ctx_.context, nullptr, &cvn); + REQUIRE(rc == TILEDB_ERR); + } + + SECTION("failure - invalid cell_val_num pointer") { + auto rc = tiledb_enumeration_get_cell_val_num( + fe.ctx_.context, fe.enumeration_, nullptr); + REQUIRE(rc == TILEDB_ERR); + } +} + +TEST_CASE( + "C API: tiledb_enumeration_get_ordered argument validation", + "[capi][enumeration]") { + FixedSizeEnumeration fe; + VarSizeEnumeration ve; + int o; + + SECTION("success") { + auto rc = + tiledb_enumeration_get_ordered(fe.ctx_.context, fe.enumeration_, &o); + REQUIRE(rc == TILEDB_OK); + REQUIRE(!o); + + rc = tiledb_enumeration_get_ordered(ve.ctx_.context, ve.enumeration_, &o); + REQUIRE(rc == TILEDB_OK); + REQUIRE(!o); + } + + SECTION("failure - invalid context") { + auto rc = tiledb_enumeration_get_ordered(nullptr, fe.enumeration_, &o); + REQUIRE(rc == TILEDB_INVALID_CONTEXT); + } + + SECTION("failure - invalid enumeration") { + auto rc = tiledb_enumeration_get_ordered(fe.ctx_.context, nullptr, &o); + REQUIRE(rc == TILEDB_ERR); + } + + SECTION("failure - invalid ordered pointer") { + auto rc = tiledb_enumeration_get_ordered( + fe.ctx_.context, fe.enumeration_, nullptr); + REQUIRE(rc == TILEDB_ERR); + } +} + +TEST_CASE( + "C API: tiledb_enumeration_get_data argument validation", + "[capi][enumeration]") { + FixedSizeEnumeration fe; + VarSizeEnumeration ve; + const void* d; + uint64_t ds; + + uint32_t fixed_expect[5] = {1, 2, 3, 4, 5}; + const char* var_expect = "foobarbazbingobango"; + + SECTION("success") { + auto rc = + tiledb_enumeration_get_data(fe.ctx_.context, fe.enumeration_, &d, &ds); + REQUIRE(rc == TILEDB_OK); + REQUIRE(std::memcmp(fixed_expect, d, sizeof(uint32_t) * 5) == 0); + REQUIRE(ds == sizeof(uint32_t) * 5); + + rc = tiledb_enumeration_get_data(ve.ctx_.context, ve.enumeration_, &d, &ds); + REQUIRE(rc == TILEDB_OK); + REQUIRE(std::memcmp(var_expect, d, strlen(var_expect)) == 0); + REQUIRE(ds == strlen(var_expect)); + } + + SECTION("failure - invalid context") { + auto rc = tiledb_enumeration_get_data(nullptr, fe.enumeration_, &d, &ds); + REQUIRE(rc == TILEDB_INVALID_CONTEXT); + } + + SECTION("failure - invalid enumeration") { + auto rc = tiledb_enumeration_get_data(fe.ctx_.context, nullptr, &d, &ds); + REQUIRE(rc == TILEDB_ERR); + } + + SECTION("failure - invalid data pointer") { + auto rc = tiledb_enumeration_get_data( + fe.ctx_.context, fe.enumeration_, nullptr, &ds); + REQUIRE(rc == TILEDB_ERR); + } + + SECTION("failure - invalid data size pointer") { + auto rc = tiledb_enumeration_get_data( + fe.ctx_.context, fe.enumeration_, &d, nullptr); + REQUIRE(rc == TILEDB_ERR); + } +} + +TEST_CASE( + "C API: tiledb_enumeration_get_offsets argument validation", + "[capi][enumeration]") { + FixedSizeEnumeration fe; + VarSizeEnumeration ve; + const void* o; + uint64_t os; + + uint64_t var_expect[5] = {0, 3, 6, 9, 14}; + + SECTION("success") { + auto rc = tiledb_enumeration_get_offsets( + fe.ctx_.context, fe.enumeration_, &o, &os); + REQUIRE(rc == TILEDB_OK); + REQUIRE(o == nullptr); + REQUIRE(os == 0); + + rc = tiledb_enumeration_get_offsets( + ve.ctx_.context, ve.enumeration_, &o, &os); + REQUIRE(rc == TILEDB_OK); + REQUIRE(std::memcmp(var_expect, o, sizeof(uint64_t) * 5) == 0); + REQUIRE(os == sizeof(uint64_t) * 5); + } + + SECTION("failure - invalid context") { + auto rc = tiledb_enumeration_get_offsets(nullptr, fe.enumeration_, &o, &os); + REQUIRE(rc == TILEDB_INVALID_CONTEXT); + } + + SECTION("failure - invalid enumeration") { + auto rc = tiledb_enumeration_get_offsets(fe.ctx_.context, nullptr, &o, &os); + REQUIRE(rc == TILEDB_ERR); + } + + SECTION("failure - invalid offsets pointer") { + auto rc = tiledb_enumeration_get_offsets( + fe.ctx_.context, fe.enumeration_, nullptr, &os); + REQUIRE(rc == TILEDB_ERR); + } + + SECTION("failure - invalid offsets size pointer") { + auto rc = tiledb_enumeration_get_offsets( + fe.ctx_.context, fe.enumeration_, &o, nullptr); + REQUIRE(rc == TILEDB_ERR); + } +} diff --git a/tiledb/sm/array/array.cc b/tiledb/sm/array/array.cc index f9a05e52cba8..5a01fc1008ed 100644 --- a/tiledb/sm/array/array.cc +++ b/tiledb/sm/array/array.cc @@ -581,6 +581,48 @@ void Array::delete_fragments_list( } } +shared_ptr Array::get_enumeration( + const std::string& enumeration_name) { + if (remote_) { + throw ArrayException("Unable to load all enumerations; Array is remote."); + } + + if (!is_open_) { + throw ArrayException("Cannot get enumeration; Array is not open"); + } + + if (array_schema_latest_->is_enumeration_loaded(enumeration_name)) { + return array_schema_latest_->get_enumeration(enumeration_name); + } + + return array_dir_.load_enumeration( + array_schema_latest_, enumeration_name, get_encryption_key()); +} + +void Array::load_all_enumerations(bool latest_only) { + if (remote_) { + throw ArrayException("Unable to load all enumerations; Array is remote."); + } + + if (!is_open_) { + throw ArrayException("Cannot load all enumerations; Array is not open"); + } + + std::vector> schemas; + if (latest_only) { + schemas.emplace_back(array_schema_latest_); + } else { + schemas.reserve(array_schemas_all_.size()); + for (auto& iter : array_schemas_all_) { + schemas.emplace_back(iter.second); + } + } + + for (auto& schema : schemas) { + array_dir_.load_all_enumerations(schema, get_encryption_key()); + } +} + bool Array::is_empty() const { return fragment_metadata_.empty(); } diff --git a/tiledb/sm/array/array.h b/tiledb/sm/array/array.h index 92f2fccf14f4..a005e15b4050 100644 --- a/tiledb/sm/array/array.h +++ b/tiledb/sm/array/array.h @@ -257,6 +257,31 @@ class Array { return fragment_metadata_; } + /** + * Get the enumeration for the given name. + * + * This function retrieves the enumeration for the given name. If the + * corresponding enumeration has not been loaded from storage it is + * loaded before this function returns. + * + * @param enumeration_name The name of the enumeration. + * @return shared_ptr or nullptr on failure. + */ + shared_ptr get_enumeration( + const std::string& enumeration_name); + + /** + * Load all enumerations for the array. + * + * Ensure that all enumerations have been loaded. If latest_only is true + * (the default) then only enumerations for the latest schema are loaded. + * When latest_only is false, all schemas have their enumerations loaded. + * + * @param latest_only Whether to load enumerations for just the latest + * schema or all schemas. + */ + void load_all_enumerations(bool latest_only = true); + /** * Returns `true` if the array is empty at the time it is opened. * The funciton returns `false` if the array is not open. diff --git a/tiledb/sm/array/array_directory.cc b/tiledb/sm/array/array_directory.cc index 6223d0536f2d..579c2d0c64b2 100644 --- a/tiledb/sm/array/array_directory.cc +++ b/tiledb/sm/array/array_directory.cc @@ -33,6 +33,7 @@ #include "tiledb/sm/array/array_directory.h" #include "tiledb/common/logger.h" #include "tiledb/common/stdx_string.h" +#include "tiledb/sm/array_schema/enumeration.h" #include "tiledb/sm/filesystem/vfs.h" #include "tiledb/sm/misc/constants.h" #include "tiledb/sm/misc/parallel_functions.h" @@ -187,6 +188,46 @@ ArrayDirectory::load_all_array_schemas( return array_schemas; } +shared_ptr ArrayDirectory::load_enumeration( + shared_ptr schema, + const std::string& enumeration_name, + const EncryptionKey& encryption_key) const { + auto timer_se = resources_.get().stats().start_timer("sm_load_enumeration"); + + auto path_name = schema->get_enumeration_path_name(enumeration_name); + + auto enmr_uri = uri_.join_path(constants::array_schema_dir_name) + .join_path(constants::array_enumerations_dir_name) + .join_path(path_name); + + auto&& tile = GenericTileIO::load(resources_, enmr_uri, 0, encryption_key); + resources_.get().stats().add_counter("read_enumeration_size", tile.size()); + + Deserializer deserializer(tile.data(), tile.size()); + auto enum_ptr = Enumeration::deserialize(deserializer); + + schema->store_enumeration(enum_ptr); + + return enum_ptr; +} + +void ArrayDirectory::load_all_enumerations( + shared_ptr schema, const EncryptionKey& encryption_key) const { + std::vector enmr_names; + for (auto& enmr_name : schema->get_enumeration_names()) { + if (schema->is_enumeration_loaded(enmr_name)) { + continue; + } + enmr_names.push_back(enmr_name); + } + + auto& tp = resources_.get().compute_tp(); + throw_if_not_ok(parallel_for(&tp, 0, enmr_names.size(), [&](size_t i) { + load_enumeration(schema, enmr_names[i], encryption_key); + return Status::Ok(); + })); +} + const URI& ArrayDirectory::uri() const { return uri_; } @@ -1205,10 +1246,12 @@ Status ArrayDirectory::compute_array_schema_uris( if (!array_schema_dir_uris.empty()) { array_schema_uris_.reserve( array_schema_uris_.size() + array_schema_dir_uris.size()); - std::copy( - array_schema_dir_uris.begin(), - array_schema_dir_uris.end(), - std::back_inserter(array_schema_uris_)); + for (auto& uri : array_schema_dir_uris) { + if (uri.last_path_part() == constants::array_enumerations_dir_name) { + continue; + } + array_schema_uris_.push_back(uri); + } } return Status::Ok(); diff --git a/tiledb/sm/array/array_directory.h b/tiledb/sm/array/array_directory.h index 76adc29f0393..8d2e35348b34 100644 --- a/tiledb/sm/array/array_directory.h +++ b/tiledb/sm/array/array_directory.h @@ -385,6 +385,29 @@ class ArrayDirectory { std::unordered_map> load_all_array_schemas(const EncryptionKey& encryption_key) const; + /** + * Load an enumeration from schema with the given name. + * + * @param schema The ArraySchema that references the enumeration name. + * @param enumeration_name The name of the enumeration to load. + * @param encryption_key The encryption key to use. + * @return shared_ptr The loaded enumeration. + */ + shared_ptr load_enumeration( + shared_ptr schema, + const std::string& enumeration_name, + const EncryptionKey& encryption_key) const; + + /** + * Load all enumerations for the given schema. + * + * @param schema The ArraySchema to load Enumerations for + * @param encryption_key The encryption key to use. + */ + void load_all_enumerations( + shared_ptr schema, + const EncryptionKey& encryption_key) const; + /** Returns the array URI. */ const URI& uri() const; diff --git a/tiledb/sm/array/test/CMakeLists.txt b/tiledb/sm/array/test/CMakeLists.txt index 3f06e3922eeb..62b222de11ea 100644 --- a/tiledb/sm/array/test/CMakeLists.txt +++ b/tiledb/sm/array/test/CMakeLists.txt @@ -28,7 +28,7 @@ include(unit_test) commence(unit_test array) this_target_sources(main.cc unit_array_directory.cc) - this_target_link_libraries(array) + this_target_link_libraries(array context_resources) conclude(unit_test) commence(unit_test consistency) diff --git a/tiledb/sm/array_schema/CMakeLists.txt b/tiledb/sm/array_schema/CMakeLists.txt index f9616f3b7092..bff3c61bb334 100644 --- a/tiledb/sm/array_schema/CMakeLists.txt +++ b/tiledb/sm/array_schema/CMakeLists.txt @@ -32,7 +32,14 @@ include(object_library) # commence(object_library attribute) this_target_sources(attribute.cc) - this_target_object_libraries(baseline buffer constants filter_pipeline range stringx) + this_target_object_libraries( + baseline + buffer + constants + filter_pipeline + range + stringx + uuid) conclude(object_library) # @@ -51,13 +58,27 @@ commence(object_library domain) this_target_object_libraries(datum dimension math) conclude(object_library) +# +# `enumeration` object library +# +commence(object_library enumeration) + this_target_sources(enumeration.cc) + this_target_object_libraries(buffer constants uuid) +conclude(object_library) + # # `array_schema` object library # commence(object_library array_schema) this_target_sources(array_schema.cc dimension_label.cc) this_target_object_libraries( - attribute domain time uri_format uuid vfs) + attribute domain enumeration time uri_format uuid vfs) conclude(object_library) +# This is linked outside the object_library scope because ContextResources +# is recompiled as part of the capi_context_stub. Including context_resources +# here like this prevents many headaches revolving around duplicate symbols +# when linking executables. +target_link_libraries(compile_array_schema PRIVATE context_resources generic_tile_io) + add_test_subdirectory() diff --git a/tiledb/sm/array_schema/array_schema.cc b/tiledb/sm/array_schema/array_schema.cc index f9c8aa8d5010..ff299b722198 100644 --- a/tiledb/sm/array_schema/array_schema.cc +++ b/tiledb/sm/array_schema/array_schema.cc @@ -39,6 +39,7 @@ #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/array_schema/dimension_label.h" #include "tiledb/sm/array_schema/domain.h" +#include "tiledb/sm/array_schema/enumeration.h" #include "tiledb/sm/buffer/buffer.h" #include "tiledb/sm/enums/array_type.h" #include "tiledb/sm/enums/compressor.h" @@ -49,7 +50,10 @@ #include "tiledb/sm/filter/compression_filter.h" #include "tiledb/sm/filter/webp_filter.h" #include "tiledb/sm/misc/hilbert.h" +#include "tiledb/sm/misc/integral_type_casts.h" #include "tiledb/sm/misc/tdb_time.h" +#include "tiledb/sm/tile/generic_tile_io.h" +#include "tiledb/storage_format/uri/generate_uri.h" #include "tiledb/storage_format/uri/parse_uri.h" #include @@ -120,6 +124,45 @@ ArraySchema::ArraySchema( uint64_t capacity, std::vector> attributes, std::vector> dim_label_refs, + std::unordered_map enumeration_path_map, + FilterPipeline cell_var_offsets_filters, + FilterPipeline cell_validity_filters, + FilterPipeline coords_filters) + : ArraySchema( + uri, + version, + timestamp_range, + name, + array_type, + allows_dups, + domain, + cell_order, + tile_order, + capacity, + attributes, + dim_label_refs, + {}, + enumeration_path_map, + cell_var_offsets_filters, + cell_validity_filters, + coords_filters) { +} + +ArraySchema::ArraySchema( + URI uri, + uint32_t version, + std::pair timestamp_range, + std::string name, + ArrayType array_type, + bool allows_dups, + shared_ptr domain, + Layout cell_order, + Layout tile_order, + uint64_t capacity, + std::vector> attributes, + std::vector> dim_label_refs, + std::vector> enumerations, + std::unordered_map enumeration_path_map, FilterPipeline cell_var_offsets_filters, FilterPipeline cell_validity_filters, FilterPipeline coords_filters) @@ -135,6 +178,7 @@ ArraySchema::ArraySchema( , capacity_(capacity) , attributes_(attributes) , dimension_labels_(dim_label_refs) + , enumeration_path_map_(enumeration_path_map) , cell_var_offsets_filters_(cell_var_offsets_filters) , cell_validity_filters_(cell_validity_filters) , coords_filters_(coords_filters) { @@ -160,6 +204,15 @@ ArraySchema::ArraySchema( dimension_label_map_[label->name()] = label.get(); } + for (auto& [enmr_name, enmr_uri] : enumeration_path_map_) { + (void)enmr_uri; + enumeration_map_[enmr_name] = nullptr; + } + + for (const auto& enmr : enumerations) { + enumeration_map_[enmr->name()] = enmr; + } + // Check array schema is valid. st = check_double_delta_compressor(coords_filters_); if (!st.ok()) { @@ -195,9 +248,13 @@ ArraySchema::ArraySchema(const ArraySchema& array_schema) { throw_if_not_ok(set_domain(array_schema.domain_)); - attribute_map_.clear(); - for (auto attr : array_schema.attributes_) - throw_if_not_ok(add_attribute(attr, false)); + enumeration_map_ = array_schema.enumeration_map_; + enumeration_path_map_ = array_schema.enumeration_path_map_; + + for (auto attr : array_schema.attributes_) { + attributes_.emplace_back(attr); + attribute_map_[attr->name()] = attr.get(); + } // Create dimension label map for (const auto& label : array_schema.dimension_labels_) { @@ -646,6 +703,11 @@ void ArraySchema::dump(FILE* out) const { attr->dump(out); } + for (auto& enmr_iter : enumeration_map_) { + fprintf(out, "\n"); + enmr_iter.second->dump(out); + } + for (auto& label : dimension_labels_) { fprintf(out, "\n"); label->dump(out); @@ -768,6 +830,21 @@ void ArraySchema::serialize(Serializer& serializer) const { for (auto& label : dimension_labels_) { label->serialize(serializer, version); } + + // Write Enumeration path map + auto enmr_num = + utils::safe_integral_cast(enumeration_map_.size()); + + serializer.write(enmr_num); + for (auto& [enmr_name, enmr_uri] : enumeration_path_map_) { + auto enmr_name_size = static_cast(enmr_name.size()); + serializer.write(enmr_name_size); + serializer.write(enmr_name.data(), enmr_name_size); + + auto enmr_uri_size = static_cast(enmr_uri.size()); + serializer.write(enmr_uri_size); + serializer.write(enmr_uri.data(), enmr_uri_size); + } } Layout ArraySchema::tile_order() const { @@ -844,6 +921,51 @@ Status ArraySchema::add_attribute( return LOG_STATUS(Status_ArraySchemaError(msg)); } + auto enmr_name = attr->get_enumeration_name(); + if (enmr_name.has_value()) { + // The referenced enumeration must exist when the attribut is added + auto iter = enumeration_map_.find(enmr_name.value()); + if (iter == enumeration_map_.end()) { + std::string msg = + "Cannot add attribute; Attribute refers to an " + "unknown enumeration named '" + + enmr_name.value() + "'."; + return LOG_STATUS(Status_ArraySchemaError(msg)); + } + + // This attribute must have an integral datatype to support Enumerations + if (!datatype_is_integer(attr->type())) { + std::string msg = "Unable to use enumeration with attribute '" + + attr->name() + + "', attribute must have an integral data type, not " + + datatype_str(attr->type()); + return LOG_STATUS(Status_ArraySchemaError(msg)); + } + + // The attribute must have a cell_val_num of 1 + if (attr->cell_val_num() != 1) { + std::string msg = + "Attributes with enumerations must have a cell_val_num of 1."; + return LOG_STATUS(Status_ArraySchemaError(msg)); + } + + auto enmr = get_enumeration(enmr_name.value()); + if (enmr == nullptr) { + throw ArraySchemaException( + "Cannot add attribute referencing enumeration '" + enmr_name.value() + + "' as the enumeration has not been loaded."); + } + + // The +1 here is because of 0 being a valid index into the enumeration. + if (datatype_max_integral_value(attr->type()) <= enmr->elem_count()) { + throw ArraySchemaException( + "Unable to use enumeration '" + enmr_name.value() + + "' for attribute '" + attr->name() + + "' because the attribute's type is not large enough to represent " + "all enumeration values."); + } + } + // Create new attribute and potentially set a default name attributes_.emplace_back(attr); attribute_map_[attr->name()] = attr.get(); @@ -955,6 +1077,150 @@ Status ArraySchema::drop_attribute(const std::string& attr_name) { return Status::Ok(); } +void ArraySchema::add_enumeration(shared_ptr enmr) { + if (enmr == nullptr) { + throw ArraySchemaException( + "Error adding enumeration. Enumeration " + "must not be nullptr."); + } + + if (enumeration_map_.find(enmr->name()) != enumeration_map_.end()) { + throw ArraySchemaException( + "Error adding enumeration. Enumeration with name '" + enmr->name() + + "' already exists in this ArraySchema."); + } + + enumeration_map_[enmr->name()] = enmr; + enumeration_path_map_[enmr->name()] = enmr->path_name(); +} + +void ArraySchema::store_enumeration(shared_ptr enmr) { + if (enmr == nullptr) { + throw ArraySchemaException( + "Error storing enumeration. Enumeration must not be nullptr."); + } + + auto name_iter = enumeration_map_.find(enmr->name()); + if (name_iter == enumeration_map_.end()) { + throw ArraySchemaException( + "Error storing enumeration. Unknown enumeration name '" + enmr->name() + + "'."); + } + + if (name_iter->second != nullptr) { + throw ArraySchemaException( + "Error storing enumeration. Enumeration named '" + enmr->name() + + "' has already been stored."); + } + + auto path_iter = enumeration_path_map_.find(enmr->name()); + if (path_iter == enumeration_path_map_.end()) { + throw ArraySchemaException( + "Error storing enumeration. Missing path name map entry."); + } + + if (path_iter->second != enmr->path_name()) { + throw ArraySchemaException( + "Error storing enumeration. Path name mismatch for enumeration " + "named '" + + enmr->name() + "'."); + } + + name_iter->second = enmr; +} + +bool ArraySchema::has_enumeration(const std::string& enmr_name) const { + return enumeration_map_.find(enmr_name) != enumeration_map_.end(); +} + +std::vector ArraySchema::get_enumeration_names() const { + std::vector enmr_names; + for (auto& entry : enumeration_path_map_) { + enmr_names.emplace_back(entry.first); + } + return enmr_names; +} + +std::vector ArraySchema::get_loaded_enumeration_names() const { + std::vector enmr_names; + for (auto& entry : enumeration_map_) { + if (entry.second != nullptr) { + enmr_names.emplace_back(entry.first); + } + } + return enmr_names; +} + +bool ArraySchema::is_enumeration_loaded( + const std::string& enumeration_name) const { + auto iter = enumeration_map_.find(enumeration_name); + + if (iter == enumeration_map_.end()) { + throw ArraySchemaException( + "Unable to check if unknown enumeration is loaded. No enumeration " + "named '" + + enumeration_name + "'."); + } + + return iter->second != nullptr; +} + +shared_ptr ArraySchema::get_enumeration( + const std::string& enmr_name) const { + auto iter = enumeration_map_.find(enmr_name); + if (iter == enumeration_map_.end()) { + throw ArraySchemaException( + "Unable to get enumeration. Unknown enumeration named '" + enmr_name + + "'."); + } + + return iter->second; +} + +const std::string& ArraySchema::get_enumeration_path_name( + const std::string& enmr_name) const { + auto iter = enumeration_path_map_.find(enmr_name); + if (iter == enumeration_path_map_.end()) { + throw ArraySchemaException( + "Unable to get enumeration path name. Unknown enumeration named '" + + enmr_name + "'."); + } + + return iter->second; +} + +void ArraySchema::drop_enumeration(const std::string& enmr_name) { + std::lock_guard lock(mtx_); + + if (enmr_name.empty()) { + throw ArraySchemaException( + "Error dropping enumeration, empty names are invalid."); + } + + auto it = enumeration_map_.find(enmr_name); + if (it == enumeration_map_.end()) { + throw ArraySchemaException( + "Error dropping enumeration, no enumeration named '" + enmr_name + + "'."); + } + + for (auto attr : attributes_) { + auto attr_enmr_name = attr->get_enumeration_name(); + if (!attr_enmr_name.has_value()) { + continue; + } + if (attr_enmr_name.value() == enmr_name) { + throw ArraySchemaException( + "Unable to drop enumeration '" + enmr_name + "' as it is used by " + + " attribute '" + attr->name() + "'."); + } + } + + // Drop from both the path and name maps. + enumeration_path_map_.erase(it->first); + enumeration_map_.erase(it); +} + // #TODO Add security validation on incoming URI ArraySchema ArraySchema::deserialize( Deserializer& deserializer, const URI& uri) { @@ -1050,6 +1316,23 @@ ArraySchema ArraySchema::deserialize( } } + // Load enumeration name to path map + std::unordered_map enumeration_path_map; + if (version >= constants::enumerations_min_format_version) { + uint32_t enmr_num = deserializer.read(); + for (uint32_t i = 0; i < enmr_num; i++) { + auto enmr_name_size = deserializer.read(); + std::string enmr_name( + deserializer.get_ptr(enmr_name_size), enmr_name_size); + + auto enmr_path_size = deserializer.read(); + std::string enmr_path_name( + deserializer.get_ptr(enmr_path_size), enmr_path_size); + + enumeration_path_map[enmr_name] = enmr_path_name; + } + } + // Validate if (cell_order == Layout::HILBERT && domain->dim_num() > Hilbert::HC_MAX_DIM) { @@ -1092,6 +1375,7 @@ ArraySchema ArraySchema::deserialize( capacity, attributes, dimension_labels, + enumeration_path_map, cell_var_filters, cell_validity_filters, coords_filters); @@ -1291,6 +1575,7 @@ const std::string& ArraySchema::name() const { /* ****************************** */ /* PRIVATE METHODS */ /* ****************************** */ + void ArraySchema::check_attribute_dimension_label_names() const { std::set names; // Check attribute and dimension names are unique. diff --git a/tiledb/sm/array_schema/array_schema.h b/tiledb/sm/array_schema/array_schema.h index d4a8bc64e15a..f9a15720bfa1 100644 --- a/tiledb/sm/array_schema/array_schema.h +++ b/tiledb/sm/array_schema/array_schema.h @@ -43,6 +43,7 @@ #include "tiledb/sm/misc/constants.h" #include "tiledb/sm/misc/hilbert.h" #include "tiledb/sm/misc/uuid.h" +#include "tiledb/sm/storage_manager/context_resources.h" using namespace tiledb::common; @@ -55,6 +56,7 @@ class ConstBuffer; class Dimension; class DimensionLabel; class Domain; +class Enumeration; enum class ArrayType : uint8_t; enum class Compressor : uint8_t; @@ -108,6 +110,8 @@ class ArraySchema { * @param tile_order The tile order. * @param capacity The tile capacity for the case of sparse fragments. * @param attributes The array attributes. + * @param dimension_labels The array dimension labels. + * @param enumeration_path_map The array enumeration path map * @param cell_var_offsets_filters * The filter pipeline run on offset tiles for var-length attributes. * @param cell_validity_filters @@ -127,6 +131,47 @@ class ArraySchema { uint64_t capacity, std::vector> attributes, std::vector> dimension_labels, + std::unordered_map enumeration_path_map, + FilterPipeline cell_var_offsets_filters, + FilterPipeline cell_validity_filters, + FilterPipeline coords_filters); + + /** Constructor. + * @param uri The URI of the array schema file. + * @param version The format version of this array schema. + * @param timestamp_range The timestamp the array schema was written. + * @param name The file name of the schema in timestamp_timestamp_uuid format. + * @param array_type The array type. + * @param allows_dups True if the (sparse) array allows coordinate duplicates. + * @param domain The array domain. + * @param cell_order The cell order. + * @param tile_order The tile order. + * @param capacity The tile capacity for the case of sparse fragments. + * @param attributes The array attributes. + * @param dimension_labels The array dimension labels. + * @param enumerations The array enumerations + * @param enumeration_path_map The array enumeration path map + * @param cell_var_offsets_filters + * The filter pipeline run on offset tiles for var-length attributes. + * @param cell_validity_filters + * The filter pipeline run on validity tiles for nullable attributes. + * @param coords_filters The filter pipeline run on coordinate tiles. + **/ + ArraySchema( + URI uri, + uint32_t version, + std::pair timestamp_range, + std::string name, + ArrayType array_type, + bool allows_dups, + shared_ptr domain, + Layout cell_order, + Layout tile_order, + uint64_t capacity, + std::vector> attributes, + std::vector> dimension_labels, + std::vector> enumerations, + std::unordered_map enumeration_path_map, FilterPipeline cell_var_offsets_filters, FilterPipeline cell_validity_filters, FilterPipeline coords_filters); @@ -363,6 +408,78 @@ class ArraySchema { */ Status drop_attribute(const std::string& attr_name); + /** + * Add an Enumeration to this ArraySchema. + * + * @param enmr The enumeration to add. + */ + void add_enumeration(shared_ptr enmr); + + /** + * Check if an enumeration exists with the given name. + * + * @param enmr_name The name to check + * @return bool Whether the enumeration exists. + */ + bool has_enumeration(const std::string& enmr_name) const; + + /** + * Store a known enumeration on this ArraySchema after the schema + * was loaded. This allows for only incuring the cost of loading an + * enumeration when it is needed. An exception is thrown if the + * Enumeration is unknown to this ArraySchema. + * + * @param enmr The Enumeration to store. + */ + void store_enumeration(shared_ptr enmr); + + /** + * Get a vector of Enumeration names. + * + * @return A vector of enumeration names. + */ + std::vector get_enumeration_names() const; + + /** + * Get a vector of loaded Enumeration names. + * + * @return A vector of loaded enumeration names. + */ + std::vector get_loaded_enumeration_names() const; + + /** + * Check if a given enumeration has already been loaded. + * + * @param enumeration_name The name of the enumeration to check + * @return bool Whether the enumeration has been loaded or not. + */ + bool is_enumeration_loaded(const std::string& enumeration_name) const; + + /** + * Get an Enumeration by name. Throws if the attribute is unknown. + * + * @param enmr_name The name of the Enumeration. + * @return shared_ptr + */ + shared_ptr get_enumeration( + const std::string& enmr_name) const; + + /** + * Get an Enumeration's object name. Throws if the attribute is unknown. + * + * @param enmr_name The name of the Enumeration. + * @return The path name of the enumeration on disk + */ + const std::string& get_enumeration_path_name( + const std::string& enmr_name) const; + + /** + * Drop an enumeration + * + * @param enumeration_name The enumeration to drop. + */ + void drop_enumeration(const std::string& enmr_name); + /** * It assigns values to the members of the object from the input buffer. * @@ -546,6 +663,13 @@ class ArraySchema { /** A map from the dimension label names to the label schemas. */ std::unordered_map dimension_label_map_; + /** A map of Enumeration names to Enumeration pointers. */ + std::unordered_map> + enumeration_map_; + + /** A map of Enumeration names to Enumeration URIs */ + std::unordered_map enumeration_path_map_; + /** The filter pipeline run on offset tiles for var-length attributes. */ FilterPipeline cell_var_offsets_filters_; @@ -591,6 +715,9 @@ class ArraySchema { void check_webp_filter() const; + // Check whether attributes referencing enumerations are valid. + void check_enumerations() const; + /** Clears all members. Use with caution! */ void clear(); }; diff --git a/tiledb/sm/array_schema/array_schema_evolution.cc b/tiledb/sm/array_schema/array_schema_evolution.cc index bf285733c71e..f2c59846ee6b 100644 --- a/tiledb/sm/array_schema/array_schema_evolution.cc +++ b/tiledb/sm/array_schema/array_schema_evolution.cc @@ -35,10 +35,12 @@ #include "tiledb/common/common.h" #include "tiledb/common/heap_memory.h" #include "tiledb/common/logger.h" +#include "tiledb/common/status.h" #include "tiledb/sm/array_schema/array_schema.h" #include "tiledb/sm/array_schema/attribute.h" #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/array_schema/domain.h" +#include "tiledb/sm/array_schema/enumeration.h" #include "tiledb/sm/buffer/buffer.h" #include "tiledb/sm/enums/array_type.h" #include "tiledb/sm/enums/compressor.h" @@ -58,6 +60,14 @@ using namespace tiledb::common; namespace tiledb { namespace sm { +/** Class for locally generated exceptions. */ +class ArraySchemaEvolutionException : public StatusException { + public: + explicit ArraySchemaEvolutionException(const std::string& msg) + : StatusException("ArraySchemaEvolution", msg) { + } +}; + /* ****************************** */ /* CONSTRUCTORS & DESTRUCTORS */ /* ****************************** */ @@ -65,6 +75,19 @@ namespace sm { ArraySchemaEvolution::ArraySchemaEvolution() { } +ArraySchemaEvolution::ArraySchemaEvolution( + std::unordered_map> attrs_to_add, + std::unordered_set attrs_to_drop, + std::unordered_map> enmrs_to_add, + std::unordered_set enmrs_to_drop, + std::pair timestamp_range) + : attributes_to_add_map_(attrs_to_add) + , attributes_to_drop_(attrs_to_drop) + , enumerations_to_add_map_(enmrs_to_add) + , enumerations_to_drop_(enmrs_to_drop) + , timestamp_range_(timestamp_range) { +} + ArraySchemaEvolution::~ArraySchemaEvolution() { clear(); } @@ -73,59 +96,66 @@ ArraySchemaEvolution::~ArraySchemaEvolution() { /* API */ /* ****************************** */ -tuple>> -ArraySchemaEvolution::evolve_schema( +shared_ptr ArraySchemaEvolution::evolve_schema( const shared_ptr& orig_schema) { std::lock_guard lock(mtx_); if (orig_schema == nullptr) { - return { - LOG_STATUS(Status_ArraySchemaEvolutionError( - "Cannot evolve schema; Input array schema is null")), - nullopt}; + throw ArraySchemaEvolutionException( + "Cannot evolve schema; Input array schema is null"); } auto schema = make_shared(HERE(), *(orig_schema.get())); + // Add enumerations. Must be done before attributes so that any attributes + // referencing enumerations won't fail to be added. + for (auto& enmr : enumerations_to_add_map_) { + schema->add_enumeration(enmr.second); + } + // Add attributes. for (auto& attr : attributes_to_add_map_) { - RETURN_NOT_OK_TUPLE(schema->add_attribute(attr.second, false), nullopt); + throw_if_not_ok(schema->add_attribute(attr.second, false)); } // Drop attributes. for (auto& attr_name : attributes_to_drop_) { bool has_attr = false; - RETURN_NOT_OK_TUPLE(schema->has_attribute(attr_name, &has_attr), nullopt); + throw_if_not_ok(schema->has_attribute(attr_name, &has_attr)); if (has_attr) { - RETURN_NOT_OK_TUPLE(schema->drop_attribute(attr_name), nullopt); + throw_if_not_ok(schema->drop_attribute(attr_name)); } } - // Do other things + // Drop enumerations + for (auto& enmr_name : enumerations_to_drop_) { + if (schema->has_enumeration(enmr_name)) { + schema->drop_enumeration(enmr_name); + } + } // Set timestamp, if specified if (std::get<0>(timestamp_range_) != 0) { - RETURN_NOT_OK_TUPLE( - schema.get()->set_timestamp_range(timestamp_range_), nullopt); - RETURN_NOT_OK_TUPLE(schema->generate_uri(timestamp_range_), nullopt); + throw_if_not_ok(schema.get()->set_timestamp_range(timestamp_range_)); + throw_if_not_ok(schema->generate_uri(timestamp_range_)); } else { // Generate new schema URI - RETURN_NOT_OK_TUPLE(schema->generate_uri(), nullopt); + throw_if_not_ok(schema->generate_uri()); } - return {Status::Ok(), schema}; + return schema; } -Status ArraySchemaEvolution::add_attribute(const Attribute* attr) { +void ArraySchemaEvolution::add_attribute(const Attribute* attr) { std::lock_guard lock(mtx_); // Sanity check if (attr == nullptr) - return LOG_STATUS(Status_ArraySchemaEvolutionError( - "Cannot add attribute; Input attribute is null")); + throw ArraySchemaEvolutionException( + "Cannot add attribute; Input attribute is null"); if (attributes_to_add_map_.find(attr->name()) != attributes_to_add_map_.end()) { - return LOG_STATUS(Status_ArraySchemaEvolutionError( - "Cannot add attribute; Input attribute name is already there")); + throw ArraySchemaEvolutionException( + "Cannot add attribute; Input attribute name is already there"); } // Create new attribute and potentially set a default name @@ -134,18 +164,14 @@ Status ArraySchemaEvolution::add_attribute(const Attribute* attr) { if (attributes_to_drop_.find(attr->name()) != attributes_to_drop_.end()) { attributes_to_drop_.erase(attr->name()); } - - return Status::Ok(); } std::vector ArraySchemaEvolution::attribute_names_to_add() const { std::lock_guard lock(mtx_); std::vector names; names.reserve(attributes_to_add_map_.size()); - for (auto it = attributes_to_add_map_.begin(); - it != attributes_to_add_map_.end(); - ++it) { - names.push_back(it->first); + for (auto& entry : attributes_to_add_map_) { + names.emplace_back(entry.first); } return names; } @@ -157,29 +183,91 @@ const Attribute* ArraySchemaEvolution::attribute_to_add( return (it == attributes_to_add_map_.end()) ? nullptr : it->second.get(); } -Status ArraySchemaEvolution::drop_attribute(const std::string& attribute_name) { +void ArraySchemaEvolution::drop_attribute(const std::string& attribute_name) { std::lock_guard lock(mtx_); attributes_to_drop_.insert(attribute_name); - if (attributes_to_add_map_.find(attribute_name) != - attributes_to_add_map_.end()) { + auto ait = attributes_to_add_map_.find(attribute_name); + if (ait != attributes_to_add_map_.end()) { // Reset the pointer and erase it - attributes_to_add_map_.erase(attribute_name); + attributes_to_add_map_.erase(ait); } - return Status::Ok(); } std::vector ArraySchemaEvolution::attribute_names_to_drop() const { std::lock_guard lock(mtx_); std::vector names; names.reserve(attributes_to_drop_.size()); - for (auto it = attributes_to_drop_.begin(); it != attributes_to_drop_.end(); - ++it) { - names.push_back(*it); + for (auto& name : attributes_to_drop_) { + names.emplace_back(name); + } + return names; +} + +void ArraySchemaEvolution::add_enumeration(shared_ptr enmr) { + std::lock_guard lock(mtx_); + + if (enmr == nullptr) { + throw ArraySchemaEvolutionException( + "Cannot add enumeration; Input enumeration is null"); + } + + auto it = enumerations_to_add_map_.find(enmr->name()); + if (it != enumerations_to_add_map_.end()) { + throw ArraySchemaEvolutionException( + "Cannot add enumeration; Input enumeration name is already added"); + } + + enumerations_to_add_map_[enmr->name()] = enmr; +} + +std::vector ArraySchemaEvolution::enumeration_names_to_add() + const { + std::lock_guard lock(mtx_); + std::vector names; + names.reserve(enumerations_to_add_map_.size()); + for (auto elem : enumerations_to_add_map_) { + names.push_back(elem.first); + } + + return names; +} + +shared_ptr ArraySchemaEvolution::enumeration_to_add( + const std::string& name) const { + std::lock_guard lock(mtx_); + auto it = enumerations_to_add_map_.find(name); + + if (it == enumerations_to_add_map_.end()) { + return nullptr; + } + + return it->second; +} + +void ArraySchemaEvolution::drop_enumeration( + const std::string& enumeration_name) { + std::lock_guard lock(mtx_); + enumerations_to_drop_.insert(enumeration_name); + + auto it = enumerations_to_add_map_.find(enumeration_name); + if (it != enumerations_to_add_map_.end()) { + // Reset the pointer and erase it + enumerations_to_add_map_.erase(it); + } +} + +std::vector ArraySchemaEvolution::enumeration_names_to_drop() + const { + std::lock_guard lock(mtx_); + std::vector names; + names.reserve(enumerations_to_drop_.size()); + for (auto& name : enumerations_to_drop_) { + names.emplace_back(name); } return names; } -Status ArraySchemaEvolution::set_timestamp_range( +void ArraySchemaEvolution::set_timestamp_range( const std::pair& timestamp_range) { if (timestamp_range.first != timestamp_range.second) { throw std::runtime_error(std::string( @@ -188,7 +276,6 @@ Status ArraySchemaEvolution::set_timestamp_range( std::to_string(timestamp_range.second) + " are not equal!")); } timestamp_range_ = timestamp_range; - return Status::Ok(); } std::pair ArraySchemaEvolution::timestamp_range() const { @@ -203,6 +290,8 @@ std::pair ArraySchemaEvolution::timestamp_range() const { void ArraySchemaEvolution::clear() { attributes_to_add_map_.clear(); attributes_to_drop_.clear(); + enumerations_to_add_map_.clear(); + enumerations_to_drop_.clear(); timestamp_range_ = {0, 0}; } diff --git a/tiledb/sm/array_schema/array_schema_evolution.h b/tiledb/sm/array_schema/array_schema_evolution.h index 6ba64b754519..21af603e4d59 100644 --- a/tiledb/sm/array_schema/array_schema_evolution.h +++ b/tiledb/sm/array_schema/array_schema_evolution.h @@ -38,7 +38,6 @@ #include #include "tiledb/common/common.h" -#include "tiledb/common/status.h" #include "tiledb/sm/filesystem/uri.h" #include "tiledb/sm/filter/filter_pipeline.h" #include "tiledb/sm/misc/constants.h" @@ -55,6 +54,7 @@ class Buffer; class ConstBuffer; class Dimension; class Domain; +class Enumeration; class ArraySchema; enum class ArrayType : uint8_t; @@ -72,6 +72,20 @@ class ArraySchemaEvolution { /** Constructor. */ ArraySchemaEvolution(); + /** Constructor. + * @param attrs_to_add Attributes to add to the schema. + * @param enmrs_to_add Enumerations to add to the schema. + * @param attrs_to_drop Attributes to remove from the schema. + * @param timestamp_range Timestamp range to use for the new schema. + */ + ArraySchemaEvolution( + std::unordered_map> attrs_to_add, + std::unordered_set attrs_to_drop, + std::unordered_map> + enmrs_to_add, + std::unordered_set enmrs_to_drop, + std::pair timestamp_range); + /** Destructor. */ ~ArraySchemaEvolution(); @@ -79,16 +93,15 @@ class ArraySchemaEvolution { /* API */ /* ********************************* */ - tuple>> evolve_schema( + shared_ptr evolve_schema( const shared_ptr& orig_schema); /** * Adds an attribute, copying the input. * * @param attr The attribute to be added - * @return Status */ - Status add_attribute(const Attribute* attr); + void add_attribute(const Attribute* attr); /** Returns the names of attributes to add. */ std::vector attribute_names_to_add() const; @@ -102,16 +115,45 @@ class ArraySchemaEvolution { /** * Drops an attribute. * - * @param attr The attribute to be dropped - * @return Status + * @param attribute_name The attribute to be dropped. */ - Status drop_attribute(const std::string& attribute_name); + void drop_attribute(const std::string& attribute_name); /** Returns the names of attributes to drop. */ std::vector attribute_names_to_drop() const; + /** + * Adds an enumeration + * + * @param enmr The enumeration to add + */ + void add_enumeration(shared_ptr enmr); + + /** Returns the names of the enumerations to add. */ + std::vector enumeration_names_to_add() const; + + /** + * Returns a constant pointer to the selected enumeration or nullptr if + * it does not exist. + * + * @param name The name of the enumeration to add + * @return shared_ptr The enumeration to add. + */ + shared_ptr enumeration_to_add( + const std::string& name) const; + + /** + * Drops an enumeration + * + * @param enumeration_name The enumeration to be dropped. + */ + void drop_enumeration(const std::string& enumeration_name); + + /** Return the names of enumerations to drop. */ + std::vector enumeration_names_to_drop() const; + /** Set a timestamp range for the array schema evolution */ - Status set_timestamp_range( + void set_timestamp_range( const std::pair& timestamp_range); /** Returns the timestamp range. */ @@ -129,6 +171,13 @@ class ArraySchemaEvolution { /** The names of array attributes to be dropped. */ std::unordered_set attributes_to_drop_; + /** Enumerations to add with any attribute. */ + std::unordered_map> + enumerations_to_add_map_; + + /** The names of array enumerations to be dropped. */ + std::unordered_set enumerations_to_drop_; + /** * A timestamp to explicitly set the timestamp of * the evolved schema. To be consistent with @@ -151,4 +200,4 @@ class ArraySchemaEvolution { } // namespace sm } // namespace tiledb -#endif // TILEDB_SCHEMA_EVOLUTION_H \ No newline at end of file +#endif // TILEDB_SCHEMA_EVOLUTION_H diff --git a/tiledb/sm/array_schema/attribute.cc b/tiledb/sm/array_schema/attribute.cc index 8c6132a5c925..bdf23e01c44c 100644 --- a/tiledb/sm/array_schema/attribute.cc +++ b/tiledb/sm/array_schema/attribute.cc @@ -38,6 +38,7 @@ #include "tiledb/sm/enums/filter_type.h" #include "tiledb/sm/filter/compression_filter.h" #include "tiledb/sm/misc/parse_argument.h" +#include "tiledb/sm/misc/uuid.h" #include "tiledb/type/range/range.h" #include @@ -115,7 +116,8 @@ Attribute::Attribute( const FilterPipeline& filter_pipeline, const ByteVecValue& fill_value, uint8_t fill_value_validity, - DataOrder order) + DataOrder order, + std::optional enumeration_name) : cell_val_num_(cell_val_num) , nullable_(nullable) , filters_(filter_pipeline) @@ -123,7 +125,8 @@ Attribute::Attribute( , type_(type) , fill_value_(fill_value) , fill_value_validity_(fill_value_validity) - , order_(order) { + , order_(order) + , enumeration_name_(enumeration_name) { } Attribute::Attribute(const Attribute* attr) { @@ -136,6 +139,7 @@ Attribute::Attribute(const Attribute* attr) { fill_value_ = attr->fill_value_; fill_value_validity_ = attr->fill_value_validity_; order_ = attr->order_; + enumeration_name_ = attr->enumeration_name_; } Attribute::~Attribute() = default; @@ -204,6 +208,17 @@ Attribute Attribute::deserialize( order = data_order_from_int(deserializer.read()); } + std::optional enmr_name; + if (version >= 19) { + auto enmr_name_length = deserializer.read(); + if (enmr_name_length > 0) { + std::string enmr_name_value; + enmr_name_value.resize(enmr_name_length); + deserializer.read(enmr_name_value.data(), enmr_name_length); + enmr_name = enmr_name_value; + } + } + return Attribute( name, datatype, @@ -212,7 +227,8 @@ Attribute Attribute::deserialize( filterpipeline, fill_value, fill_value_validity, - order); + order, + enmr_name); } void Attribute::dump(FILE* out) const { @@ -299,6 +315,18 @@ void Attribute::serialize( if (version >= 17) { serializer.write(static_cast(order_)); } + + // Write enumeration URI + if (version >= constants::enumerations_min_format_version) { + if (enumeration_name_.has_value()) { + auto enmr_name_size = + static_cast(enumeration_name_.value().size()); + serializer.write(enmr_name_size); + serializer.write(enumeration_name_.value().data(), enmr_name_size); + } else { + serializer.write(0); + } + } } void Attribute::set_cell_val_num(unsigned int cell_val_num) { @@ -480,6 +508,18 @@ DataOrder Attribute::order() const { return order_; } +void Attribute::set_enumeration_name(std::optional enmr_name) { + if (enmr_name.has_value() && enmr_name.value().empty()) { + throw AttributeStatusException( + "Invalid enumeration name, name must not be empty."); + } + enumeration_name_ = enmr_name; +} + +std::optional Attribute::get_enumeration_name() const { + return enumeration_name_; +} + /* ********************************* */ /* PRIVATE METHODS */ /* ********************************* */ diff --git a/tiledb/sm/array_schema/attribute.h b/tiledb/sm/array_schema/attribute.h index 7e3e7a4dd302..368836646bc4 100644 --- a/tiledb/sm/array_schema/attribute.h +++ b/tiledb/sm/array_schema/attribute.h @@ -51,6 +51,7 @@ namespace sm { class Buffer; class ConstBuffer; +class Enumeration; enum class Compressor : uint8_t; enum class Datatype : uint8_t; @@ -110,7 +111,8 @@ class Attribute { const FilterPipeline& filter_pipeline, const ByteVecValue& fill_value, uint8_t fill_value_validity, - DataOrder order = DataOrder::UNORDERED_DATA); + DataOrder order = DataOrder::UNORDERED_DATA, + std::optional enumeration_name = nullopt); /** * Constructor. It clones the input attribute. @@ -242,6 +244,12 @@ class Attribute { static ByteVecValue default_fill_value( Datatype datatype, uint32_t cell_val_num); + /** Set an enumeration for this attribute. */ + void set_enumeration_name(std::optional enmr_name); + + /** Get the enumeration for this attribute. */ + std::optional get_enumeration_name() const; + private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ @@ -271,8 +279,11 @@ class Attribute { /** The required order of the data stored in the attribute. */ DataOrder order_; + /** The name of the Enumeration to use for this attribute. */ + std::optional enumeration_name_; + /* ********************************* */ - /* PRIVATE ATTRIBUTES */ + /* PRIVATE METHODS */ /* ********************************* */ /** Sets the default fill value. */ diff --git a/tiledb/sm/array_schema/enumeration.cc b/tiledb/sm/array_schema/enumeration.cc new file mode 100644 index 000000000000..825f6e9ad2e4 --- /dev/null +++ b/tiledb/sm/array_schema/enumeration.cc @@ -0,0 +1,260 @@ +/** + * @file enumeration.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file implements class Enumeration. + */ + +#include +#include + +#include "tiledb/sm/misc/uuid.h" + +#include "enumeration.h" + +namespace tiledb::sm { + +/** Class for locally generated status exceptions. */ +class EnumerationException : public StatusException { + public: + explicit EnumerationException(const std::string& msg) + : StatusException("Enumeration", msg) { + } +}; + +Enumeration::Enumeration( + const std::string& name, + const std::string& path_name, + Datatype type, + uint32_t cell_val_num, + bool ordered, + const void* data, + uint64_t data_size, + const void* offsets, + uint64_t offsets_size) + : name_(name) + , path_name_(path_name) + , type_(type) + , cell_val_num_(cell_val_num) + , ordered_(ordered) + , data_(data_size) + , offsets_(offsets_size) { + ensure_datatype_is_valid(type); + + if (name.empty()) { + throw EnumerationException("Enumeration name must not be empty"); + } + + if (path_name_.empty()) { + std::string tmp_uuid; + throw_if_not_ok(uuid::generate_uuid(&tmp_uuid, false)); + path_name_ = + "__" + tmp_uuid + "_" + std::to_string(constants::enumerations_version); + } + + if (name.find("/") != std::string::npos) { + throw EnumerationException( + "Enumeration name must not contain path separators"); + } + + if (cell_val_num == 0) { + throw EnumerationException("Invalid cell_val_num in Enumeration"); + } + + if (data == nullptr || data_size == 0) { + throw EnumerationException("No attribute value data supplied."); + } + + if (var_size() && (offsets == nullptr || offsets_size == 0)) { + throw EnumerationException( + "Variable length datatype defined but offsets are not present"); + } else if (!var_size() && (offsets != nullptr || offsets_size > 0)) { + throw EnumerationException( + "Fixed length datatype defined but offsets are present"); + } + + if (var_size()) { + if (offsets_size % sizeof(uint64_t) != 0) { + throw EnumerationException( + "Invalid offsets size is not a multiple of sizeof(uint64_t)"); + } + auto offset_values = static_cast(offsets); + uint64_t last_offset = (offsets_size / sizeof(uint64_t)) - 1; + if (offset_values[last_offset] > data_size) { + throw EnumerationException( + "Provided data buffer size is too small for the provided offsets."); + } + } else { + if (data_size % cell_size() != 0) { + throw EnumerationException( + "Invalid data size is not a multiple of the cell size."); + } + } + + throw_if_not_ok(data_.write(data, 0, data_size)); + + if (offsets_size > 0) { + throw_if_not_ok(offsets_.write(offsets, 0, offsets_size)); + } + + generate_value_map(); +} + +shared_ptr Enumeration::deserialize( + Deserializer& deserializer) { + auto disk_version = deserializer.read(); + if (disk_version > constants::enumerations_version) { + throw EnumerationException( + "Invalid Enumeration version '" + std::to_string(disk_version) + + "' is newer than supported enumeration version '" + + std::to_string(constants::enumerations_version) + "'"); + } + + auto name_size = deserializer.read(); + std::string name(deserializer.get_ptr(name_size), name_size); + + auto path_name_size = deserializer.read(); + std::string path_name( + deserializer.get_ptr(path_name_size), path_name_size); + + auto type = deserializer.read(); + auto cell_val_num = deserializer.read(); + auto ordered = deserializer.read(); + + auto data_size = deserializer.read(); + const void* data = deserializer.get_ptr(data_size); + + uint64_t offsets_size = 0; + const void* offsets = nullptr; + + if (cell_val_num == constants::var_num) { + offsets_size = deserializer.read(); + offsets = deserializer.get_ptr(offsets_size); + } + + return create( + name, + path_name, + static_cast(type), + cell_val_num, + ordered, + data, + data_size, + offsets, + offsets_size); +} + +void Enumeration::serialize(Serializer& serializer) const { + serializer.write(constants::enumerations_version); + + auto name_size = static_cast(name_.size()); + serializer.write(name_size); + serializer.write(name_.data(), name_size); + + auto path_name_size = static_cast(path_name_.size()); + serializer.write(path_name_size); + serializer.write(path_name_.data(), path_name_size); + + serializer.write(static_cast(type_)); + serializer.write(cell_val_num_); + serializer.write(ordered_); + serializer.write(data_.size()); + serializer.write(data_.data(), data_.size()); + + if (var_size()) { + serializer.write(offsets_.size()); + serializer.write(offsets_.data(), offsets_.size()); + } else { + assert(cell_val_num_ < constants::var_num); + assert(offsets_.size() == 0); + } +} + +uint64_t Enumeration::index_of(UntypedDatumView value) const { + std::string_view value_view( + static_cast(value.content()), value.size()); + + auto iter = value_map_.find(value_view); + if (iter == value_map_.end()) { + return constants::enumeration_missing_value; + } + + return iter->second; +} + +void Enumeration::dump(FILE* out) const { + if (out == nullptr) { + out = stdout; + } + std::stringstream ss; + ss << "### Enumeration ###" << std::endl; + ss << "- Name: " << name_ << std::endl; + ss << "- Type: " << datatype_str(type_) << std::endl; + ss << "- Cell Val Num: " << cell_val_num_ << std::endl; + ss << "- Ordered: " << (ordered_ ? "true" : "false") << std::endl; + ss << "- Element Count: " << value_map_.size() << std::endl; + fprintf(out, "%s", ss.str().c_str()); +} + +void Enumeration::generate_value_map() { + auto char_data = data_.data_as(); + if (var_size()) { + auto offsets = offsets_.data_as(); + uint64_t num_offsets = offsets_.size() / sizeof(uint64_t); + + for (uint64_t i = 0; i < num_offsets; i++) { + uint64_t length = 0; + if (i < num_offsets - 1) { + length = offsets[i + 1] - offsets[i]; + } else { + length = data_.size() - offsets[i]; + } + + auto sv = std::string_view(char_data + offsets[i], length); + add_value_to_map(sv, i); + } + } else { + uint64_t i = 0; + auto stride = cell_size(); + while (i * stride < data_.size()) { + auto sv = std::string_view(char_data + i * stride, stride); + add_value_to_map(sv, i); + i += 1; + } + } +} + +void Enumeration::add_value_to_map(std::string_view& sv, uint64_t index) { + if (value_map_.find(sv) != value_map_.end()) { + throw EnumerationException( + "Invalid duplicated value in enumeration '" + std::string(sv) + "'"); + } + value_map_[sv] = index; +} + +} // namespace tiledb::sm diff --git a/tiledb/sm/array_schema/enumeration.h b/tiledb/sm/array_schema/enumeration.h new file mode 100644 index 000000000000..aac6ea2dd910 --- /dev/null +++ b/tiledb/sm/array_schema/enumeration.h @@ -0,0 +1,388 @@ +/** + * @file enumeration.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file defines class Enumeration. + */ + +#ifndef TILEDB_ENUMERATION_H +#define TILEDB_ENUMERATION_H + +#include + +#include "tiledb/common/common.h" +#include "tiledb/common/types/untyped_datum.h" +#include "tiledb/sm/buffer/buffer.h" +#include "tiledb/sm/enums/datatype.h" +#include "tiledb/storage_format/serialization/serializers.h" + +namespace tiledb::sm { + +/** Defines an array enumeration */ +class Enumeration { + public: + /* ********************************* */ + /* CONSTRUCTORS & DESTRUCTORS */ + /* ********************************* */ + + /** No default constructor. Use the create factory method. */ + Enumeration() = delete; + + DISABLE_COPY(Enumeration); + DISABLE_MOVE(Enumeration); + + /** Destructor. */ + ~Enumeration() = default; + + /* ********************************* */ + /* OPERATORS */ + /* ********************************* */ + + DISABLE_COPY_ASSIGN(Enumeration); + DISABLE_MOVE_ASSIGN(Enumeration); + + /* ********************************* */ + /* API */ + /* ********************************* */ + + /** Create a new Enumeration + * + * @param name The name of this Enumeration as referenced by attributes. + * @param type The datatype of the enumeration values. + * @param cell_val_num The cell_val_num of the enumeration. + * @param ordered Whether the enumeration should be considered as ordered. + * If false, prevents inequality operators in QueryConditons from + * being used with this enumeration. + * @param data A pointer to the enumerations values. + * @param data_size The length of the buffer pointed to by data. + * @param offsets If cell_var_num is constants::var_num a pointer to the + * offsets buffer. Must be null if cell_var_num is not var_num. + * @param offsets_size The size of the buffer pointed to by offsets. Must be + * zero of cell_var_num is not var_num. + * @return shared_ptr The created enumeration. + */ + static shared_ptr create( + const std::string& name, + Datatype type, + uint32_t cell_val_num, + bool ordered, + const void* data, + uint64_t data_size, + const void* offsets, + uint64_t offsets_size) { + return create( + name, + "", + type, + cell_val_num, + ordered, + data, + data_size, + offsets, + offsets_size); + } + + /** Create a new Enumeration + * + * @param name The name of this Enumeration as referenced by attributes. + * @param path_name The last URI path component of the Enumeration. + * @param type The datatype of the enumeration values. + * @param cell_val_num The cell_val_num of the enumeration. + * @param ordered Whether the enumeration should be considered as ordered. + * If false, prevents inequality operators in QueryConditons from + * being used with this enumeration. + * @param data A pointer to the enumerations values. + * @param data_size The length of the buffer pointed to by data. + * @param offsets If cell_var_num is constants::var_num a pointer to the + * offsets buffer. Must be null if cell_var_num is not var_num. + * @param offsets_size The size of the buffer pointed to by offsets. Must be + * zero of cell_var_num is not var_num. + * @return shared_ptr The created enumeration. + */ + static shared_ptr create( + const std::string& name, + const std::string& path_name, + Datatype type, + uint32_t cell_val_num, + bool ordered, + const void* data, + uint64_t data_size, + const void* offsets, + uint64_t offsets_size) { + struct EnableMakeShared : public Enumeration { + EnableMakeShared( + const std::string& name, + const std::string& path_name, + Datatype type, + uint32_t cell_val_num, + bool ordered, + const void* data, + uint64_t data_size, + const void* offsets, + uint64_t offsets_size) + : Enumeration( + name, + path_name, + type, + cell_val_num, + ordered, + data, + data_size, + offsets, + offsets_size) { + } + }; + return make_shared( + HERE(), + name, + path_name, + type, + cell_val_num, + ordered, + data, + data_size, + offsets, + offsets_size); + } + + /** + * Deserialize an enumeration + * + * @param deserializer The deserializer to deserialize from. + * @return A new Enumeration. + */ + static shared_ptr deserialize(Deserializer& deserializer); + + /** + * Serializes the enumeration into a buffer. + * + * @param serializer The object the array schema is serialized into. + */ + void serialize(Serializer& serializer) const; + + /** + * The name of this Enumeration referenced by attributes. + * + * @return The name of this Enumeration. + */ + const std::string& name() const { + return name_; + } + + /** + * The path name for this Enumeration on disk. + * + * @return The path name of this Enumeration. + */ + const std::string& path_name() const { + return path_name_; + } + + /** + * The type of the enumeration values + * + * @return Datatype The datatype of the enumeration values. + */ + Datatype type() const { + return type_; + } + + /** + * The cell_val_num of the enumeration + * + * @return uint32_t The cell_val_num of the enumeration. + */ + uint32_t cell_val_num() const { + return cell_val_num_; + } + + /** + * Get the cell_size of the enumeration + * + * This returns constants::var_size when cell_val_num is var_num, otherwise + * it returns the cell_val_num * datatype_size(type()) + * + * @return uint64_t The cell size of this enumeration. + */ + uint64_t cell_size() const { + if (var_size()) { + return constants::var_size; + } + + return cell_val_num_ * datatype_size(type_); + } + + /** + * Get the number of elements in the Enumeration. + * + * @return uint64_t The number of elements. + */ + uint64_t elem_count() const { + if (var_size()) { + return offsets_.size() / sizeof(uint64_t); + } else { + return data_.size() / cell_size(); + } + } + + /** + * Returns whether this enumeration is variable sized. + * + * @return bool Whether this enumerations is variable sized. + */ + bool var_size() const { + return cell_val_num_ == constants::var_num; + } + + /** + * Returns whether this enumeration is considered ordered or not. + * + * If an enumeration is not ordered then inequality operators in + * QueryConditions against this Enumeration are disabled. + * + * @return bool Whether this enumeration is considered ordered. + */ + bool ordered() const { + return ordered_; + } + + /** + * Returns the data buffer + * + * @return tuple A pointer to the data buffer and the + size of the buffer pointed to. + */ + span data() const { + return {static_cast(data_.data()), data_.size()}; + } + + /** + * Returns the offsets buffer + * + * @return tuple A pointer to the offsets buffer and + * the size of the buffer pointed to. + */ + span offsets() const { + return {static_cast(offsets_.data()), offsets_.size()}; + } + + /** + * Return the index of a value in the enumeration + * + * @param data An UntypedDatumView of the value to look up. + * @return uint64_t The index of the value represented by data or + * constants::missing_enumeration_value if not found. + */ + uint64_t index_of(UntypedDatumView data) const; + + /** + * Dump a textual representation of the Enumeration to the FILE + * + * @param out A file pointer to write to. If out is nullptr, use stdout + */ + void dump(FILE* out) const; + + private: + /* ********************************* */ + /* PRIVATE CONSTRUCTORS */ + /* ********************************* */ + + /** Constructor + * + * @param name The name of this Enumeration as referenced by attributes. + * @param path_name The last URI path component of the Enumeration. + * @param type The datatype of the enumeration values. + * @param cell_val_num The cell_val_num of the enumeration. + * @param ordered Whether the enumeration should be considered as ordered. + * If false, prevents inequality operators in QueryConditons from + * being used with this enumeration. + * @param data A pointer to the enumerations values. + * @param data_size The length of the buffer pointed to by data. + * @param offsets If cell_var_num is constants::var_num a pointer to the + * offsets buffer. Must be null if cell_var_num is not var_num. + * @param offsets_size The size of the buffer pointed to by offsets. Must be + * zero of cell_var_num is not var_num. + */ + Enumeration( + const std::string& name, + const std::string& path_name, + Datatype type, + uint32_t cell_val_num, + bool ordered, + const void* data, + uint64_t data_size, + const void* offsets, + uint64_t offsets_size); + + /* ********************************* */ + /* PRIVATE ATTRIBUTES */ + /* ********************************* */ + + /** The name of this Enumeration stored in the enumerations directory. */ + std::string name_; + + /** The path name of this Enumeration as stored on disk. */ + std::string path_name_; + + /** The type of enumerated values */ + Datatype type_; + + /** Number of values per enumeration value */ + uint32_t cell_val_num_; + + /** A flag which enables or disables inequality comparisons */ + bool ordered_; + + /** The list of enumeration values */ + Buffer data_; + + /** The offsets of each enumeration value if var sized. */ + Buffer offsets_; + + /** Map of values to indices */ + std::unordered_map value_map_; + + /* ********************************* */ + /* PRIVATE METHODS */ + /* ********************************* */ + + /** Populate the value_map_ */ + void generate_value_map(); + + /** + * Add a value to value_map_ + * + * @param sv A string view of the data to add. + * @param index The index of the data in the Enumeration. + */ + void add_value_to_map(std::string_view& sv, uint64_t index); +}; + +} // namespace tiledb::sm + +#endif // TILEDB_DOMAIN_H diff --git a/tiledb/sm/array_schema/test/compile_enumeration_main.cc b/tiledb/sm/array_schema/test/compile_enumeration_main.cc new file mode 100644 index 000000000000..d35fa1cca83c --- /dev/null +++ b/tiledb/sm/array_schema/test/compile_enumeration_main.cc @@ -0,0 +1,39 @@ +/** + * @file compile_enumeration_main.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "tiledb/sm/array_schema/enumeration.h" +#include "tiledb/sm/enums/datatype.h" + +int main(int, char*[]) { + try { + tiledb::sm::Enumeration::create( + "foo", tiledb::sm::Datatype::INT32, 1, false, nullptr, 0, nullptr, 0); + } catch (...) { + } + return 0; +} diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 90b6a9d17725..4642c58408ba 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -40,6 +40,7 @@ #include "tiledb/api/c_api/buffer_list/buffer_list_api_internal.h" #include "tiledb/api/c_api/config/config_api_internal.h" #include "tiledb/api/c_api/dimension/dimension_api_internal.h" +#include "tiledb/api/c_api/enumeration/enumeration_api_internal.h" #include "tiledb/api/c_api/error/error_api_internal.h" #include "tiledb/api/c_api/filter_list/filter_list_api_internal.h" #include "tiledb/api/c_api/string/string_api_internal.h" @@ -299,6 +300,34 @@ int32_t tiledb_attribute_set_cell_val_num( return TILEDB_OK; } +capi_return_t tiledb_attribute_set_enumeration_name( + tiledb_ctx_t* ctx, tiledb_attribute_t* attr, const char* enumeration_name) { + if (sanity_check(ctx, attr) == TILEDB_ERR) { + return TILEDB_ERR; + } + attr->attr_->set_enumeration_name(enumeration_name); + return TILEDB_OK; +} + +capi_return_t tiledb_attribute_get_enumeration_name( + tiledb_ctx_t* ctx, tiledb_attribute_t* attr, tiledb_string_t** name) { + if (sanity_check(ctx, attr) == TILEDB_ERR) { + return TILEDB_ERR; + } + + ensure_output_pointer_is_valid(name); + + auto enmr_name = attr->attr_->get_enumeration_name(); + if (!enmr_name.has_value()) { + *name = nullptr; + return TILEDB_OK; + } + + *name = tiledb_string_handle_t::make_handle(enmr_name.value()); + + return TILEDB_OK; +} + int32_t tiledb_attribute_get_name( tiledb_ctx_t* ctx, const tiledb_attribute_t* attr, const char** name) { if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, attr) == TILEDB_ERR) @@ -725,6 +754,21 @@ int32_t tiledb_array_schema_timestamp_range( return TILEDB_OK; } +TILEDB_EXPORT int32_t tiledb_array_schema_add_enumeration( + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_enumeration_t* enumeration) { + if (sanity_check(ctx, array_schema) == TILEDB_ERR) { + return TILEDB_ERR; + } + + api::ensure_enumeration_is_valid(enumeration); + + array_schema->array_schema_->add_enumeration(enumeration->copy()); + + return TILEDB_OK; +} + int32_t tiledb_array_schema_set_coords_filter_list( tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, @@ -786,9 +830,10 @@ int32_t tiledb_array_schema_check( return TILEDB_OK; } -int32_t tiledb_array_schema_load( +int32_t tiledb_array_schema_load_impl( tiledb_ctx_t* ctx, const char* array_uri, + int with_enumerations, tiledb_array_schema_t** array_schema) { if (sanity_check(ctx) == TILEDB_ERR) return TILEDB_ERR; @@ -822,12 +867,23 @@ int32_t tiledb_array_schema_load( return TILEDB_ERR; } + // With enumerations not supported for remote arrays currently. + if (with_enumerations) { + delete *array_schema; + *array_schema = nullptr; + auto st = Status_Error( + "Failed to load array schema;" + "remote load with enumerations is not yet supported."); + LOG_STATUS_NO_RETURN_VALUE(st); + save_error(ctx, st); + return TILEDB_ERR; + } + auto&& [st, array_schema_rest] = rest_client->get_array_schema_from_rest(uri); if (!st.ok()) { LOG_STATUS_NO_RETURN_VALUE(st); save_error(ctx, st); - delete *array_schema; return TILEDB_ERR; } (*array_schema)->array_schema_ = array_schema_rest.value(); @@ -853,13 +909,21 @@ int32_t tiledb_array_schema_load( auto st = Status_ArrayDirectoryError(le.what()); LOG_STATUS_NO_RETURN_VALUE(st); save_error(ctx, st); - delete *array_schema; return TILEDB_ERR; } // Load latest array schema - auto&& array_schema_latest = array_dir.load_array_schema_latest(key); - (*array_schema)->array_schema_ = array_schema_latest; + try { + auto&& array_schema_latest = array_dir.load_array_schema_latest(key); + if (with_enumerations) { + array_dir.load_all_enumerations(array_schema_latest, key); + } + (*array_schema)->array_schema_ = array_schema_latest; + } catch (...) { + delete *array_schema; + *array_schema = nullptr; + throw; + } } return TILEDB_OK; } @@ -1270,12 +1334,8 @@ int32_t tiledb_array_schema_evolution_add_attribute( sanity_check(ctx, attr) == TILEDB_ERR) return TILEDB_ERR; - throw_if_not_ok( - array_schema_evolution->array_schema_evolution_->add_attribute( - attr->attr_)); - return TILEDB_OK; + array_schema_evolution->array_schema_evolution_->add_attribute(attr->attr_); - // Success return TILEDB_OK; } @@ -1287,9 +1347,41 @@ int32_t tiledb_array_schema_evolution_drop_attribute( sanity_check(ctx, array_schema_evolution) == TILEDB_ERR) return TILEDB_ERR; - throw_if_not_ok( - array_schema_evolution->array_schema_evolution_->drop_attribute( - attribute_name)); + array_schema_evolution->array_schema_evolution_->drop_attribute( + attribute_name); + return TILEDB_OK; +} + +capi_return_t tiledb_array_schema_evolution_add_enumeration( + tiledb_ctx_t* ctx, + tiledb_array_schema_evolution_t* array_schema_evolution, + tiledb_enumeration_t* enumeration) { + if (sanity_check(ctx, array_schema_evolution) == TILEDB_ERR) { + return TILEDB_ERR; + } + + api::ensure_enumeration_is_valid(enumeration); + + auto enmr = enumeration->copy(); + array_schema_evolution->array_schema_evolution_->add_enumeration(enmr); + + return TILEDB_OK; +} + +capi_return_t tiledb_array_schema_evolution_drop_enumeration( + tiledb_ctx_t* ctx, + tiledb_array_schema_evolution_t* array_schema_evolution, + const char* enmr_name) { + if (sanity_check(ctx, array_schema_evolution) == TILEDB_ERR) { + return TILEDB_ERR; + } + + if (enmr_name == nullptr) { + return TILEDB_ERR; + } + + array_schema_evolution->array_schema_evolution_->drop_enumeration(enmr_name); + return TILEDB_OK; } @@ -1302,12 +1394,8 @@ int32_t tiledb_array_schema_evolution_set_timestamp_range( sanity_check(ctx, array_schema_evolution) == TILEDB_ERR) return TILEDB_ERR; - throw_if_not_ok( - array_schema_evolution->array_schema_evolution_->set_timestamp_range( - {lo, hi})); - return TILEDB_OK; - - // Success + array_schema_evolution->array_schema_evolution_->set_timestamp_range( + {lo, hi}); return TILEDB_OK; } @@ -2506,6 +2594,19 @@ int32_t tiledb_query_condition_negate( ctx, cond, nullptr, TILEDB_NOT, negated_cond); } +capi_return_t tiledb_query_condition_set_use_enumeration( + tiledb_ctx_t* ctx, + const tiledb_query_condition_t* cond, + int use_enumeration) { + if (sanity_check(ctx, cond) == TILEDB_ERR) { + return TILEDB_ERR; + } + + cond->query_condition_->set_use_enumeration( + use_enumeration != 0 ? true : false); + return TILEDB_OK; +} + /* ****************************** */ /* ARRAY */ /* ****************************** */ @@ -3411,6 +3512,37 @@ int32_t tiledb_array_evolve( return TILEDB_OK; } +capi_return_t tiledb_array_get_enumeration( + tiledb_ctx_t* ctx, + const tiledb_array_t* array, + const char* attr_name, + tiledb_enumeration_t** enumeration) { + if (sanity_check(ctx, array) == TILEDB_ERR) { + return TILEDB_ERR; + } + + if (attr_name == nullptr) { + throw api::CAPIStatusException("'attr_name' must not be null"); + } + + api::ensure_output_pointer_is_valid(enumeration); + auto ptr = array->array_->get_enumeration(attr_name); + *enumeration = tiledb_enumeration_handle_t::make_handle(ptr); + + return TILEDB_OK; +} + +capi_return_t tiledb_array_load_all_enumerations( + tiledb_ctx_t* ctx, const tiledb_array_t* array, int latest_only) { + if (sanity_check(ctx, array) == TILEDB_ERR) { + return TILEDB_ERR; + } + + array->array_->load_all_enumerations(latest_only ? true : false); + + return TILEDB_OK; +} + int32_t tiledb_array_upgrade_version( tiledb_ctx_t* ctx, const char* array_uri, tiledb_config_t* config) { // Sanity Checks @@ -5540,6 +5672,22 @@ int32_t tiledb_attribute_set_cell_val_num( ctx, attr, cell_val_num); } +capi_return_t tiledb_attribute_set_enumeration_name( + tiledb_ctx_t* ctx, + tiledb_attribute_t* attr, + const char* enumeration_name) noexcept { + return api_entry( + ctx, attr, enumeration_name); +} + +capi_return_t tiledb_attribute_get_enumeration_name( + tiledb_ctx_t* ctx, + tiledb_attribute_t* attr, + tiledb_string_t** name) noexcept { + return api_entry( + ctx, attr, name); +} + int32_t tiledb_attribute_get_name( tiledb_ctx_t* ctx, const tiledb_attribute_t* attr, @@ -5780,6 +5928,14 @@ int32_t tiledb_array_schema_timestamp_range( ctx, array_schema, lo, hi); } +int32_t tiledb_array_schema_add_enumeration( + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_enumeration_t* enumeration) noexcept { + return api_entry( + ctx, array_schema, enumeration); +} + int32_t tiledb_array_schema_set_coords_filter_list( tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, @@ -5813,8 +5969,16 @@ int32_t tiledb_array_schema_load( tiledb_ctx_t* ctx, const char* array_uri, tiledb_array_schema_t** array_schema) noexcept { - return api_entry( - ctx, array_uri, array_schema); + return api_entry( + ctx, array_uri, 0, array_schema); +} + +int32_t tiledb_array_schema_load_with_enumerations( + tiledb_ctx_t* ctx, + const char* array_uri, + tiledb_array_schema_t** array_schema) noexcept { + return api_entry( + ctx, array_uri, 1, array_schema); } int32_t tiledb_array_schema_load_with_key( @@ -5973,6 +6137,22 @@ int32_t tiledb_array_schema_evolution_drop_attribute( ctx, array_schema_evolution, attribute_name); } +int32_t tiledb_array_schema_evolution_add_enumeration( + tiledb_ctx_t* ctx, + tiledb_array_schema_evolution_t* array_schema_evolution, + tiledb_enumeration_t* enmr) noexcept { + return api_entry( + ctx, array_schema_evolution, enmr); +} + +capi_return_t tiledb_array_schema_evolution_drop_enumeration( + tiledb_ctx_t* ctx, + tiledb_array_schema_evolution_t* array_schema_evolution, + const char* enumeration_name) noexcept { + return api_entry( + ctx, array_schema_evolution, enumeration_name); +} + TILEDB_EXPORT int32_t tiledb_array_schema_evolution_set_timestamp_range( tiledb_ctx_t* ctx, tiledb_array_schema_evolution_t* array_schema_evolution, @@ -6608,6 +6788,14 @@ int32_t tiledb_query_condition_negate( ctx, cond, negated_cond); } +capi_return_t tiledb_query_condition_set_use_enumeration( + tiledb_ctx_t* const ctx, + const tiledb_query_condition_t* const cond, + int use_enumeration) noexcept { + return api_entry( + ctx, cond, use_enumeration); +} + /* ****************************** */ /* UPDATE CONDITION */ /* ****************************** */ @@ -6956,6 +7144,21 @@ int32_t tiledb_array_evolve( ctx, array_uri, array_schema_evolution); } +capi_return_t tiledb_array_get_enumeration( + tiledb_ctx_t* ctx, + const tiledb_array_t* array, + const char* attr_name, + tiledb_enumeration_t** enumeration) noexcept { + return api_entry( + ctx, array, attr_name, enumeration); +} + +capi_return_t tiledb_array_load_all_enumerations( + tiledb_ctx_t* ctx, const tiledb_array_t* array, int latest_only) noexcept { + return api_entry( + ctx, array, latest_only); +} + int32_t tiledb_array_upgrade_version( tiledb_ctx_t* ctx, const char* array_uri, diff --git a/tiledb/sm/c_api/tiledb_experimental.h b/tiledb/sm/c_api/tiledb_experimental.h index 8316351700e4..4da66d21a4a5 100644 --- a/tiledb/sm/c_api/tiledb_experimental.h +++ b/tiledb/sm/c_api/tiledb_experimental.h @@ -41,6 +41,7 @@ /* * API sections */ +#include "tiledb/api/c_api/enumeration/enumeration_api_experimental.h" #include "tiledb/api/c_api/group/group_api_external_experimental.h" #include "tiledb/api/c_api/query_plan/query_plan_api_external_experimental.h" #include "tiledb_dimension_label_experimental.h" @@ -155,6 +156,60 @@ TILEDB_EXPORT int32_t tiledb_array_schema_evolution_drop_attribute( tiledb_array_schema_evolution_t* array_schema_evolution, const char* attribute_name) TILEDB_NOEXCEPT; +/** + * Adds an enumeration to an array schema evolution. + * + * **Example:** + * + * @code{.c} + * tiledb_enumeration_t* enmr; + * void* data = get_data(); + * uint64_t data_size = get_data_size(); + * tiledb_enumeration_alloc( + * ctx, + * TILEDB_INT64, + * cell_val_num, + * FALSE, + * data, + * data_size, + * nullptr, + * 0, + * &enumeration); + * tiledb_array_schema_evolution_add_enumeration(ctx, array_schema_evolution, + * enmr); + * @endcode + * + * @param ctx The TileDB context. + * @param array_schema_evolution The schema evolution. + * @param enumeration The enumeration to be added. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_schema_evolution_add_enumeration( + tiledb_ctx_t* ctx, + tiledb_array_schema_evolution_t* array_schema_evolution, + tiledb_enumeration_t* enumeration) TILEDB_NOEXCEPT; + +/** + * Drops an enumeration from an array schema evolution. + * + * **Example:** + * + * @code{.c} + * const char* enumeration_name = "enumeration_1"; + * tiledb_array_schema_evolution_drop_enumeration(ctx, array_schema_evolution, + * enumeration_name); + * @endcode + * + * @param ctx The TileDB context. + * @param array_schema_evolution The schema evolution. + * @param enumeration_name The name of the enumeration to be dropped. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_schema_evolution_drop_enumeration( + tiledb_ctx_t* ctx, + tiledb_array_schema_evolution_t* array_schema_evolution, + const char* enumeration_name) TILEDB_NOEXCEPT; + /** * Sets timestamp range in an array schema evolution * This function sets the output timestamp of the committed array schema after @@ -210,6 +265,103 @@ TILEDB_EXPORT int32_t tiledb_array_schema_timestamp_range( uint64_t* lo, uint64_t* hi) TILEDB_NOEXCEPT; +/** + * Adds an enumeration to an array schema. + * + * **Example:** + * + * @code{.c} + * tiledb_enumeration_t* enumeration; + * tiledb_enumeration_alloc( + * ctx, + * "enumeration_name", + * TILEDB_INT64, + * 1, + * FALSE, + * data, + * data_size, + * nullptr, + * 0, + * &enumeration); + * tiledb_array_schema_add_enumeration(ctx, array_schema, enumeration); + * @endcode + * + * @param ctx The TileDB context. + * @param array_schema The array schema. + * @param enumeration The enumeration to add with the attribute + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT int32_t tiledb_array_schema_add_enumeration( + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_enumeration_t* enumeration) TILEDB_NOEXCEPT; + +/** + * Retrieves the schema of an array from the disk with all enumerations loaded, + * creating an array schema struct. + * + * **Example:** + * + * @code{.c} + * tiledb_array_schema_t* array_schema; + * tiledb_array_schema_load(ctx, "s3://tiledb_bucket/my_array", &array_schema); + * // Make sure to free the array schema in the end + * @endcode + * + * @param ctx The TileDB context. + * @param array_uri The array whose schema will be retrieved. + * @param array_schema The array schema to be retrieved, or `NULL` upon error. + * @return `TILEDB_OK` for success and `TILEDB_OOM` or `TILEDB_ERR` for error. + */ +TILEDB_EXPORT int32_t tiledb_array_schema_load_with_enumerations( + tiledb_ctx_t* ctx, + const char* array_uri, + tiledb_array_schema_t** array_schema) TILEDB_NOEXCEPT; + +/* ********************************* */ +/* ATTRIBUTE ENUMERATIONS */ +/* ********************************* */ + +/** + * Set the enumeration name on an attribute. + * + * **Example:** + * + * @code{.c} + * tiledb_attribute_set_enumeration_name(ctx, attr, "enumeration_name"); + * @endcode + * + * @param ctx The TileDB context. + * @param attr The target attribute. + * @param enumeration_name The name of the enumeration to use for the attribute. + * @return `TILEDB_OK` for success, and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_attribute_set_enumeration_name( + tiledb_ctx_t* ctx, + tiledb_attribute_t* attr, + const char* enumeration_name) TILEDB_NOEXCEPT; + +/** + * Get the attribute's enumeration name if it has one. + * + * **Example:** + * + * @code{.c} + * tiledb_string_t* name; + * tiledb_attribute_get_enumeration_name(ctx, attr, &name); + * @endcode + * + * @param ctx The TileDB context. + * @param attr The target attribute. + * @param name The name of the attribute, nullptr if the attribute does not + * have an associated enumeration. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_attribute_get_enumeration_name( + tiledb_ctx_t* ctx, + tiledb_attribute_t* attr, + tiledb_string_t** name) TILEDB_NOEXCEPT; + /* ********************************* */ /* ARRAY */ /* ********************************* */ @@ -269,6 +421,52 @@ TILEDB_EXPORT int32_t tiledb_array_evolve( const char* array_uri, tiledb_array_schema_evolution_t* array_schema_evolution) TILEDB_NOEXCEPT; +/** + * Retrieves an attribute's enumeration given the attribute name (key). + * + * **Example:** + * + * The following retrieves the first attribute in the schema. + * + * @code{.c} + * tiledb_attribute_t* attr; + * tiledb_array_schema_get_enumeration( + * ctx, array_schema, "attr_0", &enumeration); + * // Make sure to delete the retrieved attribute in the end. + * @endcode + * + * @param ctx The TileDB context. + * @param array The TileDB array. + * @param name The name (key) of the attribute from which to + * retrieve the enumeration. + * @param enumeration The enumeration object to retrieve. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_get_enumeration( + tiledb_ctx_t* ctx, + const tiledb_array_t* array, + const char* name, + tiledb_enumeration_t** enumeration) TILEDB_NOEXCEPT; + +/** + * Load all enumerations for the array. + * + * **Example:** + * + * @code{.c} + * tiledb_array_load_all_enumerations(ctx, array); + * @endcode + * + * @param ctx The TileDB context. + * @param array The TileDB array. + * @param latest_only If non-zero, only load enumerations for the latest schema. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_load_all_enumerations( + tiledb_ctx_t* ctx, + const tiledb_array_t* array, + int latest_only) TILEDB_NOEXCEPT; + /** * Upgrades an array to the latest format version. * @@ -378,6 +576,39 @@ TILEDB_EXPORT int32_t tiledb_query_get_relevant_fragment_num( const tiledb_query_t* query, uint64_t* relevant_fragment_num) TILEDB_NOEXCEPT; +/* ********************************* */ +/* QUERY CONDITION */ +/* ********************************* */ + +/** + * Disable the use of enumerations on the given QueryCondition + * + * **Example:** + * + * @code{.c} + * tiledb_query_condition_t* query_condition; + * tiledb_query_condition_alloc(ctx, &query_condition); + * uint32_t value = 5; + * tiledb_query_condition_init( + * ctx, + * query_condition, + * "longitude", + * &value, + * sizeof(value), + * TILEDB_LT); + * tiledb_query_condition_set_use_enumeration(ctx, query_condition, 0); + * @endcode + * + * @param[in] ctx The TileDB context. + * @param[in] cond The query condition + * @param[in] use_enumeration Non-zero to use the associated enumeration + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT int32_t tiledb_query_condition_set_use_enumeration( + tiledb_ctx_t* ctx, + const tiledb_query_condition_t* cond, + int use_enumeration) TILEDB_NOEXCEPT; + /* ********************************* */ /* QUERY STATUS DETAILS */ /* ********************************* */ diff --git a/tiledb/sm/cpp_api/array_experimental.h b/tiledb/sm/cpp_api/array_experimental.h new file mode 100644 index 000000000000..34df1696d898 --- /dev/null +++ b/tiledb/sm/cpp_api/array_experimental.h @@ -0,0 +1,77 @@ +/** + * @file array_experimental.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares the experimental C++ API for arrays. + */ + +#ifndef TILEDB_CPP_API_ARRAY_EXPERIMENTAL_H +#define TILEDB_CPP_API_ARRAY_EXPERIMENTAL_H + +#include "array.h" +#include "enumeration_experimental.h" +#include "tiledb_experimental.h" + +#include + +namespace tiledb { +class ArrayExperimental { + public: + /** + * Get the Enumeration from the attribute with name `attr_name`. + * + * @param ctx The context to use. + * @param array The array containing the attribute. + * @param attr_name The name of the attribute to get the enumeration from. + * @return Enumeration The enumeration. + */ + static Enumeration get_enumeration( + const Context& ctx, const Array& array, const std::string& attr_name) { + tiledb_enumeration_t* enmr; + ctx.handle_error(tiledb_array_get_enumeration( + ctx.ptr().get(), array.ptr().get(), attr_name.c_str(), &enmr)); + return Enumeration(ctx, enmr); + } + + /** + * Load all enumerations for the array + * + * @param ctx The context to use. + * @param array The array to load enumerations for. + * @param latest_only Whether to only load enumerations for the latest schema. + */ + static void load_all_enumerations( + const Context& ctx, const Array& array, bool latest_only = true) { + ctx.handle_error(tiledb_array_load_all_enumerations( + ctx.ptr().get(), array.ptr().get(), latest_only ? 1 : 0)); + } +}; + +} // namespace tiledb + +#endif diff --git a/tiledb/sm/cpp_api/array_schema_evolution.h b/tiledb/sm/cpp_api/array_schema_evolution.h index 33cec0454661..a8397974e0a4 100644 --- a/tiledb/sm/cpp_api/array_schema_evolution.h +++ b/tiledb/sm/cpp_api/array_schema_evolution.h @@ -151,6 +151,48 @@ class ArraySchemaEvolution { return *this; } + /** + * Adds an Enumeration to the array schema evolution. + * + * **Example:** + * @code{.cpp} + * tiledb::Context ctx; + * tiledb::ArraySchemaEvolution schema_evolution(ctx); + * std::vector values = {"red", "green", "blue"}; + * schema_evolution.add_enumeration(Enumeration::create(ctx, "an_enumeration", + * values)); + * @endcode + * + * @param attr The Attribute to add + * @return Reference to this `ArraySchemaEvolution` instance. + */ + ArraySchemaEvolution& add_enumeration(const Enumeration& enmr) { + auto& ctx = ctx_.get(); + ctx.handle_error(tiledb_array_schema_evolution_add_enumeration( + ctx.ptr().get(), evolution_.get(), enmr.ptr().get())); + return *this; + } + + /** + * Drops an enumeration. + * + * **Example:** + * @code{.cpp} + * tiledb::Context ctx; + * tiledb::ArraySchemaEvolution schema_evolution(ctx); + * schema_evolution.drop_enumeration("enumeration_name"); + * @endcode + * + * @param enumeration_name The enumeration to be dropped + * @return Reference to this `ArraySchemaEvolution` instance. + */ + ArraySchemaEvolution& drop_enumeration(const std::string& enumeration_name) { + auto& ctx = ctx_.get(); + ctx.handle_error(tiledb_array_schema_evolution_drop_enumeration( + ctx.ptr().get(), evolution_.get(), enumeration_name.c_str())); + return *this; + } + /** * Sets timestamp range. * @@ -217,4 +259,4 @@ class ArraySchemaEvolution { } // namespace tiledb -#endif // TILEDB_CPP_API_ARRAY_SCHEMA_EVOLUTION_H \ No newline at end of file +#endif // TILEDB_CPP_API_ARRAY_SCHEMA_EVOLUTION_H diff --git a/tiledb/sm/cpp_api/array_schema_experimental.h b/tiledb/sm/cpp_api/array_schema_experimental.h index e712860126a6..1bbee96ed689 100644 --- a/tiledb/sm/cpp_api/array_schema_experimental.h +++ b/tiledb/sm/cpp_api/array_schema_experimental.h @@ -35,6 +35,7 @@ #include "array_schema.h" #include "dimension_label_experimental.h" +#include "enumeration_experimental.h" #include "filter_list.h" #include "tiledb_experimental.h" @@ -43,6 +44,22 @@ namespace tiledb { class ArraySchemaExperimental { public: + /** + * Load an ArraySchema from the given URI with all of its enumerations. + * + * @param ctx The TileDB context. + * @param uri The URI to load from. + * @return ArraySchema The loaded array schema. + */ + static ArraySchema load_with_enumerations( + const Context& ctx, const std::string& uri) { + tiledb_ctx_t* c_ctx = ctx.ptr().get(); + tiledb_array_schema_t* schema; + ctx.handle_error(tiledb_array_schema_load_with_enumerations( + c_ctx, uri.c_str(), &schema)); + return ArraySchema(ctx, schema); + } + /** * Adds a DimensionLabel to the array. * @@ -154,6 +171,21 @@ class ArraySchemaExperimental { ctx.ptr().get(), array_schema.ptr().get(), name.c_str(), &dim_label)); return DimensionLabel(ctx, dim_label); } + + /** + * Add an enumeration to the array schema. + * + * @param ctx TileDB context. + * @param array_schema Target array schema. + * @param enmr The enumeration to add. + */ + static void add_enumeration( + const Context& ctx, + const ArraySchema& array_schema, + const Enumeration& enmr) { + ctx.handle_error(tiledb_array_schema_add_enumeration( + ctx.ptr().get(), array_schema.ptr().get(), enmr.ptr().get())); + } }; } // namespace tiledb diff --git a/tiledb/sm/cpp_api/attribute_experimental.h b/tiledb/sm/cpp_api/attribute_experimental.h new file mode 100644 index 000000000000..7070f1188cd3 --- /dev/null +++ b/tiledb/sm/cpp_api/attribute_experimental.h @@ -0,0 +1,94 @@ +/** + * @file attribute_experimental.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares the experimental C++ API for attributes. + */ + +#ifndef TILEDB_CPP_API_ATTRIBUTE_EXPERIMENTAL_H +#define TILEDB_CPP_API_ATTRIBUTE_EXPERIMENTAL_H + +#include "attribute.h" +#include "context.h" +#include "enumeration_experimental.h" + +#include + +namespace tiledb { +class AttributeExperimental { + public: + /** + * Set the enumeration name for an attribute. + * + * @param ctx TileDB context. + * @param attribute Target attribute. + * @param enumeration_name The name of the enumeration to set. + */ + static void set_enumeration_name( + const Context& ctx, + Attribute& attribute, + const std::string& enumeration_name) { + ctx.handle_error(tiledb_attribute_set_enumeration_name( + ctx.ptr().get(), attribute.ptr().get(), enumeration_name.c_str())); + } + + /** + * Get the attribute's enumeration name. + * + * @param ctx TileDB context. + * @param attribute Target attribute. + * @return std::optional The enumeration name if one exists. + */ + static std::optional get_enumeration_name( + const Context& ctx, Attribute& attribute) { + // Get the enumeration name as a string handle + tiledb_string_t* enmr_name; + tiledb_ctx_t* c_ctx = ctx.ptr().get(); + ctx.handle_error(tiledb_attribute_get_enumeration_name( + c_ctx, attribute.ptr().get(), &enmr_name)); + + if (enmr_name == nullptr) { + return std::nullopt; + } + + // Convert string handle to a std::string + const char* name_ptr; + size_t name_len; + ctx.handle_error(tiledb_string_view(enmr_name, &name_ptr, &name_len)); + std::string ret(name_ptr, name_len); + + // Release the string handle + ctx.handle_error(tiledb_string_free(&enmr_name)); + + return ret; + } +}; + +} // namespace tiledb + +#endif diff --git a/tiledb/sm/cpp_api/deleter.h b/tiledb/sm/cpp_api/deleter.h index c72d8ef71a1f..61f9a5d2c92f 100644 --- a/tiledb/sm/cpp_api/deleter.h +++ b/tiledb/sm/cpp_api/deleter.h @@ -111,6 +111,10 @@ class Deleter { tiledb_domain_free(&p); } + void operator()(tiledb_enumeration_t* p) const { + tiledb_enumeration_free(&p); + } + void operator()(tiledb_vfs_t* p) const { tiledb_vfs_free(&p); } diff --git a/tiledb/sm/cpp_api/enumeration_experimental.h b/tiledb/sm/cpp_api/enumeration_experimental.h new file mode 100644 index 000000000000..af4b8aa1bf80 --- /dev/null +++ b/tiledb/sm/cpp_api/enumeration_experimental.h @@ -0,0 +1,369 @@ +/** + * @file enumeration_experimental.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares the C++ API for the TileDB Enumeration class. + */ + +#ifndef TILEDB_CPP_API_ENUMERATION_EXPERIMENTAL_H +#define TILEDB_CPP_API_ENUMERATION_EXPERIMENTAL_H + +#include "context.h" +#include "tiledb.h" +#include "tiledb_experimental.h" + +namespace tiledb { + +class Enumeration { + public: + /* ********************************* */ + /* CONSTRUCTORS & DESTRUCTORS */ + /* ********************************* */ + + /** + * Create an enumeration by wrapping a pointer allocated by the C API + * + * @param ctx The Context to use. + * @param enmr The tiledb_enumeration_t allocated by the C API. + */ + Enumeration(const Context& ctx, tiledb_enumeration_t* enmr) + : ctx_(ctx) { + enumeration_ = std::shared_ptr(enmr, deleter_); + } + + Enumeration(const Enumeration&) = default; + Enumeration(Enumeration&&) = default; + Enumeration& operator=(const Enumeration&) = default; + Enumeration& operator=(Enumeration&&) = default; + + /** Destructor. */ + ~Enumeration() = default; + + /* ********************************* */ + /* API */ + /* ********************************* */ + + /** Get the C TileDB context object. + * + * @return shared_ptr + */ + std::shared_ptr ptr() const { + return enumeration_; + } + + /** + * Get the name of the enumeration + * + * @return std::string The name of the enumeration. + */ + std::string name() const { + // Get the enumeration name as a string handle + tiledb_string_t* enmr_name; + tiledb_ctx_t* c_ctx = ctx_.get().ptr().get(); + ctx_.get().handle_error( + tiledb_enumeration_get_name(c_ctx, enumeration_.get(), &enmr_name)); + + // Convert string handle to a std::string + const char* name_ptr; + size_t name_len; + ctx_.get().handle_error( + tiledb_string_view(enmr_name, &name_ptr, &name_len)); + std::string ret(name_ptr, name_len); + + // Release the string handle + ctx_.get().handle_error(tiledb_string_free(&enmr_name)); + + return ret; + } + + /** + * Get the type of the enumeration + * + * @return tiledb_datatype_t The datatype of the enumeration values. + */ + tiledb_datatype_t type() const { + tiledb_datatype_t ret; + tiledb_ctx_t* c_ctx = ctx_.get().ptr().get(); + ctx_.get().handle_error( + tiledb_enumeration_get_type(c_ctx, enumeration_.get(), &ret)); + return ret; + } + + /** + * Get the cell_val_num of the enumeration + * + * @return uint32_t The cell_val_num of the enumeration. + */ + uint32_t cell_val_num() const { + uint32_t ret; + tiledb_ctx_t* c_ctx = ctx_.get().ptr().get(); + ctx_.get().handle_error( + tiledb_enumeration_get_cell_val_num(c_ctx, enumeration_.get(), &ret)); + return ret; + } + + /** + * Get whether this enumeration is considered ordered. + * + * If an enumeration is not considered ordered inequality operators are + * disabled in QueryConditions applied against the enumeration values. + * + * @return bool Whether the enumeration is considered ordered. + */ + bool ordered() const { + int is_ordered; + tiledb_ctx_t* c_ctx = ctx_.get().ptr().get(); + ctx_.get().handle_error( + tiledb_enumeration_get_ordered(c_ctx, enumeration_.get(), &is_ordered)); + return is_ordered ? true : false; + } + + /** + * Convert the enumeration values into a vector + * + * @param vec The vector that will store the values of the enumeration. + */ + template + std::vector, T>> + as_vector() { + tiledb_ctx_t* c_ctx = ctx_.get().ptr().get(); + + const void* data; + uint64_t data_size; + ctx_.get().handle_error(tiledb_enumeration_get_data( + c_ctx, enumeration_.get(), &data, &data_size)); + + const T* elems = static_cast(data); + size_t count = data_size / sizeof(T); + + std::vector ret; + ret.reserve(count); + for (size_t i = 0; i < count; i++) { + ret.push_back(elems[i]); + } + + return ret; + } + + /** + * Convert the enumeration values into a vector of std::string values + * + * @param vec The vector used to return the string data. + */ + template + std::vector, T>> + as_vector() { + tiledb_ctx_t* c_ctx = ctx_.get().ptr().get(); + + const void* data; + uint64_t data_size; + ctx_.get().handle_error(tiledb_enumeration_get_data( + c_ctx, enumeration_.get(), &data, &data_size)); + + const void* offsets; + uint64_t offsets_size; + ctx_.get().handle_error(tiledb_enumeration_get_offsets( + c_ctx, enumeration_.get(), &offsets, &offsets_size)); + + const char* str_data = static_cast(data); + const uint64_t* elems = static_cast(offsets); + size_t count = offsets_size / sizeof(uint64_t); + + std::vector ret; + ret.reserve(count); + for (size_t i = 0; i < count; i++) { + uint64_t len; + if (i + 1 < count) { + len = elems[i + 1] - elems[i]; + } else { + len = data_size - elems[i]; + } + ret.emplace_back(str_data + elems[i], len); + } + + return ret; + } + + /** + * Dump a string representation of the Enumeration to the given FILE pointer. + * + * @param out A FILE pointer to write to. stdout is used if nullptr is passed. + */ + void dump(FILE* out = nullptr) const { + ctx_.get().handle_error(tiledb_enumeration_dump( + ctx_.get().ptr().get(), enumeration_.get(), out)); + } + + /* ********************************* */ + /* STATIC FUNCTIONS */ + /* ********************************* */ + + /** + * Create an enumeration from a vector of trivial values (i.e., int's or other + * integral or floating point values) + * + * @param ctx The context to use. + * @param values The list of values to use for this enumeration. + * @param ordered Whether or not to consider this enumeration ordered. + * @param type The datatype of the enumeration values. This is automatically + * deduced if not provided. + */ + template * = nullptr> + static Enumeration create( + const Context& ctx, + const std::string& name, + std::vector& values, + bool ordered = false, + tiledb_datatype_t type = static_cast(255)) { + using DataT = impl::TypeHandler; + + if (type == static_cast(255)) { + type = DataT::tiledb_type; + } + + return create( + ctx, + name, + type, + DataT::tiledb_num, + ordered, + values.data(), + values.size() * sizeof(T), + nullptr, + 0); + } + + /** + * Create an enumeration from a vector of strings + * + * @param ctx The context to use. + * @param values The vector of values for the enumeration. + * @param ordered Whether to consider the enumerationv alues as ordered. + * @param type The datatype of the enumeration values. This is automatically + * deduced if not provided. However, this can be used to override the + * deduced type if need be. For instance, TILEDB_STRING_ASCII is the + * default type for strings but TILEDB_STRING_UTF8 can be specified. + */ + template * = nullptr> + static Enumeration create( + const Context& ctx, + const std::string& name, + std::vector>& values, + bool ordered = false, + tiledb_datatype_t type = static_cast(255)) { + using DataT = impl::TypeHandler; + static_assert( + DataT::tiledb_num == 1, "Enumeration string values cannot be compound"); + + if (type == static_cast(255)) { + type = DataT::tiledb_type; + } + + uint64_t total_size = 0; + for (auto v : values) { + total_size += v.size() * sizeof(T); + } + + std::vector data(total_size, 0); + std::vector offsets; + offsets.reserve(values.size()); + uint64_t curr_offset = 0; + + for (auto v : values) { + std::memcpy(data.data() + curr_offset, v.data(), v.size() * sizeof(T)); + offsets.push_back(curr_offset); + curr_offset += v.size() * sizeof(T); + } + + return create( + ctx, + name, + type, + TILEDB_VAR_NUM, + ordered, + data.data(), + total_size, + offsets.data(), + offsets.size() * sizeof(uint64_t)); + } + + /** + * Create an enumeration + * + * @param ctx The context to use. + * @param type The datatype of the enumeration values. + * @param cell_val_num The cell_val_num of the enumeration. + * @param ordered Whether this enumeration should be considered ordered. + * @param data A pointer to a buffer of values for this enumeration. + * @param data_size The size of the buffer pointed to by data. + * @param offsets A pointer to the offsets buffer if required. + * @param offsets_size The size of the buffer pointed to by offsets. + */ + static Enumeration create( + const Context& ctx, + const std::string& name, + tiledb_datatype_t type, + uint32_t cell_val_num, + bool ordered, + const void* data, + uint64_t data_size, + const void* offsets, + uint64_t offsets_size) { + tiledb_enumeration_t* enumeration; + ctx.handle_error(tiledb_enumeration_alloc( + ctx.ptr().get(), + name.c_str(), + type, + cell_val_num, + ordered, + data, + data_size, + offsets, + offsets_size, + &enumeration)); + return Enumeration(ctx, enumeration); + } + + private: + /* ********************************* */ + /* PRIVATE ATTRIBUTES */ + /* ********************************* */ + + /** The TileDB context. */ + std::reference_wrapper ctx_; + + /** Deleter wrapper. */ + impl::Deleter deleter_; + + /** Pointer to the TileDB C Enumeration object. */ + std::shared_ptr enumeration_; +}; + +} // namespace tiledb + +#endif // TILEDB_CPP_API_ENUMERATION_EXPERIMENTAL_H diff --git a/tiledb/sm/cpp_api/query_condition_experimental.h b/tiledb/sm/cpp_api/query_condition_experimental.h new file mode 100644 index 000000000000..8d982b4e5624 --- /dev/null +++ b/tiledb/sm/cpp_api/query_condition_experimental.h @@ -0,0 +1,57 @@ +/** + * @file query_condition_experimental.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares the experimental C++ API for the query condition. + */ + +#ifndef TILEDB_CPP_API_QUERY_CONDITION_EXPERIMENTAL_H +#define TILEDB_CPP_API_QUERY_CONDITION_EXPERIMENTAL_H + +#include "query_condition.h" + +namespace tiledb { +class QueryConditionExperimental { + public: + /** + * Set whether or not to use the associated enumeration. + * + * @param ctx TileDB Context. + * @param cond TileDB Query Condition. + * @param use_enumeration Whether to use the associated enumeration. + */ + static void set_use_enumeration( + const Context& ctx, QueryCondition& cond, bool use_enumeration) { + ctx.handle_error(tiledb_query_condition_set_use_enumeration( + ctx.ptr().get(), cond.ptr().get(), use_enumeration ? 1 : 0)); + } +}; + +} // namespace tiledb + +#endif diff --git a/tiledb/sm/cpp_api/tiledb_experimental b/tiledb/sm/cpp_api/tiledb_experimental index df23d84ae518..2bdb0d673e4b 100644 --- a/tiledb/sm/cpp_api/tiledb_experimental +++ b/tiledb/sm/cpp_api/tiledb_experimental @@ -33,11 +33,15 @@ #ifndef TILEDB_EXPERIMENTAL_CPP_H #define TILEDB_EXPERIMENTAL_CPP_H +#include "array_experimental.h" #include "array_schema_evolution.h" #include "array_schema_experimental.h" +#include "attribute_experimental.h" #include "dimension_label_experimental.h" +#include "enumeration_experimental.h" #include "consolidation_plan_experimental.h" #include "group_experimental.h" +#include "query_condition_experimental.h" #include "query_experimental.h" #include "subarray_experimental.h" diff --git a/tiledb/sm/enums/datatype.h b/tiledb/sm/enums/datatype.h index c2050d6d674b..dfa59ef3f420 100644 --- a/tiledb/sm/enums/datatype.h +++ b/tiledb/sm/enums/datatype.h @@ -45,6 +45,12 @@ using namespace tiledb::common; namespace tiledb { namespace sm { +#if defined(_WIN32) && defined(TIME_MS) +#pragma message("WARNING: Windows.h may have already been included before") +#pragma message(" tiledb/sm/enums/datatype.h which will likely cause a") +#pragma message(" syntax error while defining the Datatype enum class.") +#endif + /** Defines a datatype. */ enum class Datatype : uint8_t { #define TILEDB_DATATYPE_ENUM(id) id @@ -325,11 +331,43 @@ inline bool datatype_is_string(Datatype type) { /** Returns true if the input datatype is an integer type. */ inline bool datatype_is_integer(Datatype type) { return ( - type == Datatype::BLOB || type == Datatype::BOOL || - type == Datatype::INT8 || type == Datatype::UINT8 || - type == Datatype::INT16 || type == Datatype::UINT16 || - type == Datatype::INT32 || type == Datatype::UINT32 || - type == Datatype::INT64 || type == Datatype::UINT64); + type == Datatype::BOOL || type == Datatype::INT8 || + type == Datatype::UINT8 || type == Datatype::INT16 || + type == Datatype::UINT16 || type == Datatype::INT32 || + type == Datatype::UINT32 || type == Datatype::INT64 || + type == Datatype::UINT64); +} + +/** + * Return the largest integral value for the provided type. + * + * @param type The Datatype to return the maximum value of. + * @returns uint64_t The maximum integral value for the provided type. + */ +inline uint64_t datatype_max_integral_value(Datatype type) { + switch (type) { + case Datatype::BOOL: + return static_cast(std::numeric_limits::max()); + case Datatype::INT8: + return static_cast(std::numeric_limits::max()); + case Datatype::UINT8: + return static_cast(std::numeric_limits::max()); + case Datatype::INT16: + return static_cast(std::numeric_limits::max()); + case Datatype::UINT16: + return static_cast(std::numeric_limits::max()); + case Datatype::INT32: + return static_cast(std::numeric_limits::max()); + case Datatype::UINT32: + return static_cast(std::numeric_limits::max()); + case Datatype::INT64: + return static_cast(std::numeric_limits::max()); + case Datatype::UINT64: + return static_cast(std::numeric_limits::max()); + default: + throw std::runtime_error( + "Datatype (" + datatype_str(type) + ") is not integral."); + } } /** Returns true if the input datatype is a real type. */ diff --git a/tiledb/sm/filter/bit_width_reduction_filter.cc b/tiledb/sm/filter/bit_width_reduction_filter.cc index f114382e8e4c..08c14ab68205 100644 --- a/tiledb/sm/filter/bit_width_reduction_filter.cc +++ b/tiledb/sm/filter/bit_width_reduction_filter.cc @@ -108,7 +108,8 @@ Status BitWidthReductionFilter::run_forward( auto tile_type_size = static_cast(datatype_size(tile_type)); // If bit width compression can't work, just return the input unmodified. - if (!datatype_is_integer(tile_type) || tile_type_size == 1) { + if ((!datatype_is_integer(tile_type) && tile_type != Datatype::BLOB) || + tile_type_size == 1) { RETURN_NOT_OK(output->append_view(input)); RETURN_NOT_OK(output_metadata->append_view(input_metadata)); return Status::Ok(); @@ -292,7 +293,8 @@ Status BitWidthReductionFilter::run_reverse( auto tile_type_size = static_cast(datatype_size(tile_type)); // If bit width compression wasn't applied, just return the input unmodified. - if (!datatype_is_integer(tile_type) || tile_type_size == 1) { + if ((!datatype_is_integer(tile_type) && tile_type != Datatype::BLOB) || + tile_type_size == 1) { RETURN_NOT_OK(output->append_view(input)); RETURN_NOT_OK(output_metadata->append_view(input_metadata)); return Status::Ok(); diff --git a/tiledb/sm/filter/positive_delta_filter.cc b/tiledb/sm/filter/positive_delta_filter.cc index 5b4212b8ad2f..85ed05b8b761 100644 --- a/tiledb/sm/filter/positive_delta_filter.cc +++ b/tiledb/sm/filter/positive_delta_filter.cc @@ -70,7 +70,7 @@ Status PositiveDeltaFilter::run_forward( auto tile_type = tile.type(); // If encoding can't work, just return the input unmodified. - if (!datatype_is_integer(tile_type)) { + if (!datatype_is_integer(tile_type) && tile_type != Datatype::BLOB) { RETURN_NOT_OK(output->append_view(input)); RETURN_NOT_OK(output_metadata->append_view(input_metadata)); return Status::Ok(); @@ -250,7 +250,7 @@ Status PositiveDeltaFilter::run_reverse( auto tile_type = tile.type(); // If encoding wasn't applied, just return the input unmodified. - if (!datatype_is_integer(tile_type)) { + if (!datatype_is_integer(tile_type) && tile_type != Datatype::BLOB) { RETURN_NOT_OK(output->append_view(input)); RETURN_NOT_OK(output_metadata->append_view(input_metadata)); return Status::Ok(); diff --git a/tiledb/sm/misc/constants.cc b/tiledb/sm/misc/constants.cc index bb268dfec46f..60f70639fa8c 100644 --- a/tiledb/sm/misc/constants.cc +++ b/tiledb/sm/misc/constants.cc @@ -93,6 +93,9 @@ const std::string fragment_metadata_filename = "__fragment_metadata.tdb"; /** The array dimension labels directory name. */ const std::string array_dimension_labels_dir_name = "__labels"; +/** The array enumerations directory name. */ +const std::string array_enumerations_dir_name = "__enumerations"; + /** The array directory names. */ const std::vector array_dir_names = { array_schema_dir_name, @@ -100,7 +103,8 @@ const std::vector array_dir_names = { array_fragment_meta_dir_name, array_fragments_dir_name, array_commits_dir_name, - array_dimension_labels_dir_name}; + array_dimension_labels_dir_name, + array_enumerations_dir_name}; /** The default tile capacity. */ const uint64_t capacity = 10000; @@ -222,6 +226,9 @@ const uint32_t empty_ucs4 = 0; /** The special value for an empty ANY. */ const uint8_t empty_any = 0; +/** The return value for missing entries in an Enumeration */ +const uint64_t enumeration_missing_value = std::numeric_limits::max(); + /** The file suffix used in TileDB. */ const std::string file_suffix = ".tdb"; @@ -660,7 +667,7 @@ const int32_t library_version[3] = { TILEDB_VERSION_MAJOR, TILEDB_VERSION_MINOR, TILEDB_VERSION_PATCH}; /** The TileDB serialization base format version number. */ -const format_version_t base_format_version = 19; +const format_version_t base_format_version = 20; /** * The TileDB serialization format version number. @@ -685,6 +692,12 @@ const format_version_t deletes_min_version = 16; /** The lowest version supported for updates. */ const format_version_t updates_min_version = 16; +/** The lowest version supported format version for enumerations. */ +const format_version_t enumerations_min_format_version = 20; + +/** The current enumerations version. */ +const format_version_t enumerations_version = 0; + /** The maximum size of a tile chunk (unit of compression) in bytes. */ const uint64_t max_tile_chunk_size = 64 * 1024; diff --git a/tiledb/sm/misc/constants.h b/tiledb/sm/misc/constants.h index fa059291b493..2b023dc4210c 100644 --- a/tiledb/sm/misc/constants.h +++ b/tiledb/sm/misc/constants.h @@ -83,6 +83,9 @@ extern const std::string array_commits_dir_name; /** The array dimension labels directory name. */ extern const std::string array_dimension_labels_dir_name; +/** The array enumerations directory name. */ +extern const std::string array_enumerations_dir_name; + /** The array directory names. */ extern const std::vector array_dir_names; @@ -203,6 +206,9 @@ extern const uint32_t empty_ucs4; /** The special value for an empty ANY. */ extern const uint8_t empty_any; +/** The return value for missing entries in an Enumeration */ +extern const uint64_t enumeration_missing_value; + /** The file suffix used in TileDB. */ extern const std::string file_suffix; @@ -654,6 +660,12 @@ extern const format_version_t deletes_min_version; /** The lowest version supported for updates. */ extern const format_version_t updates_min_version; +/** The lowest version supported for enumerations. */ +extern const format_version_t enumerations_min_format_version; + +/** The current Enumerations version. */ +extern const format_version_t enumerations_version; + /** The maximum size of a tile chunk (unit of compression) in bytes. */ extern const uint64_t max_tile_chunk_size; @@ -737,7 +749,6 @@ extern const uint64_t s3_min_multipart_part_size; * global order writes intermediate chunks */ extern const std::string s3_multipart_buffering_dirname; - } // namespace constants } // namespace sm diff --git a/tiledb/sm/misc/integral_type_casts.h b/tiledb/sm/misc/integral_type_casts.h new file mode 100644 index 000000000000..a82611765368 --- /dev/null +++ b/tiledb/sm/misc/integral_type_casts.h @@ -0,0 +1,144 @@ +/** + * @file type_casts.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file contains safe type casting utilities. + */ + +#ifndef TILEDB_TYPE_CASTS_H +#define TILEDB_TYPE_CASTS_H + +#include +#include + +#include "tiledb/common/unreachable.h" +#include "tiledb/sm/enums/datatype.h" +#include "tiledb/sm/misc/types.h" + +namespace tiledb::sm::utils { + +/** + * Safely convert integral values between different bit widths checking for + * invalid conversions. These casts are named "safe" because they only allow + * casts that result in the same semantic value in the target type. This means + * that some casts that would be allowed by static_cast or similar facilities + * are rejected. Most notably, casts between types with different signed-ness + * are more thoroughly checked for correctness. + * + * This would likely be significantly faster if there was a cross platform + * interface for `__builtin_clzll`. However, the current needs don't require + * the absolute fastest speed as this is currently only used once per + * Enumeration attribute per query (as opposed to once per query condition + * comparison or something crazy). + * + * @param src The value to convert. + * @return The converted value. + */ +template +Target safe_integral_cast(Source src) { + using SourceLimits = std::numeric_limits; + using TargetLimits = std::numeric_limits; + + static_assert(SourceLimits::is_integer, "Source is not an integral type"); + static_assert(TargetLimits::is_integer, "Target is not an integral type"); + + Target ret = static_cast(src); + + // If it can't round trip, its an invalid cast. Note that the converse + // is not true as a sign could have changed for types of the same bit width + // but different signed-ness. + if (static_cast(ret) != src) { + throw std::invalid_argument("Invalid integral cast: Roundtrip failed"); + } + + // If the sign changed + if ((src < 0 && ret >= 0) || (src >= 0 && ret < 0)) { + throw std::invalid_argument("Invalid integral cast: Sign changed"); + } + + return ret; +} + +/** + * Use `safe_integral_cast` to convert an integral value into a specific + * Datatype stored in a ByteVecValue. + * + * @param value The value to convert. + * @param type The datatype to convert the value to. + * @param dest A ByteVecValue to store the converted value in. + */ +template +void safe_integral_cast_to_datatype( + Source value, Datatype type, ByteVecValue& dest) { + if (!datatype_is_integer(type)) { + throw std::invalid_argument("Datatype must be integral"); + } + + if (type == Datatype::BLOB) { + throw std::invalid_argument( + "Datatype::BLOB not supported in integral conversion"); + } + + switch (type) { + case Datatype::BOOL: + dest.assign_as(safe_integral_cast(value)); + return; + case Datatype::INT8: + dest.assign_as(safe_integral_cast(value)); + return; + case Datatype::UINT8: + dest.assign_as(safe_integral_cast(value)); + return; + case Datatype::INT16: + dest.assign_as(safe_integral_cast(value)); + return; + case Datatype::UINT16: + dest.assign_as(safe_integral_cast(value)); + return; + case Datatype::INT32: + dest.assign_as(safe_integral_cast(value)); + return; + case Datatype::UINT32: + dest.assign_as(safe_integral_cast(value)); + return; + case Datatype::INT64: + dest.assign_as(safe_integral_cast(value)); + return; + case Datatype::UINT64: + dest.assign_as(safe_integral_cast(value)); + return; + default: + throw std::logic_error("Definitions of integral types are mismatched."); + } + + ::stdx::unreachable(); +} + +} // namespace tiledb::sm::utils + +#endif // TILEDB_TYPE_CASTS_H diff --git a/tiledb/sm/misc/test/CMakeLists.txt b/tiledb/sm/misc/test/CMakeLists.txt index 6b9adac5e5b3..0c6bd18206d1 100644 --- a/tiledb/sm/misc/test/CMakeLists.txt +++ b/tiledb/sm/misc/test/CMakeLists.txt @@ -30,5 +30,12 @@ commence(unit_test misc) this_target_link_libraries(tiledb_test_support_lib) # change to `this_target_include_directories` when available target_include_directories(unit_misc PRIVATE "${CMAKE_SOURCE_DIR}") - this_target_sources(main.cc unit_bytevecvalue.cc unit_hilbert.cc unit_math.cc unit_uuid.cc) + this_target_sources( + main.cc + unit_bytevecvalue.cc + unit_hilbert.cc + unit_integral_type_casts.cc + unit_math.cc + unit_uuid.cc + ) conclude(unit_test) diff --git a/tiledb/sm/misc/test/unit_integral_type_casts.cc b/tiledb/sm/misc/test/unit_integral_type_casts.cc new file mode 100644 index 000000000000..26bb9dd190a6 --- /dev/null +++ b/tiledb/sm/misc/test/unit_integral_type_casts.cc @@ -0,0 +1,511 @@ +/** + * @file unit_integral_type_casts.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * Tests the C++ API for enumeration related functions. + */ + +#include +#include + +#include "test/support/tdb_catch.h" +#include "tiledb/sm/misc/integral_type_casts.h" + +// These tests are for the safe_integral_cast and safe_itegral_cast_to_datatype +// functions found in `tiledb/sm/misc/integral_type_casts.h`. +// +// There are two main tests in this file, one for safe_integral_cast and one +// for safe_integral_cast_to_datatype. These tests work by generating a +// comprehensive sample of test data to exhaustively check that casts between +// all supported integral datatypes are covered. Generating tests in this +// manner allows us to assert whether a cast should succeed vs one that should +// fail. +// +// Both tests have the same general outline. First, pass a callback to the +// run_test function which is responsible for generating a TestCase value +// which is nothing more than a type width (i.e., sizeof(int32_t)) and a +// set of random bytes in length 1 to byte_width. This ensures that we're +// able to test casts that should succeed (i.e., 5 stored in an int32_t is +// castable to a uint8_t) and casts that should fail (i.e., 300 stored in an +// int32_t is not castable to either int8_t or uint8_t). +// +// The callback passed to `run_test` is then responsible for enumerating all +// of the test types for cast checks. Both tests have the same naming pattern +// of: +// +// 1. check_$function_name - Based on TestCase.byte_width, pick the source +// datatype for the test +// 2. dispatch_$function_name - Attempt to convert the source test data to all +// target types. +// 3. run_$functon_name - Attempt the cast for the given TestData, source, and +// target types. These use the should_cast_succeed helper to know whether +// a given cast should succeed or fail. +// +// Beyond the main test layouts, there are two important helper functions +// that will be useful for understanding. +// +// First, `should_cast_succeed` which takes the source type and value, and a +// target type and returns a boolean on whether a cast should succeed. This +// is the crux of these tests as this logic is the concise list of rules that +// dictate when a cast is safe or not. +// +// The second function is `calculate_width` which is called by +// `should_cast_succeed`. This calculates the number of bits and bytes required +// to hold a given value and informs `should_cast_succeed` when a cast is +// representable in a target type. If the code in this function looks a little +// weird, you'll want to review 2's complement representation on why it counts +// things as it does. + +using namespace tiledb::sm; + +/* ********************************* */ +/* CONSTANTS */ +/* ********************************* */ + +// Widths of integral types int8_t to int64_t +constexpr int TYPE_WIDTHS[] = {1, 2, 4, 8}; +constexpr int NUM_TYPE_WIDTHS = 4; + +// Width of widest type (i.e., sizeof(int64_t)) +constexpr int MAX_WIDTH = 8; + +// An aribtrary number of iterations to run per test +constexpr int RANDOM_ITERATIONS = 500; + +// A POD struct representing a test case +struct TestCase { + // The width of the type, i.e., sizeof(int8_t), sizeof(uint32_t), etc, this + // is obviously restricted to the values 1, 2, 4, 8. + uint8_t type_width; + + // Storage for the generated value's data. LSB is data[0] + uint8_t data[MAX_WIDTH]; +}; + +/* ********************************* */ +/* TEST CALLBACKS */ +/* ********************************* */ + +// Test implementation for util::safe_integral_cast +void check_safe_integral_cast(const TestCase& tc); + +// Test implementation for util::safe_integral_cast_to_datatype +void check_safe_integral_cast_to_datatype(const TestCase& tc); + +/* ********************************* */ +/* TEST HELPER FUNCTIONS */ +/* ********************************* */ + +// Test driver function +template +void run_test(TestFunc func); + +// Apply the test function against each source type based on tc.type_width +template +void dispatch(TestFunc func, const TestCase& tc); + +// Return a boolean indicating whether the given cast should succeed or fail +template +bool should_cast_succeed(Source src); + +template +bool should_cast_succeed(Source src, Datatype dtype); + +// Calculate the width of a value in both bits and bytes. +template +std::tuple calculate_width(Type val); + +// Fills in a TestCase's data, byte_width, and msb_set members +void generate_case(TestCase& tc, uint8_t gen_byte_width, bool set_msb); + +// Convert a uint8_t[] to integral type T +template +T buf_to_value(const uint8_t* buffer); + +/* ********************************* */ +/* safe_integral_cast */ +/* ********************************* */ + +TEST_CASE("util::safe_integral_cast", "[safe-integral-casts]") { + run_test(check_safe_integral_cast); +} + +template +void dispatch_safe_integral_cast(const TestCase& tc); + +template +void run_safe_integral_cast(const TestCase&, Source src); + +void check_safe_integral_cast(const TestCase& tc) { + switch (tc.type_width) { + case 1: + dispatch_safe_integral_cast(tc); + dispatch_safe_integral_cast(tc); + return; + case 2: + dispatch_safe_integral_cast(tc); + dispatch_safe_integral_cast(tc); + return; + case 4: + dispatch_safe_integral_cast(tc); + dispatch_safe_integral_cast(tc); + return; + case 8: + dispatch_safe_integral_cast(tc); + dispatch_safe_integral_cast(tc); + return; + default: + throw std::logic_error("Invalid type_width"); + } +} + +template +void dispatch_safe_integral_cast(const TestCase& tc) { + Source val = buf_to_value(tc.data); + + run_safe_integral_cast(tc, val); + run_safe_integral_cast(tc, val); + + run_safe_integral_cast(tc, val); + run_safe_integral_cast(tc, val); + + run_safe_integral_cast(tc, val); + run_safe_integral_cast(tc, val); + + run_safe_integral_cast(tc, val); + run_safe_integral_cast(tc, val); +} + +template +void run_safe_integral_cast(const TestCase&, Source src) { + if (should_cast_succeed(src)) { + auto tgt = utils::safe_integral_cast(src); + REQUIRE(src == utils::safe_integral_cast(tgt)); + } else { + REQUIRE_THROWS(utils::safe_integral_cast(src)); + } +} + +/* ********************************* */ +/* safe_integral_cast_to_datatype */ +/* ********************************* */ + +TEST_CASE("util::safe_integral_cast_to_datatype", "[safe-integral-casts]") { + run_test(check_safe_integral_cast_to_datatype); +} + +TEST_CASE( + "util::safe_integral_cast_to_datatype bad type", + "[safe-integral-casts][error]") { + ByteVecValue bvv; + REQUIRE_THROWS(utils::safe_integral_cast_to_datatype(5, Datatype::BLOB, bvv)); + REQUIRE_THROWS( + utils::safe_integral_cast_to_datatype(5, Datatype::STRING_ASCII, bvv)); +} + +template +void dispatch_safe_integral_cast_to_datatype(const TestCase& tc); + +template +void run_safe_integral_cast_to_datatype( + const TestCase&, Source src, Datatype dt); + +void check_safe_integral_cast_to_datatype(const TestCase& tc) { + switch (tc.type_width) { + case 1: + dispatch_safe_integral_cast_to_datatype(tc); + dispatch_safe_integral_cast_to_datatype(tc); + return; + case 2: + dispatch_safe_integral_cast_to_datatype(tc); + dispatch_safe_integral_cast_to_datatype(tc); + return; + case 4: + dispatch_safe_integral_cast_to_datatype(tc); + dispatch_safe_integral_cast_to_datatype(tc); + return; + case 8: + dispatch_safe_integral_cast_to_datatype(tc); + dispatch_safe_integral_cast_to_datatype(tc); + return; + default: + throw std::logic_error("Invalid type_width"); + } +} + +template +void dispatch_safe_integral_cast_to_datatype(const TestCase& tc) { + Source val = buf_to_value(tc.data); + + std::vector datatypes = { + Datatype::BOOL, + Datatype::INT8, + Datatype::UINT8, + Datatype::INT16, + Datatype::UINT16, + Datatype::INT32, + Datatype::UINT32, + Datatype::INT64, + Datatype::UINT64}; + + for (auto dt : datatypes) { + run_safe_integral_cast_to_datatype(tc, val, dt); + } +} + +template +void run_safe_integral_cast_to_datatype( + const TestCase&, Source src, Datatype dtype) { + ByteVecValue dest; + if (should_cast_succeed(src, dtype)) { + utils::safe_integral_cast_to_datatype(src, dtype, dest); + switch (dtype) { + case Datatype::BOOL: + REQUIRE(dest.rvalue_as() == (uint8_t)src); + return; + case Datatype::INT8: + REQUIRE(dest.rvalue_as() == (int8_t)src); + return; + case Datatype::UINT8: + REQUIRE(dest.rvalue_as() == (uint8_t)src); + return; + case Datatype::INT16: + REQUIRE(dest.rvalue_as() == (int16_t)src); + return; + case Datatype::UINT16: + REQUIRE(dest.rvalue_as() == (uint16_t)src); + return; + case Datatype::INT32: + REQUIRE(dest.rvalue_as() == (int32_t)src); + return; + case Datatype::UINT32: + REQUIRE(dest.rvalue_as() == (uint32_t)src); + return; + case Datatype::INT64: + REQUIRE(dest.rvalue_as() == (int64_t)src); + return; + case Datatype::UINT64: + REQUIRE(dest.rvalue_as() == (uint64_t)src); + return; + default: + throw std::logic_error("Invalid datatype for test"); + } + } else { + REQUIRE_THROWS( + utils::safe_integral_cast_to_datatype(src, dtype, dest)); + } +} + +/* ********************************* */ +/* TEST HELPER IMPLEMENTATIONS */ +/* ********************************* */ + +template +void run_test(TestFunc tfunc) { + TestCase tc; + for (uint8_t i = 0; i < NUM_TYPE_WIDTHS; i++) { + tc.type_width = TYPE_WIDTHS[i]; + + // Always check no bits set edge case + memset(tc.data, 0, MAX_WIDTH); + tfunc(tc); + + for (uint8_t gen_width = 1; gen_width <= tc.type_width; gen_width++) { + for (int i = 0; i < RANDOM_ITERATIONS; i++) { + generate_case(tc, gen_width, false); + tfunc(tc); + + generate_case(tc, gen_width, true); + tfunc(tc); + } + } + + // Always check all bits set edge case + memset(tc.data, 255, MAX_WIDTH); + tfunc(tc); + } +} + +template +bool should_cast_succeed(Source src) { + auto [bit_width, byte_width] = calculate_width(src); + bool tgt_signed = std::numeric_limits::is_signed; + uint8_t tgt_size = sizeof(Target); + + // Obviously, if we have more bytes in src than the Target type can hold, + // the cast will fail. + if (byte_width > tgt_size) { + return false; + } + + // Unsigned types can't hold negative values, so if src is negative + // and Target is not signed, we must fail. + if (src < 0 && !tgt_signed) { + return false; + } + + // When converting from a positive value to a signed type we have to make + // sure that the signed type has more than bit_width + 1 bits available so + // that the resulting value doesn't become negative. + if ((src > 0 && tgt_signed) && (bit_width >= (tgt_size * 8))) { + return false; + } + + // When converting to a negative value to a signed type, we have to make + // sure the target has enough bits to represent the value. + if ((src < 0 && tgt_signed) && (bit_width >= (tgt_size * 8))) { + return false; + } + + // Otherwise, the cast should succeed + return true; +} + +template +bool should_cast_succeed(Source src, Datatype dtype) { + switch (dtype) { + case Datatype::BOOL: + return should_cast_succeed(src); + case Datatype::INT8: + return should_cast_succeed(src); + case Datatype::UINT8: + return should_cast_succeed(src); + case Datatype::INT16: + return should_cast_succeed(src); + case Datatype::UINT16: + return should_cast_succeed(src); + case Datatype::INT32: + return should_cast_succeed(src); + case Datatype::UINT32: + return should_cast_succeed(src); + case Datatype::INT64: + return should_cast_succeed(src); + case Datatype::UINT64: + return should_cast_succeed(src); + default: + throw std::logic_error("Invalid datatype for test"); + } +} + +template +std::tuple calculate_width(Type val) { + // Calculate the number of bits required to hold src. For positive values, + // this is the largest numbered bit set to 1, for negative values its the + // largest numbered bit set to 0. + + uint8_t bit_width = 0; + if (val > 0) { + // For positive values, we find the highest set bit by counting the number + // of right shifts required before the value is zero. + uint8_t i = 0; + while (val > 0) { + val >>= 1; + i += 1; + } + bit_width = i; + } else if (val < 0) { + // For negative values, we find the highest unset bit by counting the + // number of left shifts before the value becomes greater than or equal + // to zero. + uint8_t i = 0; + while (val < 0) { + val <<= 1; + i += 1; + } + bit_width = sizeof(Type) * 8 - i; + } + + uint8_t byte_width = bit_width / 8; + if (bit_width % 8 != 0) { + byte_width += 1; + } + + return {bit_width, byte_width}; +} + +// A simple function for generating random uint64_t values +uint64_t generate_value() { + static std::random_device rand_dev; + static std::mt19937_64 generator(rand_dev()); + static std::uniform_int_distribution dist; + return dist(generator); +} + +// Generate a TestCase with the given gen_width number of random bytes. The +// only curisoity is the set_msb value which forces the most significant bit +// of the most significant byte to 0 or 1. Forcing the msb to 1 allows us to +// assert that we're testing two important cases. First, casting negative +// values between different types widths, and second, checking that large +// unsigned values of a given type that aren't representable in the signed +// type of the same byte width. +void generate_case(TestCase& tc, uint8_t gen_width, bool set_msb) { + // Reset TestCase state + memset(tc.data, 0, MAX_WIDTH); + + // Convince GCC that we'll not write beyond MAX_WIDTH bytes into tc.data + if (gen_width > MAX_WIDTH) { + throw std::logic_error("Invalid gen_width in test case"); + } + + // A standard bit shift approach for converting integral values into + // an array of uint8_t values. Its important to note that the << and >> + // operators are endian-ness agnostic. So this just masks everything but + // the least-significant-byte, converts that to a uint8_t and stores it + // in the array. Then drop the least-significant byte of the random value + // by shifting 8 bits to the right. + // + // TestCase.byte_width is calculated here since its perfectly valid for + // a generated value to not have any bits set in any given byte which can + // cause expected failures to unexpected succeed if its the MSB that is + // all zeroes. + uint64_t rand_value = generate_value(); + for (int i = 0; i < gen_width; i++) { + tc.data[i] = static_cast(rand_value & 0xFF); + rand_value >>= 8; + } + + // If set_msb is false, set the gen_width's byte msb to 0, else set to 1 + if (!set_msb) { + tc.data[gen_width - 1] &= 0x7F; + } else { + tc.data[gen_width - 1] |= 0x80; + } +} + +// Convert a uint8_t buffer to the given type T. This works using the standard +// bit twiddling approach to progressive bitwise-or one byte at a time in a +// most-to-least significant byte order. +template +T buf_to_value(const uint8_t* buffer) { + T ret = 0; + for (int8_t i = sizeof(T) - 1; i >= 0; i--) { + T new_byte = (T)buffer[i]; + ret = (ret << 8) | new_byte; + } + return ret; +} diff --git a/tiledb/sm/misc/utils.cc b/tiledb/sm/misc/utils.cc index a3ac5a6aa967..3124e4e0af1a 100644 --- a/tiledb/sm/misc/utils.cc +++ b/tiledb/sm/misc/utils.cc @@ -33,6 +33,7 @@ #include "tiledb/sm/misc/utils.h" #include "tiledb/common/logger.h" +#include "tiledb/common/unreachable.h" #include "tiledb/sm/enums/datatype.h" #include "tiledb/sm/filesystem/uri.h" #include "tiledb/sm/misc/constants.h" diff --git a/tiledb/sm/query/ast/CMakeLists.txt b/tiledb/sm/query/ast/CMakeLists.txt index 4568075fbeb7..c46c57fd7244 100644 --- a/tiledb/sm/query/ast/CMakeLists.txt +++ b/tiledb/sm/query/ast/CMakeLists.txt @@ -39,7 +39,7 @@ list(APPEND SOURCES # commence(object_library query_ast) this_target_sources(${SOURCES}) - this_target_object_libraries(array_schema baseline) + this_target_object_libraries(array_schema generic_tile_io baseline) conclude(object_library) add_test_subdirectory() diff --git a/tiledb/sm/query/ast/query_ast.cc b/tiledb/sm/query/ast/query_ast.cc index ddec36e555ea..eb77c708dc7b 100644 --- a/tiledb/sm/query/ast/query_ast.cc +++ b/tiledb/sm/query/ast/query_ast.cc @@ -31,6 +31,8 @@ */ #include "query_ast.h" +#include "tiledb/sm/array_schema/enumeration.h" +#include "tiledb/sm/misc/integral_type_casts.h" using namespace tiledb::common; @@ -60,10 +62,65 @@ void ASTNodeVal::get_field_names( field_name_set.insert(field_name_); } +void ASTNodeVal::get_enumeration_field_names( + std::unordered_set& field_name_set) const { + if (use_enumeration_) { + field_name_set.insert(field_name_); + } +} + bool ASTNodeVal::is_backwards_compatible() const { return true; } +void ASTNodeVal::rewrite_enumeration_conditions( + const ArraySchema& array_schema) { + // This is called by the Query class before applying a query condition. This + // works by looking up each related enumeration and translating the + // condition's value to reflect the underlying index value. I.e., if the + // query condition was created with `my_attr = "foo"`, and `my_attr` is an + // attribute with an enumeration, this will replace the condition's value + // with the index value returned by `Enumeration::index_of()`. + + if (!use_enumeration_) { + return; + } + + if (!array_schema.is_attr(field_name_)) { + return; + } + + auto attr = array_schema.attribute(field_name_); + auto enmr_name = attr->get_enumeration_name(); + if (!enmr_name.has_value()) { + return; + } + + auto enumeration = array_schema.get_enumeration(enmr_name.value()); + if (!enumeration) { + throw std::logic_error( + "Missing requried enumeration for field '" + field_name_ + "'"); + } + + if (!enumeration->ordered() && + (op_ != QueryConditionOp::EQ && op_ != QueryConditionOp::NE)) { + throw std::logic_error( + "Cannot apply an inequality operator against an unordered Enumeration"); + } + + auto idx = enumeration->index_of(condition_value_view_); + auto val_size = datatype_size(attr->type()); + + condition_value_data_ = ByteVecValue(val_size); + utils::safe_integral_cast_to_datatype( + idx, attr->type(), condition_value_data_); + + condition_value_view_ = + UntypedDatumView(condition_value_data_.data(), val_size); + + use_enumeration_ = false; +} + Status ASTNodeVal::check_node_validity(const ArraySchema& array_schema) const { const uint64_t condition_value_size = condition_value_data_.size(); @@ -186,6 +243,14 @@ const QueryConditionCombinationOp& ASTNodeVal::get_combination_op() const { "value node."); } +bool ASTNodeVal::use_enumeration() const { + return use_enumeration_; +} + +void ASTNodeVal::set_use_enumeration(bool use_enumeration) { + use_enumeration_ = use_enumeration; +} + bool ASTNodeExpr::is_expr() const { return true; } @@ -205,6 +270,13 @@ void ASTNodeExpr::get_field_names( } } +void ASTNodeExpr::get_enumeration_field_names( + std::unordered_set& field_name_set) const { + for (const auto& child : nodes_) { + child->get_enumeration_field_names(field_name_set); + } +} + bool ASTNodeExpr::is_backwards_compatible() const { if (combination_op_ != QueryConditionCombinationOp::AND) { return false; @@ -217,6 +289,13 @@ bool ASTNodeExpr::is_backwards_compatible() const { return true; } +void ASTNodeExpr::rewrite_enumeration_conditions( + const ArraySchema& array_schema) { + for (auto& child : nodes_) { + child->rewrite_enumeration_conditions(array_schema); + } +} + Status ASTNodeExpr::check_node_validity(const ArraySchema& array_schema) const { // If the node is a compound expression node, ensure there are at least // two children in the node and then run a check on each child node. @@ -275,9 +354,22 @@ const QueryConditionOp& ASTNodeExpr::get_op() const { const std::vector>& ASTNodeExpr::get_children() const { return nodes_; } + const QueryConditionCombinationOp& ASTNodeExpr::get_combination_op() const { return combination_op_; } +bool ASTNodeExpr::use_enumeration() const { + throw std::runtime_error( + "ASTNodeExpr::use_enumeration: Cannot get enumeration status from " + "an AST expression node."); +} + +void ASTNodeExpr::set_use_enumeration(bool use_enumeration) { + for (auto& child : nodes_) { + child->set_use_enumeration(use_enumeration); + } +} + } // namespace sm } // namespace tiledb diff --git a/tiledb/sm/query/ast/query_ast.h b/tiledb/sm/query/ast/query_ast.h index 30a08884d905..2a7f35e23898 100644 --- a/tiledb/sm/query/ast/query_ast.h +++ b/tiledb/sm/query/ast/query_ast.h @@ -91,11 +91,19 @@ class ASTNode { * @brief Gets the set of field names from all the value nodes in the ASTNode. * * @param field_name_set The set variable the function populates. - * @return std::unordered_set& Set of the field names in the - * node. */ virtual void get_field_names( std::unordered_set& field_name_set) const = 0; + + /** + * @brief Gets the set of field names from all the value nodes that reference + * an enumerated field. + * + * @param field_name_set The set variable the function populates. + */ + virtual void get_enumeration_field_names( + std::unordered_set& field_name_set) const = 0; + /** * @brief Returns true if the AST is previously supported by previous versions * of TileDB. This means that the AST should only have AND combination ops, @@ -106,6 +114,15 @@ class ASTNode { */ virtual bool is_backwards_compatible() const = 0; + /** + * @brief Update an node value condition values that refer to enumerated + * attributes. + * + * @param array_schema The array schema with all relevant enumerations loaded. + */ + virtual void rewrite_enumeration_conditions( + const ArraySchema& array_schema) = 0; + /** * @brief Checks whether the node is valid based on the array schema of the * array that the query condition that contains this AST node will execute a @@ -176,6 +193,26 @@ class ASTNode { */ virtual const QueryConditionCombinationOp& get_combination_op() const = 0; + /** + * @brief Return whether this node's condition should be applied against + * the attributes enumerated values or the underlying index data if + * applicable for a given attribute. + * + * @return bool If true, apply this condition against the enumerated values. + */ + virtual bool use_enumeration() const = 0; + + /** + * @brief By default, a query condition is applied against an enumeration's + * values. This can be disabled to apply a given condition against the + * underlying integer data stored for the attribute by passing `false` to + * this method. + * + * @param use_enumeration A bool indicating whether this condition should be + * applied against the enumerations values or not. + */ + virtual void set_use_enumeration(bool use_enumeration) = 0; + /** * @brief Default virtual destructor. */ @@ -196,12 +233,15 @@ class ASTNodeVal : public ASTNode { * @param condition_value_size The byte size of condition_value. * @param op The relational operation between the value of the field * and `condition_value`. + * @param use_enumeration Whether or not to use an associated + * enumeration at query time. */ ASTNodeVal( const std::string& field_name, const void* const condition_value, const uint64_t condition_value_size, - const QueryConditionOp op) + const QueryConditionOp op, + bool use_enumeration = true) : field_name_(field_name) , condition_value_data_(condition_value_size) , condition_value_view_( @@ -209,7 +249,8 @@ class ASTNodeVal : public ASTNode { (void*)"" : condition_value_data_.data()), condition_value_data_.size()) - , op_(op) { + , op_(op) + , use_enumeration_(use_enumeration) { if (condition_value_view_.size() != 0) { memcpy( condition_value_data_.data(), condition_value, condition_value_size); @@ -228,7 +269,8 @@ class ASTNodeVal : public ASTNode { (void*)"" : condition_value_data_.data()), condition_value_data_.size()) - , op_(rhs.op_) { + , op_(rhs.op_) + , use_enumeration_(rhs.use_enumeration_) { } /** @@ -243,7 +285,8 @@ class ASTNodeVal : public ASTNode { (void*)"" : condition_value_data_.data()), condition_value_data_.size()) - , op_(negate_query_condition_op(rhs.op_)) { + , op_(negate_query_condition_op(rhs.op_)) + , use_enumeration_(rhs.use_enumeration_) { } /** @@ -284,12 +327,19 @@ class ASTNodeVal : public ASTNode { * @brief Gets the set of field names from all the value nodes in the ASTNode. * * @param field_name_set The set variable the function populates. - * @return std::unordered_set& Set of the field names in the - * node. */ void get_field_names( std::unordered_set& field_name_set) const override; + /** + * @brief Gets the set of field names from all the value nodes that reference + * an enumerated field. + * + * @param field_name_set The set variable the function populates. + */ + void get_enumeration_field_names( + std::unordered_set& field_name_set) const override; + /** * @brief Returns true if the AST is previously supported by previous versions * of TileDB. This means that the AST should only have AND combination ops, @@ -300,6 +350,17 @@ class ASTNodeVal : public ASTNode { */ bool is_backwards_compatible() const override; + /** + * Update any node value condition values that refer to enumerated attributes. + * For any value condition on an attribute that has an Enumeration, the + * user specified value is passed to the attribute's enumeration `index_of` + * method to replace the user provided value with the Enumeration's value + * index. + * + * @param array_schema The array schema with all relevant enumerations loaded. + */ + void rewrite_enumeration_conditions(const ArraySchema& array_schema) override; + /** * @brief Checks whether the node is valid based on the array schema of the * array that the query condition that contains this AST node will execute a @@ -370,6 +431,26 @@ class ASTNodeVal : public ASTNode { */ const QueryConditionCombinationOp& get_combination_op() const override; + /** + * @brief Return whether this node's condition should be applied against + * the attributes enumerated values or the underlying index data if + * applicable for a given attribute. + * + * @return bool If true, apply this condition against the enumerated values. + */ + bool use_enumeration() const override; + + /** + * @brief By default, a query condition is applied against an enumeration's + * values. This can be disabled to apply a given condition against the + * underlying integer data stored for the attribute by passing `false` to + * this method. + * + * @param use_enumeration A bool indicating whether this condition should be + * applied against the enumerations values or not. + */ + void set_use_enumeration(bool use_enumeration) override; + private: /** The attribute name. */ std::string field_name_; @@ -382,6 +463,9 @@ class ASTNodeVal : public ASTNode { /** The comparison operator. */ QueryConditionOp op_; + + /** Whether this condiiton applies to enumerated values if applicable */ + bool use_enumeration_; }; /** @@ -461,12 +545,19 @@ class ASTNodeExpr : public ASTNode { * @brief Gets the set of field names from all the value nodes in the ASTNode. * * @param field_name_set The set variable the function populates. - * @return std::unordered_set& Set of the field names in the - * node. */ void get_field_names( std::unordered_set& field_name_set) const override; + /** + * @brief Gets the set of field names from all the value nodes that reference + * an enumerated field. + * + * @param field_name_set The set variable the function populates. + */ + void get_enumeration_field_names( + std::unordered_set& field_name_set) const override; + /** * @brief Returns true if the AST is previously supported by previous versions * of TileDB. This means that the AST should only have AND combination ops, @@ -477,6 +568,14 @@ class ASTNodeExpr : public ASTNode { */ bool is_backwards_compatible() const override; + /** + * @brief Update an node value condition values that refer to enumerated + * attributes. + * + * @param array_schema The array schema with all relevant enumerations loaded. + */ + void rewrite_enumeration_conditions(const ArraySchema& array_schema) override; + /** * @brief Checks whether the node is valid based on the array schema of the * array that the query condition that contains this AST node will execute a @@ -547,6 +646,31 @@ class ASTNodeExpr : public ASTNode { */ const QueryConditionCombinationOp& get_combination_op() const override; + /** + * @brief Return whether this node's condition should be applied against + * the attributes enumerated values or the underlying index data if + * applicable for a given attribute. + * + * This method always throws when called on an expression node. + * + * @return bool If true, apply this condition against the enumerated values. + */ + bool use_enumeration() const override; + + /** + * @brief By default, a query condition is applied against an enumeration's + * values. This can be disabled to apply a given condition against the + * underlying integer data stored for the attribute by passing `false` to + * this method. + * + * When called on an expression node this value is recursively applied + * against all value nodes in the AST. + * + * @param use_enumeration A bool indicating whether this condition should be + * applied against the enumerations values or not. + */ + void set_use_enumeration(bool use_enumeration) override; + private: /** The node list **/ std::vector> nodes_; @@ -558,4 +682,4 @@ class ASTNodeExpr : public ASTNode { } // namespace sm } // namespace tiledb -#endif // TILEDB_QUERY_AST_H \ No newline at end of file +#endif // TILEDB_QUERY_AST_H diff --git a/tiledb/sm/query/query.cc b/tiledb/sm/query/query.cc index c496e5244115..cc4bc0857675 100644 --- a/tiledb/sm/query/query.cc +++ b/tiledb/sm/query/query.cc @@ -795,6 +795,37 @@ Status Query::process() { } } + if (condition_.has_value()) { + auto& names = condition_->enumeration_field_names(); + std::unordered_set deduped_enmr_names; + for (auto name : names) { + auto attr = array_schema_->attribute(name); + if (attr == nullptr) { + continue; + } + auto enmr_name = attr->get_enumeration_name(); + if (enmr_name.has_value()) { + deduped_enmr_names.insert(enmr_name.value()); + } + } + std::vector enmr_names; + enmr_names.reserve(deduped_enmr_names.size()); + for (auto& enmr_name : deduped_enmr_names) { + enmr_names.emplace_back(enmr_name); + } + + throw_if_not_ok(parallel_for( + storage_manager_->compute_tp(), + 0, + enmr_names.size(), + [&](const uint64_t i) { + array_->get_enumeration(enmr_names[i]); + return Status::Ok(); + })); + + condition_->rewrite_enumeration_conditions(array_schema()); + } + // Update query status. status_ = QueryStatus::INPROGRESS; diff --git a/tiledb/sm/query/query_condition.cc b/tiledb/sm/query/query_condition.cc index a8c78c8ff409..b93f78e21af3 100644 --- a/tiledb/sm/query/query_condition.cc +++ b/tiledb/sm/query/query_condition.cc @@ -123,6 +123,15 @@ Status QueryCondition::init( return Status::Ok(); } +void QueryCondition::rewrite_enumeration_conditions( + const ArraySchema& array_schema) { + if (!tree_) { + return; + } + + tree_->rewrite_enumeration_conditions(array_schema); +} + Status QueryCondition::check(const ArraySchema& array_schema) const { if (!tree_) { return Status::Ok(); @@ -144,6 +153,7 @@ Status QueryCondition::combine( } combined_cond->field_names_.clear(); + combined_cond->enumeration_field_names_.clear(); combined_cond->tree_ = this->tree_->combine(rhs.tree_, combination_op); return Status::Ok(); } @@ -158,6 +168,7 @@ Status QueryCondition::negate( } combined_cond->field_names_.clear(); + combined_cond->enumeration_field_names_.clear(); combined_cond->tree_ = this->tree_->get_negated_tree(); return Status::Ok(); } @@ -174,6 +185,15 @@ std::unordered_set& QueryCondition::field_names() const { return field_names_; } +std::unordered_set& QueryCondition::enumeration_field_names() + const { + if (enumeration_field_names_.empty() && tree_ != nullptr) { + tree_->get_enumeration_field_names(enumeration_field_names_); + } + + return enumeration_field_names_; +} + uint64_t QueryCondition::condition_timestamp() const { if (condition_marker_.empty()) { return 0; @@ -188,6 +208,10 @@ uint64_t QueryCondition::condition_timestamp() const { return timestamps.first; } +void QueryCondition::set_use_enumeration(bool use_enumeration) { + tree_->set_use_enumeration(use_enumeration); +} + /** Full template specialization for `char*` and `QueryConditionOp::LT`. */ template <> struct QueryCondition::BinaryCmpNullChecks { diff --git a/tiledb/sm/query/query_condition.h b/tiledb/sm/query/query_condition.h index 717f047ceab0..b0c16154ce3d 100644 --- a/tiledb/sm/query/query_condition.h +++ b/tiledb/sm/query/query_condition.h @@ -112,6 +112,15 @@ class QueryCondition { uint64_t condition_value_size, const QueryConditionOp& op); + /** + * Translate any query conditions against enumerated attributes to the + * underlying attribute type. + * + * @param array_schema The current array schema with all required enumerations + * loaded. + */ + void rewrite_enumeration_conditions(const ArraySchema& array_schema); + /** * Verifies that the current state contains supported comparison * operations. Currently, we support the following: @@ -160,6 +169,12 @@ class QueryCondition { */ std::unordered_set& field_names() const; + /** + * Returns a set of all unique field names that reference an enumerated + * attribute condition in the AST representing the query condition. + */ + std::unordered_set& enumeration_field_names() const; + /** * Returns the timestamp for this condition. */ @@ -245,6 +260,17 @@ class QueryCondition { */ uint64_t condition_index() const; + /** + * By default, a query condition is applied against the enumerated values + * of an attribute. Setting use_enumeration to false prevents the translation + * and applies this query condition directly against the underlying integral + * attribute data. + * + * @param use_enumeration A bool indicating whether to use the enumeration + * values. + */ + void set_use_enumeration(bool use_enumeration); + private: /* ********************************* */ /* PRIVATE DATATYPES */ @@ -294,6 +320,9 @@ class QueryCondition { /** Caches all field names in the value nodes of the AST. */ mutable std::unordered_set field_names_; + /** Caches all field names that references enumerations in the AST. */ + mutable std::unordered_set enumeration_field_names_; + /* ********************************* */ /* PRIVATE METHODS */ /* ********************************* */ diff --git a/tiledb/sm/serialization/array_schema.cc b/tiledb/sm/serialization/array_schema.cc index 126cddb37d82..a9a57277a820 100644 --- a/tiledb/sm/serialization/array_schema.cc +++ b/tiledb/sm/serialization/array_schema.cc @@ -42,6 +42,7 @@ #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/array_schema/dimension_label.h" #include "tiledb/sm/array_schema/domain.h" +#include "tiledb/sm/array_schema/enumeration.h" #include "tiledb/sm/enums/array_type.h" #include "tiledb/sm/enums/compressor.h" #include "tiledb/sm/enums/data_order.h" @@ -352,6 +353,11 @@ void attribute_to_capnp( const auto& filters = attribute->filters(); auto filter_pipeline_builder = attribute_builder->initFilterPipeline(); throw_if_not_ok(filter_pipeline_to_capnp(&filters, &filter_pipeline_builder)); + + auto enmr_name = attribute->get_enumeration_name(); + if (enmr_name.has_value()) { + attribute_builder->setEnumerationName(enmr_name.value()); + } } shared_ptr attribute_from_capnp( @@ -399,6 +405,11 @@ shared_ptr attribute_from_capnp( datatype, attribute_reader.getCellValNum()); } + std::optional enmr_name; + if (attribute_reader.hasEnumerationName()) { + enmr_name = attribute_reader.getEnumerationName(); + } + return tiledb::common::make_shared( HERE(), attribute_reader.getName(), @@ -408,7 +419,8 @@ shared_ptr attribute_from_capnp( *(filters.get()), fill_value_vec, fill_value_validity, - data_order); + data_order, + enmr_name); } Status dimension_to_capnp( @@ -763,6 +775,61 @@ shared_ptr dimension_label_from_capnp( is_relative); } +void enumeration_to_capnp( + shared_ptr enumeration, + capnp::Enumeration::Builder& enmr_builder) { + enmr_builder.setName(enumeration->name()); + enmr_builder.setType(datatype_str(enumeration->type())); + enmr_builder.setCellValNum(enumeration->cell_val_num()); + enmr_builder.setOrdered(enumeration->ordered()); + + auto dspan = enumeration->data(); + enmr_builder.setData(::kj::arrayPtr(dspan.data(), dspan.size())); + + if (enumeration->var_size()) { + auto ospan = enumeration->offsets(); + enmr_builder.setOffsets(::kj::arrayPtr(ospan.data(), ospan.size())); + } +} + +shared_ptr enumeration_from_capnp( + const capnp::Enumeration::Reader& reader) { + auto name = reader.getName(); + auto path_name = reader.getPathName(); + Datatype datatype = Datatype::ANY; + throw_if_not_ok(datatype_enum(reader.getType(), &datatype)); + + if (!reader.hasData()) { + throw SerializationStatusException( + "[Deserialization::enumeration_from_capnp] Deserialization of " + "Enumeration is missing its data buffer."); + } + + auto data_reader = reader.getData().asBytes(); + auto data = data_reader.begin(); + auto data_size = data_reader.size(); + + const void* offsets = nullptr; + uint64_t offsets_size = 0; + + if (reader.hasOffsets()) { + auto offsets_reader = reader.getOffsets().asBytes(); + offsets = offsets_reader.begin(); + offsets_size = offsets_reader.size(); + } + + return Enumeration::create( + name, + path_name, + datatype, + reader.getCellValNum(), + reader.getOrdered(), + data, + data_size, + offsets, + offsets_size); +} + Status array_schema_to_capnp( const ArraySchema& array_schema, capnp::ArraySchema::Builder* array_schema_builder, @@ -809,9 +876,9 @@ Status array_schema_to_capnp( // Attributes const unsigned num_attrs = array_schema.attribute_num(); - auto attributes_buidler = array_schema_builder->initAttributes(num_attrs); + auto attribute_builders = array_schema_builder->initAttributes(num_attrs); for (size_t i = 0; i < num_attrs; i++) { - auto attribute_builder = attributes_buidler[i]; + auto attribute_builder = attribute_builders[i]; attribute_to_capnp(array_schema.attribute(i), &attribute_builder); } @@ -833,6 +900,27 @@ Status array_schema_to_capnp( } } + // Loaded enumerations + auto loaded_enmr_names = array_schema.get_loaded_enumeration_names(); + const unsigned num_loaded_enmrs = loaded_enmr_names.size(); + auto enmr_builders = array_schema_builder->initEnumerations(num_loaded_enmrs); + for (size_t i = 0; i < num_loaded_enmrs; i++) { + auto enmr = array_schema.get_enumeration(loaded_enmr_names[i]); + auto builder = enmr_builders[i]; + enumeration_to_capnp(enmr, builder); + } + + // Enumeration path map + auto enmr_names = array_schema.get_enumeration_names(); + const unsigned num_enmr_names = enmr_names.size(); + auto enmr_path_map_builders = + array_schema_builder->initEnumerationPathMap(num_enmr_names); + for (size_t i = 0; i < num_enmr_names; i++) { + auto enmr_path_name = array_schema.get_enumeration_path_name(enmr_names[i]); + enmr_path_map_builders[i].setKey(enmr_names[i]); + enmr_path_map_builders[i].setValue(enmr_path_name); + } + return Status::Ok(); } @@ -999,6 +1087,39 @@ ArraySchema array_schema_from_capnp( } } + // Loaded enumerations + std::vector> enumerations; + if (schema_reader.hasEnumerations()) { + auto enmr_readers = schema_reader.getEnumerations(); + enumerations.reserve(enmr_readers.size()); + try { + for (auto&& enmr_reader : enmr_readers) { + enumerations.emplace_back(enumeration_from_capnp(enmr_reader)); + } + } catch (const std::exception& e) { + std::throw_with_nested(std::runtime_error( + "[Deserialization::array_schema_from_capnp] Cannot deserialize " + "enumerations")); + } + } + + // Enumeration path map + std::unordered_map enmr_path_map; + if (schema_reader.hasEnumerationPathMap()) { + auto enmr_path_map_readers = schema_reader.getEnumerationPathMap(); + try { + for (auto&& kv_reader : enmr_path_map_readers) { + auto enmr_name = kv_reader.getKey(); + auto enmr_path_name = kv_reader.getValue(); + enmr_path_map[enmr_name] = enmr_path_name; + } + } catch (const std::exception& e) { + std::throw_with_nested(std::runtime_error( + "[Deserialization::array_schema_from_capnp] Cannot deserialize " + "enumeration path map")); + } + } + // Set the range if we have two values // #TODO Add security validation std::pair timestamp_range; @@ -1028,6 +1149,8 @@ ArraySchema array_schema_from_capnp( capacity, attributes, dimension_labels, + enumerations, + enmr_path_map, cell_var_offsets_filters, cell_validity_filters, coords_filters); diff --git a/tiledb/sm/serialization/array_schema.h b/tiledb/sm/serialization/array_schema.h index b926c3990a41..53b8be082c0a 100644 --- a/tiledb/sm/serialization/array_schema.h +++ b/tiledb/sm/serialization/array_schema.h @@ -119,6 +119,25 @@ void dimension_label_to_capnp( shared_ptr dimension_label_from_capnp( const capnp::DimensionLabel::Reader& reader); +/** + * Serialize an Enumeration do cap'n proto object + * + * @param enumeration Enumeration to serialize. + * @param enmr_builder Cap'n proto class. + */ +void enumeration_to_capnp( + shared_ptr enumeration, + capnp::Enumeration::Builder& enmr_builder); + +/** + * Deserialize a dimension label from a cap'n proto object + * + * @param reader Cap'n proto reader object + * @return A new Enumeration + */ +shared_ptr enumeration_from_capnp( + const capnp::Enumeration::Reader& reader); + #endif // TILEDB_SERIALIZATION /** diff --git a/tiledb/sm/serialization/array_schema_evolution.cc b/tiledb/sm/serialization/array_schema_evolution.cc index cb06262467c4..369b3c15f8f1 100644 --- a/tiledb/sm/serialization/array_schema_evolution.cc +++ b/tiledb/sm/serialization/array_schema_evolution.cc @@ -42,6 +42,7 @@ #include "tiledb/sm/array_schema/attribute.h" #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/array_schema/domain.h" +#include "tiledb/sm/array_schema/enumeration.h" #include "tiledb/sm/enums/array_type.h" #include "tiledb/sm/enums/compressor.h" #include "tiledb/sm/enums/datatype.h" @@ -102,6 +103,32 @@ Status array_schema_evolution_to_capnp( attribute_to_capnp(attr_to_add, &attribute_builder); } + auto enmr_names_to_add = array_schema_evolution->enumeration_names_to_add(); + auto num_enmrs = enmr_names_to_add.size(); + + if (num_enmrs > 0) { + auto enmrs_to_add_builder = + array_schema_evolution_builder->initEnumerationsToAdd(num_enmrs); + for (size_t i = 0; i < num_enmrs; i++) { + auto enmr = + array_schema_evolution->enumeration_to_add(enmr_names_to_add[i]); + auto builder = enmrs_to_add_builder[i]; + enumeration_to_capnp(enmr, builder); + } + } + + // Enumerations to drop + std::vector enmr_names_to_drop = + array_schema_evolution->enumeration_names_to_drop(); + + auto enumerations_to_drop_builder = + array_schema_evolution_builder->initEnumerationsToDrop( + enmr_names_to_drop.size()); + + for (size_t i = 0; i < enmr_names_to_drop.size(); i++) { + enumerations_to_drop_builder.set(i, enmr_names_to_drop[i]); + } + auto timestamp_builder = array_schema_evolution_builder->initTimestampRange(2); const auto& timestamp_range = array_schema_evolution->timestamp_range(); @@ -111,34 +138,55 @@ Status array_schema_evolution_to_capnp( return Status::Ok(); } -Status array_schema_evolution_from_capnp( - const capnp::ArraySchemaEvolution::Reader& evolution_reader, - tdb_unique_ptr* array_schema_evolution) { - array_schema_evolution->reset(tdb_new(ArraySchemaEvolution)); - // Set attributes to drop +tdb_unique_ptr array_schema_evolution_from_capnp( + const capnp::ArraySchemaEvolution::Reader& evolution_reader) { + // Create attributes to add + std::unordered_map> attrs_to_add; + auto attrs_to_add_reader = evolution_reader.getAttributesToAdd(); + for (auto attr_reader : attrs_to_add_reader) { + auto attr = attribute_from_capnp(attr_reader); + attrs_to_add[attr->name()] = attr; + } + + // Create attributes to drop + std::unordered_set attrs_to_drop; auto attributes_to_drop_reader = evolution_reader.getAttributesToDrop(); for (auto attr_reader : attributes_to_drop_reader) { std::string attr_name = std::string(attr_reader.cStr()); - RETURN_NOT_OK((*array_schema_evolution)->drop_attribute(attr_name)); + attrs_to_drop.insert(attr_name); } - // Set attributes to add - auto attributes_to_add_reader = evolution_reader.getAttributesToAdd(); - for (auto attr_reader : attributes_to_add_reader) { - auto attr = attribute_from_capnp(attr_reader); - RETURN_NOT_OK((*array_schema_evolution)->add_attribute(attr.get())); + // Create enumerations to add + std::unordered_map> enmrs_to_add; + auto enmrs_to_add_reader = evolution_reader.getEnumerationsToAdd(); + for (auto enmr_reader : enmrs_to_add_reader) { + auto enmr = enumeration_from_capnp(enmr_reader); + enmrs_to_add[enmr->name()] = enmr; } + // Create enumerations to drop + std::unordered_set enmrs_to_drop; + auto enmrs_to_drop_reader = evolution_reader.getEnumerationsToDrop(); + for (auto enmr_reader : enmrs_to_drop_reader) { + std::string enmr_name = std::string(enmr_reader.cStr()); + enmrs_to_drop.insert(enmr_name); + } + + std::pair ts_range; // Set the range if we have two values if (evolution_reader.hasTimestampRange() && evolution_reader.getTimestampRange().size() >= 2) { const auto& timestamp_range = evolution_reader.getTimestampRange(); - throw_if_not_ok((*array_schema_evolution) - ->set_timestamp_range(std::make_pair( - timestamp_range[0], timestamp_range[1]))); + ts_range = std::make_pair(timestamp_range[0], timestamp_range[1]); } - return Status::Ok(); + return tdb_unique_ptr(tdb_new( + ArraySchemaEvolution, + attrs_to_add, + attrs_to_drop, + enmrs_to_add, + enmrs_to_drop, + ts_range)); } Status array_schema_evolution_serialize( @@ -216,8 +264,8 @@ Status array_schema_evolution_deserialize( array_schema_evolution_builder); capnp::ArraySchemaEvolution::Reader array_schema_evolution_reader = array_schema_evolution_builder.asReader(); - RETURN_NOT_OK(array_schema_evolution_from_capnp( - array_schema_evolution_reader, &decoded_array_schema_evolution)); + decoded_array_schema_evolution = + array_schema_evolution_from_capnp(array_schema_evolution_reader); break; } case SerializationType::CAPNP: { @@ -228,8 +276,8 @@ Status array_schema_evolution_deserialize( serialized_buffer.size() / sizeof(::capnp::word))); capnp::ArraySchemaEvolution::Reader array_schema_evolution_reader = reader.getRoot(); - RETURN_NOT_OK(array_schema_evolution_from_capnp( - array_schema_evolution_reader, &decoded_array_schema_evolution)); + decoded_array_schema_evolution = + array_schema_evolution_from_capnp(array_schema_evolution_reader); break; } default: { diff --git a/tiledb/sm/serialization/posix/tiledb-rest.capnp.c++ b/tiledb/sm/serialization/posix/tiledb-rest.capnp.c++ index 08df30190b05..86e22aaddac3 100644 --- a/tiledb/sm/serialization/posix/tiledb-rest.capnp.c++ +++ b/tiledb/sm/serialization/posix/tiledb-rest.capnp.c++ @@ -635,17 +635,17 @@ const ::capnp::_::RawSchema s_facceeafd4472c68 = { 1, 2, i_facceeafd4472c68, nullptr, nullptr, { &s_facceeafd4472c68, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<274> b_d71de32f98e296fe = { +static const ::capnp::_::AlignedData<315> b_d71de32f98e296fe = { { 0, 0, 0, 0, 5, 0, 6, 0, 254, 150, 226, 152, 47, 227, 29, 215, 18, 0, 0, 0, 1, 0, 2, 0, 127, 216, 135, 181, 36, 146, 125, 181, - 13, 0, 7, 0, 0, 0, 0, 0, + 15, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 242, 0, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 0, 0, 0, 79, 3, 0, 0, + 29, 0, 0, 0, 191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, @@ -653,112 +653,126 @@ static const ::capnp::_::AlignedData<274> b_d71de32f98e296fe = { 112, 58, 65, 114, 114, 97, 121, 83, 99, 104, 101, 109, 97, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, - 60, 0, 0, 0, 3, 0, 4, 0, + 68, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 149, 1, 0, 0, 82, 0, 0, 0, + 205, 1, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 148, 1, 0, 0, 3, 0, 1, 0, - 160, 1, 0, 0, 2, 0, 1, 0, + 204, 1, 0, 0, 3, 0, 1, 0, + 216, 1, 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 157, 1, 0, 0, 90, 0, 0, 0, + 213, 1, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 156, 1, 0, 0, 3, 0, 1, 0, - 184, 1, 0, 0, 2, 0, 1, 0, + 212, 1, 0, 0, 3, 0, 1, 0, + 240, 1, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 181, 1, 0, 0, 74, 0, 0, 0, + 237, 1, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 180, 1, 0, 0, 3, 0, 1, 0, - 192, 1, 0, 0, 2, 0, 1, 0, + 236, 1, 0, 0, 3, 0, 1, 0, + 248, 1, 0, 0, 2, 0, 1, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 189, 1, 0, 0, 82, 0, 0, 0, + 245, 1, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 188, 1, 0, 0, 3, 0, 1, 0, - 200, 1, 0, 0, 2, 0, 1, 0, + 244, 1, 0, 0, 3, 0, 1, 0, + 0, 2, 0, 0, 2, 0, 1, 0, 4, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 197, 1, 0, 0, 170, 0, 0, 0, + 253, 1, 0, 0, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 200, 1, 0, 0, 3, 0, 1, 0, - 212, 1, 0, 0, 2, 0, 1, 0, + 0, 2, 0, 0, 3, 0, 1, 0, + 12, 2, 0, 0, 2, 0, 1, 0, 5, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 209, 1, 0, 0, 58, 0, 0, 0, + 9, 2, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 204, 1, 0, 0, 3, 0, 1, 0, - 216, 1, 0, 0, 2, 0, 1, 0, + 4, 2, 0, 0, 3, 0, 1, 0, + 16, 2, 0, 0, 2, 0, 1, 0, 6, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 213, 1, 0, 0, 170, 0, 0, 0, + 13, 2, 0, 0, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 216, 1, 0, 0, 3, 0, 1, 0, - 228, 1, 0, 0, 2, 0, 1, 0, + 16, 2, 0, 0, 3, 0, 1, 0, + 28, 2, 0, 0, 2, 0, 1, 0, 7, 0, 0, 0, 6, 0, 0, 0, 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 225, 1, 0, 0, 82, 0, 0, 0, + 25, 2, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 224, 1, 0, 0, 3, 0, 1, 0, - 236, 1, 0, 0, 2, 0, 1, 0, + 24, 2, 0, 0, 3, 0, 1, 0, + 36, 2, 0, 0, 2, 0, 1, 0, 8, 0, 0, 0, 7, 0, 0, 0, 0, 0, 1, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 233, 1, 0, 0, 34, 0, 0, 0, + 33, 2, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 228, 1, 0, 0, 3, 0, 1, 0, - 240, 1, 0, 0, 2, 0, 1, 0, + 28, 2, 0, 0, 3, 0, 1, 0, + 40, 2, 0, 0, 2, 0, 1, 0, 9, 0, 0, 0, 8, 0, 0, 0, 0, 0, 1, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 237, 1, 0, 0, 66, 0, 0, 0, + 37, 2, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 232, 1, 0, 0, 3, 0, 1, 0, - 4, 2, 0, 0, 2, 0, 1, 0, + 32, 2, 0, 0, 3, 0, 1, 0, + 60, 2, 0, 0, 2, 0, 1, 0, 10, 0, 0, 0, 64, 0, 0, 0, 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 2, 0, 0, 138, 0, 0, 0, + 57, 2, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 4, 2, 0, 0, 3, 0, 1, 0, - 16, 2, 0, 0, 2, 0, 1, 0, + 60, 2, 0, 0, 3, 0, 1, 0, + 72, 2, 0, 0, 2, 0, 1, 0, 11, 0, 0, 0, 9, 0, 0, 0, 0, 0, 1, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 13, 2, 0, 0, 186, 0, 0, 0, + 69, 2, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 16, 2, 0, 0, 3, 0, 1, 0, - 28, 2, 0, 0, 2, 0, 1, 0, + 72, 2, 0, 0, 3, 0, 1, 0, + 84, 2, 0, 0, 2, 0, 1, 0, 12, 0, 0, 0, 10, 0, 0, 0, 0, 0, 1, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 25, 2, 0, 0, 42, 0, 0, 0, + 81, 2, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 20, 2, 0, 0, 3, 0, 1, 0, - 32, 2, 0, 0, 2, 0, 1, 0, + 76, 2, 0, 0, 3, 0, 1, 0, + 88, 2, 0, 0, 2, 0, 1, 0, 13, 0, 0, 0, 11, 0, 0, 0, 0, 0, 1, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 2, 0, 0, 122, 0, 0, 0, + 85, 2, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 28, 2, 0, 0, 3, 0, 1, 0, - 56, 2, 0, 0, 2, 0, 1, 0, + 84, 2, 0, 0, 3, 0, 1, 0, + 112, 2, 0, 0, 2, 0, 1, 0, 14, 0, 0, 0, 12, 0, 0, 0, 0, 0, 1, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 53, 2, 0, 0, 130, 0, 0, 0, + 109, 2, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 52, 2, 0, 0, 3, 0, 1, 0, - 80, 2, 0, 0, 2, 0, 1, 0, + 108, 2, 0, 0, 3, 0, 1, 0, + 136, 2, 0, 0, 2, 0, 1, 0, + 15, 0, 0, 0, 13, 0, 0, 0, + 0, 0, 1, 0, 15, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 133, 2, 0, 0, 106, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 132, 2, 0, 0, 3, 0, 1, 0, + 160, 2, 0, 0, 2, 0, 1, 0, + 16, 0, 0, 0, 14, 0, 0, 0, + 0, 0, 1, 0, 16, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 157, 2, 0, 0, 154, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 160, 2, 0, 0, 3, 0, 1, 0, + 188, 2, 0, 0, 2, 0, 1, 0, 97, 114, 114, 97, 121, 84, 121, 112, 101, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, @@ -907,6 +921,33 @@ static const ::capnp::_::AlignedData<274> b_d71de32f98e296fe = { 222, 209, 12, 209, 98, 141, 255, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 110, 117, 109, 101, 114, 97, 116, + 105, 111, 110, 115, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 180, 185, 33, 204, 25, 47, 11, 208, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 110, 117, 109, 101, 114, 97, 116, + 105, 111, 110, 80, 97, 116, 104, 77, + 97, 112, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 151, 188, 17, 242, 43, 223, 218, 227, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, } @@ -917,13 +958,15 @@ static const ::capnp::_::RawSchema* const d_d71de32f98e296fe[] = { &s_92ad78f56de3d76a, &s_bc4583f733eac4f5, &s_ceff8d62d10cd1de, + &s_d00b2f19cc21b9b4, &s_de030f447664754c, + &s_e3dadf2bf211bc97, }; -static const uint16_t m_d71de32f98e296fe[] = {10, 0, 1, 2, 3, 4, 14, 5, 12, 6, 7, 13, 8, 11, 9}; -static const uint16_t i_d71de32f98e296fe[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}; +static const uint16_t m_d71de32f98e296fe[] = {10, 0, 1, 2, 3, 4, 14, 5, 16, 15, 12, 6, 7, 13, 8, 11, 9}; +static const uint16_t i_d71de32f98e296fe[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; const ::capnp::_::RawSchema s_d71de32f98e296fe = { - 0xd71de32f98e296fe, b_d71de32f98e296fe.words, 274, d_d71de32f98e296fe, m_d71de32f98e296fe, - 4, 15, i_d71de32f98e296fe, nullptr, nullptr, { &s_d71de32f98e296fe, nullptr, nullptr, 0, 0, nullptr } + 0xd71de32f98e296fe, b_d71de32f98e296fe.words, 315, d_d71de32f98e296fe, m_d71de32f98e296fe, + 6, 17, i_d71de32f98e296fe, nullptr, nullptr, { &s_d71de32f98e296fe, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE static const ::capnp::_::AlignedData<174> b_ceff8d62d10cd1de = { @@ -1114,17 +1157,17 @@ const ::capnp::_::RawSchema s_ceff8d62d10cd1de = { 1, 10, i_ceff8d62d10cd1de, nullptr, nullptr, { &s_ceff8d62d10cd1de, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<80> b_a1b81d67548230d4 = { +static const ::capnp::_::AlignedData<122> b_a1b81d67548230d4 = { { 0, 0, 0, 0, 5, 0, 6, 0, 212, 48, 130, 84, 103, 29, 184, 161, 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, - 3, 0, 7, 0, 0, 0, 0, 0, + 5, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 58, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 0, 0, 0, 175, 0, 0, 0, + 33, 0, 0, 0, 31, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, @@ -1133,28 +1176,42 @@ static const ::capnp::_::AlignedData<80> b_a1b81d67548230d4 = { 99, 104, 101, 109, 97, 69, 118, 111, 108, 117, 116, 105, 111, 110, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, - 12, 0, 0, 0, 3, 0, 4, 0, + 20, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 69, 0, 0, 0, 138, 0, 0, 0, + 125, 0, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 72, 0, 0, 0, 3, 0, 1, 0, - 100, 0, 0, 0, 2, 0, 1, 0, + 128, 0, 0, 0, 3, 0, 1, 0, + 156, 0, 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 97, 0, 0, 0, 130, 0, 0, 0, + 153, 0, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 96, 0, 0, 0, 3, 0, 1, 0, - 124, 0, 0, 0, 2, 0, 1, 0, + 152, 0, 0, 0, 3, 0, 1, 0, + 180, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 121, 0, 0, 0, 122, 0, 0, 0, + 177, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 120, 0, 0, 0, 3, 0, 1, 0, - 148, 0, 0, 0, 2, 0, 1, 0, + 176, 0, 0, 0, 3, 0, 1, 0, + 204, 0, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 201, 0, 0, 0, 146, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 204, 0, 0, 0, 3, 0, 1, 0, + 232, 0, 0, 0, 2, 0, 1, 0, + 4, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 229, 0, 0, 0, 154, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 232, 0, 0, 0, 3, 0, 1, 0, + 4, 1, 0, 0, 2, 0, 1, 0, 97, 116, 116, 114, 105, 98, 117, 116, 101, 115, 84, 111, 68, 114, 111, 112, 0, 0, 0, 0, 0, 0, 0, 0, @@ -1192,6 +1249,34 @@ static const ::capnp::_::AlignedData<80> b_a1b81d67548230d4 = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 110, 117, 109, 101, 114, 97, 116, + 105, 111, 110, 115, 84, 111, 65, 100, + 100, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 180, 185, 33, 204, 25, 47, 11, 208, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 110, 117, 109, 101, 114, 97, 116, + 105, 111, 110, 115, 84, 111, 68, 114, + 111, 112, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, } @@ -1200,25 +1285,26 @@ static const ::capnp::_::AlignedData<80> b_a1b81d67548230d4 = { #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_a1b81d67548230d4[] = { &s_92ad78f56de3d76a, + &s_d00b2f19cc21b9b4, }; -static const uint16_t m_a1b81d67548230d4[] = {1, 0, 2}; -static const uint16_t i_a1b81d67548230d4[] = {0, 1, 2}; +static const uint16_t m_a1b81d67548230d4[] = {1, 0, 3, 4, 2}; +static const uint16_t i_a1b81d67548230d4[] = {0, 1, 2, 3, 4}; const ::capnp::_::RawSchema s_a1b81d67548230d4 = { - 0xa1b81d67548230d4, b_a1b81d67548230d4.words, 80, d_a1b81d67548230d4, m_a1b81d67548230d4, - 1, 3, i_a1b81d67548230d4, nullptr, nullptr, { &s_a1b81d67548230d4, nullptr, nullptr, 0, 0, nullptr } + 0xa1b81d67548230d4, b_a1b81d67548230d4.words, 122, d_a1b81d67548230d4, m_a1b81d67548230d4, + 2, 5, i_a1b81d67548230d4, nullptr, nullptr, { &s_a1b81d67548230d4, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<144> b_92ad78f56de3d76a = { +static const ::capnp::_::AlignedData<160> b_92ad78f56de3d76a = { { 0, 0, 0, 0, 5, 0, 6, 0, 106, 215, 227, 109, 245, 120, 173, 146, 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, - 5, 0, 7, 0, 0, 0, 0, 0, + 6, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 226, 0, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 0, 0, 0, 199, 1, 0, 0, + 29, 0, 0, 0, 255, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, @@ -1226,63 +1312,70 @@ static const ::capnp::_::AlignedData<144> b_92ad78f56de3d76a = { 112, 58, 65, 116, 116, 114, 105, 98, 117, 116, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, - 32, 0, 0, 0, 3, 0, 4, 0, + 36, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 209, 0, 0, 0, 90, 0, 0, 0, + 237, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 208, 0, 0, 0, 3, 0, 1, 0, - 220, 0, 0, 0, 2, 0, 1, 0, + 236, 0, 0, 0, 3, 0, 1, 0, + 248, 0, 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 217, 0, 0, 0, 42, 0, 0, 0, + 245, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 212, 0, 0, 0, 3, 0, 1, 0, - 224, 0, 0, 0, 2, 0, 1, 0, + 240, 0, 0, 0, 3, 0, 1, 0, + 252, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 221, 0, 0, 0, 42, 0, 0, 0, + 249, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 216, 0, 0, 0, 3, 0, 1, 0, - 228, 0, 0, 0, 2, 0, 1, 0, + 244, 0, 0, 0, 3, 0, 1, 0, + 0, 1, 0, 0, 2, 0, 1, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 225, 0, 0, 0, 122, 0, 0, 0, + 253, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 224, 0, 0, 0, 3, 0, 1, 0, - 236, 0, 0, 0, 2, 0, 1, 0, + 252, 0, 0, 0, 3, 0, 1, 0, + 8, 1, 0, 0, 2, 0, 1, 0, 4, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 233, 0, 0, 0, 82, 0, 0, 0, + 5, 1, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 232, 0, 0, 0, 3, 0, 1, 0, - 244, 0, 0, 0, 2, 0, 1, 0, + 4, 1, 0, 0, 3, 0, 1, 0, + 16, 1, 0, 0, 2, 0, 1, 0, 5, 0, 0, 0, 32, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 241, 0, 0, 0, 74, 0, 0, 0, + 13, 1, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 240, 0, 0, 0, 3, 0, 1, 0, - 252, 0, 0, 0, 2, 0, 1, 0, + 12, 1, 0, 0, 3, 0, 1, 0, + 24, 1, 0, 0, 2, 0, 1, 0, 6, 0, 0, 0, 33, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 249, 0, 0, 0, 146, 0, 0, 0, + 21, 1, 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 252, 0, 0, 0, 3, 0, 1, 0, - 8, 1, 0, 0, 2, 0, 1, 0, + 24, 1, 0, 0, 3, 0, 1, 0, + 36, 1, 0, 0, 2, 0, 1, 0, 7, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 5, 1, 0, 0, 50, 0, 0, 0, + 33, 1, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1, 0, 0, 3, 0, 1, 0, - 12, 1, 0, 0, 2, 0, 1, 0, + 28, 1, 0, 0, 3, 0, 1, 0, + 40, 1, 0, 0, 2, 0, 1, 0, + 8, 0, 0, 0, 5, 0, 0, 0, + 0, 0, 1, 0, 8, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 37, 1, 0, 0, 130, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 36, 1, 0, 0, 3, 0, 1, 0, + 48, 1, 0, 0, 2, 0, 1, 0, 99, 101, 108, 108, 86, 97, 108, 78, 117, 109, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, @@ -1346,6 +1439,15 @@ static const ::capnp::_::AlignedData<144> b_92ad78f56de3d76a = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 114, 100, 101, 114, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 110, 117, 109, 101, 114, 97, 116, + 105, 111, 110, 78, 97, 109, 101, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -1359,11 +1461,147 @@ static const ::capnp::_::AlignedData<144> b_92ad78f56de3d76a = { static const ::capnp::_::RawSchema* const d_92ad78f56de3d76a[] = { &s_bc4583f733eac4f5, }; -static const uint16_t m_92ad78f56de3d76a[] = {0, 4, 6, 3, 1, 5, 7, 2}; -static const uint16_t i_92ad78f56de3d76a[] = {0, 1, 2, 3, 4, 5, 6, 7}; +static const uint16_t m_92ad78f56de3d76a[] = {0, 8, 4, 6, 3, 1, 5, 7, 2}; +static const uint16_t i_92ad78f56de3d76a[] = {0, 1, 2, 3, 4, 5, 6, 7, 8}; const ::capnp::_::RawSchema s_92ad78f56de3d76a = { - 0x92ad78f56de3d76a, b_92ad78f56de3d76a.words, 144, d_92ad78f56de3d76a, m_92ad78f56de3d76a, - 1, 8, i_92ad78f56de3d76a, nullptr, nullptr, { &s_92ad78f56de3d76a, nullptr, nullptr, 0, 0, nullptr } + 0x92ad78f56de3d76a, b_92ad78f56de3d76a.words, 160, d_92ad78f56de3d76a, m_92ad78f56de3d76a, + 1, 9, i_92ad78f56de3d76a, nullptr, nullptr, { &s_92ad78f56de3d76a, nullptr, nullptr, 0, 0, nullptr } +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<125> b_d00b2f19cc21b9b4 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 180, 185, 33, 204, 25, 47, 11, 208, + 18, 0, 0, 0, 1, 0, 1, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 5, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 242, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 29, 0, 0, 0, 143, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 69, 110, 117, 109, 101, 114, + 97, 116, 105, 111, 110, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 28, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 181, 0, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 176, 0, 0, 0, 3, 0, 1, 0, + 188, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 185, 0, 0, 0, 74, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 184, 0, 0, 0, 3, 0, 1, 0, + 196, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 193, 0, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 188, 0, 0, 0, 3, 0, 1, 0, + 200, 0, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 197, 0, 0, 0, 90, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 196, 0, 0, 0, 3, 0, 1, 0, + 208, 0, 0, 0, 2, 0, 1, 0, + 4, 0, 0, 0, 32, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 205, 0, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 200, 0, 0, 0, 3, 0, 1, 0, + 212, 0, 0, 0, 2, 0, 1, 0, + 5, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 5, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 209, 0, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 204, 0, 0, 0, 3, 0, 1, 0, + 216, 0, 0, 0, 2, 0, 1, 0, + 6, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 6, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 213, 0, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 208, 0, 0, 0, 3, 0, 1, 0, + 220, 0, 0, 0, 2, 0, 1, 0, + 110, 97, 109, 101, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 112, 97, 116, 104, 78, 97, 109, 101, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 121, 112, 101, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 101, 108, 108, 86, 97, 108, 78, + 117, 109, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 111, 114, 100, 101, 114, 101, 100, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 100, 97, 116, 97, 0, 0, 0, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 111, 102, 102, 115, 101, 116, 115, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; +::capnp::word const* const bp_d00b2f19cc21b9b4 = b_d00b2f19cc21b9b4.words; +#if !CAPNP_LITE +static const uint16_t m_d00b2f19cc21b9b4[] = {3, 5, 0, 6, 4, 1, 2}; +static const uint16_t i_d00b2f19cc21b9b4[] = {0, 1, 2, 3, 4, 5, 6}; +const ::capnp::_::RawSchema s_d00b2f19cc21b9b4 = { + 0xd00b2f19cc21b9b4, b_d00b2f19cc21b9b4.words, 125, nullptr, m_d00b2f19cc21b9b4, + 0, 7, i_d00b2f19cc21b9b4, nullptr, nullptr, { &s_d00b2f19cc21b9b4, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE static const ::capnp::_::AlignedData<143> b_d20a578112fa92a2 = { @@ -4271,17 +4509,17 @@ const ::capnp::_::RawSchema s_cbe1e7c13508aa2c = { 1, 4, i_cbe1e7c13508aa2c, nullptr, nullptr, { &s_cbe1e7c13508aa2c, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<65> b_dac6a7f675c57409 = { +static const ::capnp::_::AlignedData<81> b_dac6a7f675c57409 = { { 0, 0, 0, 0, 5, 0, 6, 0, 9, 116, 197, 117, 246, 167, 198, 218, - 18, 0, 0, 0, 1, 0, 0, 0, + 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 18, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 0, 0, 0, 175, 0, 0, 0, + 33, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, @@ -4290,28 +4528,35 @@ static const ::capnp::_::AlignedData<65> b_dac6a7f675c57409 = { 105, 111, 110, 67, 108, 97, 117, 115, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, - 12, 0, 0, 0, 3, 0, 4, 0, + 16, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 69, 0, 0, 0, 82, 0, 0, 0, + 97, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 68, 0, 0, 0, 3, 0, 1, 0, - 80, 0, 0, 0, 2, 0, 1, 0, + 96, 0, 0, 0, 3, 0, 1, 0, + 108, 0, 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 77, 0, 0, 0, 50, 0, 0, 0, + 105, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 72, 0, 0, 0, 3, 0, 1, 0, - 84, 0, 0, 0, 2, 0, 1, 0, + 100, 0, 0, 0, 3, 0, 1, 0, + 112, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 81, 0, 0, 0, 26, 0, 0, 0, + 109, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 76, 0, 0, 0, 3, 0, 1, 0, - 88, 0, 0, 0, 2, 0, 1, 0, + 104, 0, 0, 0, 3, 0, 1, 0, + 116, 0, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 113, 0, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 112, 0, 0, 0, 3, 0, 1, 0, + 124, 0, 0, 0, 2, 0, 1, 0, 102, 105, 101, 108, 100, 78, 97, 109, 101, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, @@ -4335,19 +4580,28 @@ static const ::capnp::_::AlignedData<65> b_dac6a7f675c57409 = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 117, 115, 101, 69, 110, 117, 109, 101, + 114, 97, 116, 105, 111, 110, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, } }; ::capnp::word const* const bp_dac6a7f675c57409 = b_dac6a7f675c57409.words; #if !CAPNP_LITE -static const uint16_t m_dac6a7f675c57409[] = {0, 2, 1}; -static const uint16_t i_dac6a7f675c57409[] = {0, 1, 2}; +static const uint16_t m_dac6a7f675c57409[] = {0, 2, 3, 1}; +static const uint16_t i_dac6a7f675c57409[] = {0, 1, 2, 3}; const ::capnp::_::RawSchema s_dac6a7f675c57409 = { - 0xdac6a7f675c57409, b_dac6a7f675c57409.words, 65, nullptr, m_dac6a7f675c57409, - 0, 3, i_dac6a7f675c57409, nullptr, nullptr, { &s_dac6a7f675c57409, nullptr, nullptr, 0, 0, nullptr } + 0xdac6a7f675c57409, b_dac6a7f675c57409.words, 81, nullptr, m_dac6a7f675c57409, + 0, 4, i_dac6a7f675c57409, nullptr, nullptr, { &s_dac6a7f675c57409, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<116> b_afc739d5c01e6496 = { +static const ::capnp::_::AlignedData<132> b_afc739d5c01e6496 = { { 0, 0, 0, 0, 5, 0, 6, 0, 150, 100, 30, 192, 213, 57, 199, 175, 18, 0, 0, 0, 1, 0, 1, 0, @@ -4357,7 +4611,7 @@ static const ::capnp::_::AlignedData<116> b_afc739d5c01e6496 = { 21, 0, 0, 0, 210, 0, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 0, 0, 0, 87, 1, 0, 0, + 29, 0, 0, 0, 143, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, @@ -4365,49 +4619,56 @@ static const ::capnp::_::AlignedData<116> b_afc739d5c01e6496 = { 112, 58, 65, 83, 84, 78, 111, 100, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, - 24, 0, 0, 0, 3, 0, 4, 0, + 28, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 153, 0, 0, 0, 106, 0, 0, 0, + 181, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 152, 0, 0, 0, 3, 0, 1, 0, - 164, 0, 0, 0, 2, 0, 1, 0, + 180, 0, 0, 0, 3, 0, 1, 0, + 192, 0, 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 161, 0, 0, 0, 82, 0, 0, 0, + 189, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 160, 0, 0, 0, 3, 0, 1, 0, - 172, 0, 0, 0, 2, 0, 1, 0, + 188, 0, 0, 0, 3, 0, 1, 0, + 200, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 169, 0, 0, 0, 50, 0, 0, 0, + 197, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 164, 0, 0, 0, 3, 0, 1, 0, - 176, 0, 0, 0, 2, 0, 1, 0, + 192, 0, 0, 0, 3, 0, 1, 0, + 204, 0, 0, 0, 2, 0, 1, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 173, 0, 0, 0, 26, 0, 0, 0, + 201, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 168, 0, 0, 0, 3, 0, 1, 0, - 180, 0, 0, 0, 2, 0, 1, 0, + 196, 0, 0, 0, 3, 0, 1, 0, + 208, 0, 0, 0, 2, 0, 1, 0, 4, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 177, 0, 0, 0, 74, 0, 0, 0, + 205, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 176, 0, 0, 0, 3, 0, 1, 0, - 204, 0, 0, 0, 2, 0, 1, 0, + 204, 0, 0, 0, 3, 0, 1, 0, + 232, 0, 0, 0, 2, 0, 1, 0, 5, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 201, 0, 0, 0, 114, 0, 0, 0, + 229, 0, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 200, 0, 0, 0, 3, 0, 1, 0, - 212, 0, 0, 0, 2, 0, 1, 0, + 228, 0, 0, 0, 3, 0, 1, 0, + 240, 0, 0, 0, 2, 0, 1, 0, + 6, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 6, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 237, 0, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 236, 0, 0, 0, 3, 0, 1, 0, + 248, 0, 0, 0, 2, 0, 1, 0, 105, 115, 69, 120, 112, 114, 101, 115, 115, 105, 111, 110, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, @@ -4462,6 +4723,15 @@ static const ::capnp::_::AlignedData<116> b_afc739d5c01e6496 = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 117, 115, 101, 69, 110, 117, 109, 101, + 114, 97, 116, 105, 111, 110, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, } }; @@ -4470,11 +4740,11 @@ static const ::capnp::_::AlignedData<116> b_afc739d5c01e6496 = { static const ::capnp::_::RawSchema* const d_afc739d5c01e6496[] = { &s_afc739d5c01e6496, }; -static const uint16_t m_afc739d5c01e6496[] = {4, 5, 1, 0, 3, 2}; -static const uint16_t i_afc739d5c01e6496[] = {0, 1, 2, 3, 4, 5}; +static const uint16_t m_afc739d5c01e6496[] = {4, 5, 1, 0, 3, 6, 2}; +static const uint16_t i_afc739d5c01e6496[] = {0, 1, 2, 3, 4, 5, 6}; const ::capnp::_::RawSchema s_afc739d5c01e6496 = { - 0xafc739d5c01e6496, b_afc739d5c01e6496.words, 116, d_afc739d5c01e6496, m_afc739d5c01e6496, - 1, 6, i_afc739d5c01e6496, nullptr, nullptr, { &s_afc739d5c01e6496, nullptr, nullptr, 0, 0, nullptr } + 0xafc739d5c01e6496, b_afc739d5c01e6496.words, 132, d_afc739d5c01e6496, m_afc739d5c01e6496, + 1, 7, i_afc739d5c01e6496, nullptr, nullptr, { &s_afc739d5c01e6496, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE static const ::capnp::_::AlignedData<73> b_eaf57cb9871fc06f = { @@ -9010,6 +9280,14 @@ constexpr ::capnp::Kind Attribute::_capnpPrivate::kind; constexpr ::capnp::_::RawSchema const* Attribute::_capnpPrivate::schema; #endif // !CAPNP_LITE +// Enumeration +constexpr uint16_t Enumeration::_capnpPrivate::dataWordSize; +constexpr uint16_t Enumeration::_capnpPrivate::pointerCount; +#if !CAPNP_LITE +constexpr ::capnp::Kind Enumeration::_capnpPrivate::kind; +constexpr ::capnp::_::RawSchema const* Enumeration::_capnpPrivate::schema; +#endif // !CAPNP_LITE + // AttributeBufferHeader constexpr uint16_t AttributeBufferHeader::_capnpPrivate::dataWordSize; constexpr uint16_t AttributeBufferHeader::_capnpPrivate::pointerCount; diff --git a/tiledb/sm/serialization/posix/tiledb-rest.capnp.h b/tiledb/sm/serialization/posix/tiledb-rest.capnp.h index 260cbaa58fb9..05ccbc0a6b37 100644 --- a/tiledb/sm/serialization/posix/tiledb-rest.capnp.h +++ b/tiledb/sm/serialization/posix/tiledb-rest.capnp.h @@ -23,6 +23,7 @@ CAPNP_DECLARE_SCHEMA(d71de32f98e296fe); CAPNP_DECLARE_SCHEMA(ceff8d62d10cd1de); CAPNP_DECLARE_SCHEMA(a1b81d67548230d4); CAPNP_DECLARE_SCHEMA(92ad78f56de3d76a); +CAPNP_DECLARE_SCHEMA(d00b2f19cc21b9b4); CAPNP_DECLARE_SCHEMA(d20a578112fa92a2); CAPNP_DECLARE_SCHEMA(95e26a84d32d8223); CAPNP_DECLARE_SCHEMA(a2a652536db09fa0); @@ -196,7 +197,7 @@ struct ArraySchema { class Pipeline; struct _capnpPrivate { - CAPNP_DECLARE_STRUCT_HEADER(d71de32f98e296fe, 2, 13) + CAPNP_DECLARE_STRUCT_HEADER(d71de32f98e296fe, 2, 15) #if !CAPNP_LITE static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; @@ -230,7 +231,7 @@ struct ArraySchemaEvolution { class Pipeline; struct _capnpPrivate { - CAPNP_DECLARE_STRUCT_HEADER(a1b81d67548230d4, 0, 3) + CAPNP_DECLARE_STRUCT_HEADER(a1b81d67548230d4, 0, 5) #if !CAPNP_LITE static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; @@ -247,7 +248,24 @@ struct Attribute { class Pipeline; struct _capnpPrivate { - CAPNP_DECLARE_STRUCT_HEADER(92ad78f56de3d76a, 1, 5) + CAPNP_DECLARE_STRUCT_HEADER(92ad78f56de3d76a, 1, 6) +#if !CAPNP_LITE + static constexpr ::capnp::_::RawBrandedSchema const* brand() { + return &schema->defaultBrand; + } +#endif // !CAPNP_LITE + }; +}; + +struct Enumeration { + Enumeration() = delete; + + class Reader; + class Builder; + class Pipeline; + + struct _capnpPrivate { + CAPNP_DECLARE_STRUCT_HEADER(d00b2f19cc21b9b4, 1, 5) #if !CAPNP_LITE static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; @@ -820,7 +838,7 @@ struct ConditionClause { class Pipeline; struct _capnpPrivate { - CAPNP_DECLARE_STRUCT_HEADER(dac6a7f675c57409, 0, 3) + CAPNP_DECLARE_STRUCT_HEADER(dac6a7f675c57409, 1, 3) #if !CAPNP_LITE static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; @@ -2441,6 +2459,18 @@ class ArraySchema::Reader { ::capnp::Kind::STRUCT>::Reader getDimensionLabels() const; + inline bool hasEnumerations() const; + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader + getEnumerations() const; + + inline bool hasEnumerationPathMap() const; + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>::Reader + getEnumerationPathMap() const; + private: ::capnp::_::StructReader _reader; template @@ -2640,6 +2670,48 @@ class ArraySchema::Builder { ::capnp::Kind::STRUCT>> disownDimensionLabels(); + inline bool hasEnumerations(); + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder + getEnumerations(); + inline void setEnumerations(::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader value); + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder + initEnumerations(unsigned int size); + inline void adoptEnumerations( + ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>&& value); + inline ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>> + disownEnumerations(); + + inline bool hasEnumerationPathMap(); + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>::Builder + getEnumerationPathMap(); + inline void setEnumerationPathMap(::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>::Reader value); + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>::Builder + initEnumerationPathMap(unsigned int size); + inline void adoptEnumerationPathMap( + ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>>&& value); + inline ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>> + disownEnumerationPathMap(); + private: ::capnp::_::StructBuilder _builder; template @@ -2883,6 +2955,16 @@ class ArraySchemaEvolution::Reader { inline ::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>::Reader getTimestampRange() const; + inline bool hasEnumerationsToAdd() const; + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader + getEnumerationsToAdd() const; + + inline bool hasEnumerationsToDrop() const; + inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Reader + getEnumerationsToDrop() const; + private: ::capnp::_::StructReader _reader; template @@ -2972,6 +3054,43 @@ class ArraySchemaEvolution::Builder { inline ::capnp::Orphan<::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>> disownTimestampRange(); + inline bool hasEnumerationsToAdd(); + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder + getEnumerationsToAdd(); + inline void setEnumerationsToAdd( + ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader value); + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder + initEnumerationsToAdd(unsigned int size); + inline void adoptEnumerationsToAdd( + ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>&& value); + inline ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>> + disownEnumerationsToAdd(); + + inline bool hasEnumerationsToDrop(); + inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Builder + getEnumerationsToDrop(); + inline void setEnumerationsToDrop( + ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Reader value); + inline void setEnumerationsToDrop( + ::kj::ArrayPtr value); + inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Builder + initEnumerationsToDrop(unsigned int size); + inline void adoptEnumerationsToDrop( + ::capnp::Orphan<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>&& + value); + inline ::capnp::Orphan<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>> + disownEnumerationsToDrop(); + private: ::capnp::_::StructBuilder _builder; template @@ -3042,6 +3161,9 @@ class Attribute::Reader { inline bool hasOrder() const; inline ::capnp::Text::Reader getOrder() const; + inline bool hasEnumerationName() const; + inline ::capnp::Text::Reader getEnumerationName() const; + private: ::capnp::_::StructReader _reader; template @@ -3131,6 +3253,13 @@ class Attribute::Builder { inline void adoptOrder(::capnp::Orphan<::capnp::Text>&& value); inline ::capnp::Orphan<::capnp::Text> disownOrder(); + inline bool hasEnumerationName(); + inline ::capnp::Text::Builder getEnumerationName(); + inline void setEnumerationName(::capnp::Text::Reader value); + inline ::capnp::Text::Builder initEnumerationName(unsigned int size); + inline void adoptEnumerationName(::capnp::Orphan<::capnp::Text>&& value); + inline ::capnp::Orphan<::capnp::Text> disownEnumerationName(); + private: ::capnp::_::StructBuilder _builder; template @@ -3163,6 +3292,153 @@ class Attribute::Pipeline { }; #endif // !CAPNP_LITE +class Enumeration::Reader { + public: + typedef Enumeration Reads; + + Reader() = default; + inline explicit Reader(::capnp::_::StructReader base) + : _reader(base) { + } + + inline ::capnp::MessageSize totalSize() const { + return _reader.totalSize().asPublic(); + } + +#if !CAPNP_LITE + inline ::kj::StringTree toString() const { + return ::capnp::_::structString(_reader, *_capnpPrivate::brand()); + } +#endif // !CAPNP_LITE + + inline bool hasName() const; + inline ::capnp::Text::Reader getName() const; + + inline bool hasPathName() const; + inline ::capnp::Text::Reader getPathName() const; + + inline bool hasType() const; + inline ::capnp::Text::Reader getType() const; + + inline ::uint32_t getCellValNum() const; + + inline bool getOrdered() const; + + inline bool hasData() const; + inline ::capnp::Data::Reader getData() const; + + inline bool hasOffsets() const; + inline ::capnp::Data::Reader getOffsets() const; + + private: + ::capnp::_::StructReader _reader; + template + friend struct ::capnp::ToDynamic_; + template + friend struct ::capnp::_::PointerHelpers; + template + friend struct ::capnp::List; + friend class ::capnp::MessageBuilder; + friend class ::capnp::Orphanage; +}; + +class Enumeration::Builder { + public: + typedef Enumeration Builds; + + Builder() = delete; // Deleted to discourage incorrect usage. + // You can explicitly initialize to nullptr instead. + inline Builder(decltype(nullptr)) { + } + inline explicit Builder(::capnp::_::StructBuilder base) + : _builder(base) { + } + inline operator Reader() const { + return Reader(_builder.asReader()); + } + inline Reader asReader() const { + return *this; + } + + inline ::capnp::MessageSize totalSize() const { + return asReader().totalSize(); + } +#if !CAPNP_LITE + inline ::kj::StringTree toString() const { + return asReader().toString(); + } +#endif // !CAPNP_LITE + + inline bool hasName(); + inline ::capnp::Text::Builder getName(); + inline void setName(::capnp::Text::Reader value); + inline ::capnp::Text::Builder initName(unsigned int size); + inline void adoptName(::capnp::Orphan<::capnp::Text>&& value); + inline ::capnp::Orphan<::capnp::Text> disownName(); + + inline bool hasPathName(); + inline ::capnp::Text::Builder getPathName(); + inline void setPathName(::capnp::Text::Reader value); + inline ::capnp::Text::Builder initPathName(unsigned int size); + inline void adoptPathName(::capnp::Orphan<::capnp::Text>&& value); + inline ::capnp::Orphan<::capnp::Text> disownPathName(); + + inline bool hasType(); + inline ::capnp::Text::Builder getType(); + inline void setType(::capnp::Text::Reader value); + inline ::capnp::Text::Builder initType(unsigned int size); + inline void adoptType(::capnp::Orphan<::capnp::Text>&& value); + inline ::capnp::Orphan<::capnp::Text> disownType(); + + inline ::uint32_t getCellValNum(); + inline void setCellValNum(::uint32_t value); + + inline bool getOrdered(); + inline void setOrdered(bool value); + + inline bool hasData(); + inline ::capnp::Data::Builder getData(); + inline void setData(::capnp::Data::Reader value); + inline ::capnp::Data::Builder initData(unsigned int size); + inline void adoptData(::capnp::Orphan<::capnp::Data>&& value); + inline ::capnp::Orphan<::capnp::Data> disownData(); + + inline bool hasOffsets(); + inline ::capnp::Data::Builder getOffsets(); + inline void setOffsets(::capnp::Data::Reader value); + inline ::capnp::Data::Builder initOffsets(unsigned int size); + inline void adoptOffsets(::capnp::Orphan<::capnp::Data>&& value); + inline ::capnp::Orphan<::capnp::Data> disownOffsets(); + + private: + ::capnp::_::StructBuilder _builder; + template + friend struct ::capnp::ToDynamic_; + friend class ::capnp::Orphanage; + template + friend struct ::capnp::_::PointerHelpers; +}; + +#if !CAPNP_LITE +class Enumeration::Pipeline { + public: + typedef Enumeration Pipelines; + + inline Pipeline(decltype(nullptr)) + : _typeless(nullptr) { + } + inline explicit Pipeline(::capnp::AnyPointer::Pipeline&& typeless) + : _typeless(kj::mv(typeless)) { + } + + private: + ::capnp::AnyPointer::Pipeline _typeless; + friend class ::capnp::PipelineHook; + template + friend struct ::capnp::ToDynamic_; +}; +#endif // !CAPNP_LITE + class AttributeBufferHeader::Reader { public: typedef AttributeBufferHeader Reads; @@ -7193,6 +7469,8 @@ class ConditionClause::Reader { inline bool hasOp() const; inline ::capnp::Text::Reader getOp() const; + inline bool getUseEnumeration() const; + private: ::capnp::_::StructReader _reader; template @@ -7253,6 +7531,9 @@ class ConditionClause::Builder { inline void adoptOp(::capnp::Orphan<::capnp::Text>&& value); inline ::capnp::Orphan<::capnp::Text> disownOp(); + inline bool getUseEnumeration(); + inline void setUseEnumeration(bool value); + private: ::capnp::_::StructBuilder _builder; template @@ -7321,6 +7602,8 @@ class ASTNode::Reader { inline bool hasCombinationOp() const; inline ::capnp::Text::Reader getCombinationOp() const; + inline bool getUseEnumeration() const; + private: ::capnp::_::StructReader _reader; template @@ -7411,6 +7694,9 @@ class ASTNode::Builder { inline void adoptCombinationOp(::capnp::Orphan<::capnp::Text>&& value); inline ::capnp::Orphan<::capnp::Text> disownCombinationOp(); + inline bool getUseEnumeration(); + inline void setUseEnumeration(bool value); + private: ::capnp::_::StructBuilder _builder; template @@ -15475,54 +15761,199 @@ ArraySchema::Builder::disownDimensionLabels() { ::capnp::POINTERS)); } -inline ::uint32_t DimensionLabel::Reader::getDimensionId() const { - return _reader.getDataField<::uint32_t>( - ::capnp::bounded<0>() * ::capnp::ELEMENTS); -} - -inline ::uint32_t DimensionLabel::Builder::getDimensionId() { - return _builder.getDataField<::uint32_t>( - ::capnp::bounded<0>() * ::capnp::ELEMENTS); -} -inline void DimensionLabel::Builder::setDimensionId(::uint32_t value) { - _builder.setDataField<::uint32_t>( - ::capnp::bounded<0>() * ::capnp::ELEMENTS, value); -} - -inline bool DimensionLabel::Reader::hasName() const { - return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) +inline bool ArraySchema::Reader::hasEnumerations() const { + return !_reader.getPointerField(::capnp::bounded<13>() * ::capnp::POINTERS) .isNull(); } -inline bool DimensionLabel::Builder::hasName() { - return !_builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) +inline bool ArraySchema::Builder::hasEnumerations() { + return !_builder.getPointerField(::capnp::bounded<13>() * ::capnp::POINTERS) .isNull(); } -inline ::capnp::Text::Reader DimensionLabel::Reader::getName() const { - return ::capnp::_::PointerHelpers<::capnp::Text>::get( - _reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); -} -inline ::capnp::Text::Builder DimensionLabel::Builder::getName() { - return ::capnp::_::PointerHelpers<::capnp::Text>::get( - _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader +ArraySchema::Reader::getEnumerations() const { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>::get(_reader + .getPointerField( + ::capnp::bounded<13>() * + ::capnp::POINTERS)); } -inline void DimensionLabel::Builder::setName(::capnp::Text::Reader value) { - ::capnp::_::PointerHelpers<::capnp::Text>::set( - _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), - value); +inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder +ArraySchema::Builder::getEnumerations() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>::get(_builder + .getPointerField( + ::capnp::bounded<13>() * + ::capnp::POINTERS)); } -inline ::capnp::Text::Builder DimensionLabel::Builder::initName( - unsigned int size) { - return ::capnp::_::PointerHelpers<::capnp::Text>::init( - _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), - size); +inline void ArraySchema::Builder::setEnumerations( + ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>:: + set(_builder.getPointerField(::capnp::bounded<13>() * ::capnp::POINTERS), + value); } -inline void DimensionLabel::Builder::adoptName( - ::capnp::Orphan<::capnp::Text>&& value) { - ::capnp::_::PointerHelpers<::capnp::Text>::adopt( - _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), - kj::mv(value)); +inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder +ArraySchema::Builder::initEnumerations(unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>:: + init( + _builder.getPointerField(::capnp::bounded<13>() * ::capnp::POINTERS), + size); } -inline ::capnp::Orphan<::capnp::Text> DimensionLabel::Builder::disownName() { +inline void ArraySchema::Builder::adoptEnumerations( + ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>&& value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>:: + adopt( + _builder.getPointerField(::capnp::bounded<13>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>> +ArraySchema::Builder::disownEnumerations() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>::disown(_builder + .getPointerField( + ::capnp::bounded<13>() * + ::capnp::POINTERS)); +} + +inline bool ArraySchema::Reader::hasEnumerationPathMap() const { + return !_reader.getPointerField(::capnp::bounded<14>() * ::capnp::POINTERS) + .isNull(); +} +inline bool ArraySchema::Builder::hasEnumerationPathMap() { + return !_builder.getPointerField(::capnp::bounded<14>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp:: + List<::tiledb::sm::serialization::capnp::KV, ::capnp::Kind::STRUCT>::Reader + ArraySchema::Reader::getEnumerationPathMap() const { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>>::get(_reader + .getPointerField( + ::capnp::bounded<14>() * + ::capnp::POINTERS)); +} +inline ::capnp:: + List<::tiledb::sm::serialization::capnp::KV, ::capnp::Kind::STRUCT>::Builder + ArraySchema::Builder::getEnumerationPathMap() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>>::get(_builder + .getPointerField( + ::capnp::bounded<14>() * + ::capnp::POINTERS)); +} +inline void ArraySchema::Builder::setEnumerationPathMap( + ::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>::Reader value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>>:: + set(_builder.getPointerField(::capnp::bounded<14>() * ::capnp::POINTERS), + value); +} +inline ::capnp:: + List<::tiledb::sm::serialization::capnp::KV, ::capnp::Kind::STRUCT>::Builder + ArraySchema::Builder::initEnumerationPathMap(unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>>:: + init( + _builder.getPointerField(::capnp::bounded<14>() * ::capnp::POINTERS), + size); +} +inline void ArraySchema::Builder::adoptEnumerationPathMap( + ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>>&& value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>>:: + adopt( + _builder.getPointerField(::capnp::bounded<14>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>> +ArraySchema::Builder::disownEnumerationPathMap() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>>::disown(_builder + .getPointerField( + ::capnp::bounded<14>() * + ::capnp::POINTERS)); +} + +inline ::uint32_t DimensionLabel::Reader::getDimensionId() const { + return _reader.getDataField<::uint32_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS); +} + +inline ::uint32_t DimensionLabel::Builder::getDimensionId() { + return _builder.getDataField<::uint32_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS); +} +inline void DimensionLabel::Builder::setDimensionId(::uint32_t value) { + _builder.setDataField<::uint32_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS, value); +} + +inline bool DimensionLabel::Reader::hasName() const { + return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) + .isNull(); +} +inline bool DimensionLabel::Builder::hasName() { + return !_builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::Text::Reader DimensionLabel::Reader::getName() const { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder DimensionLabel::Builder::getName() { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline void DimensionLabel::Builder::setName(::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers<::capnp::Text>::set( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + value); +} +inline ::capnp::Text::Builder DimensionLabel::Builder::initName( + unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::Text>::init( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + size); +} +inline void DimensionLabel::Builder::adoptName( + ::capnp::Orphan<::capnp::Text>&& value) { + ::capnp::_::PointerHelpers<::capnp::Text>::adopt( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::Text> DimensionLabel::Builder::disownName() { return ::capnp::_::PointerHelpers<::capnp::Text>::disown( _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); } @@ -15962,6 +16393,136 @@ ArraySchemaEvolution::Builder::disownTimestampRange() { _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS)); } +inline bool ArraySchemaEvolution::Reader::hasEnumerationsToAdd() const { + return !_reader.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS) + .isNull(); +} +inline bool ArraySchemaEvolution::Builder::hasEnumerationsToAdd() { + return !_builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader +ArraySchemaEvolution::Reader::getEnumerationsToAdd() const { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>::get(_reader + .getPointerField( + ::capnp::bounded<3>() * + ::capnp::POINTERS)); +} +inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder +ArraySchemaEvolution::Builder::getEnumerationsToAdd() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>::get(_builder + .getPointerField( + ::capnp::bounded<3>() * + ::capnp::POINTERS)); +} +inline void ArraySchemaEvolution::Builder::setEnumerationsToAdd( + ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>:: + set(_builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS), + value); +} +inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder +ArraySchemaEvolution::Builder::initEnumerationsToAdd(unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>:: + init( + _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS), + size); +} +inline void ArraySchemaEvolution::Builder::adoptEnumerationsToAdd( + ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>&& value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>:: + adopt( + _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>> +ArraySchemaEvolution::Builder::disownEnumerationsToAdd() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>::disown(_builder + .getPointerField( + ::capnp::bounded<3>() * + ::capnp::POINTERS)); +} + +inline bool ArraySchemaEvolution::Reader::hasEnumerationsToDrop() const { + return !_reader.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS) + .isNull(); +} +inline bool ArraySchemaEvolution::Builder::hasEnumerationsToDrop() { + return !_builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Reader +ArraySchemaEvolution::Reader::getEnumerationsToDrop() const { + return ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::get( + _reader.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS)); +} +inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Builder +ArraySchemaEvolution::Builder::getEnumerationsToDrop() { + return ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::get( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS)); +} +inline void ArraySchemaEvolution::Builder::setEnumerationsToDrop( + ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Reader value) { + ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::set( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS), + value); +} +inline void ArraySchemaEvolution::Builder::setEnumerationsToDrop( + ::kj::ArrayPtr value) { + ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::set( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS), + value); +} +inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Builder +ArraySchemaEvolution::Builder::initEnumerationsToDrop(unsigned int size) { + return ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::init( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS), + size); +} +inline void ArraySchemaEvolution::Builder::adoptEnumerationsToDrop( + ::capnp::Orphan<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>&& + value) { + ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::adopt( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>> +ArraySchemaEvolution::Builder::disownEnumerationsToDrop() { + return ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::disown( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS)); +} + inline ::uint32_t Attribute::Reader::getCellValNum() const { return _reader.getDataField<::uint32_t>( ::capnp::bounded<0>() * ::capnp::ELEMENTS); @@ -16207,6 +16768,263 @@ inline ::capnp::Orphan<::capnp::Text> Attribute::Builder::disownOrder() { _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS)); } +inline bool Attribute::Reader::hasEnumerationName() const { + return !_reader.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS) + .isNull(); +} +inline bool Attribute::Builder::hasEnumerationName() { + return !_builder.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::Text::Reader Attribute::Reader::getEnumerationName() const { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _reader.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder Attribute::Builder::getEnumerationName() { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _builder.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS)); +} +inline void Attribute::Builder::setEnumerationName( + ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers<::capnp::Text>::set( + _builder.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS), + value); +} +inline ::capnp::Text::Builder Attribute::Builder::initEnumerationName( + unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::Text>::init( + _builder.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS), + size); +} +inline void Attribute::Builder::adoptEnumerationName( + ::capnp::Orphan<::capnp::Text>&& value) { + ::capnp::_::PointerHelpers<::capnp::Text>::adopt( + _builder.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::Text> +Attribute::Builder::disownEnumerationName() { + return ::capnp::_::PointerHelpers<::capnp::Text>::disown( + _builder.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS)); +} + +inline bool Enumeration::Reader::hasName() const { + return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) + .isNull(); +} +inline bool Enumeration::Builder::hasName() { + return !_builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::Text::Reader Enumeration::Reader::getName() const { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder Enumeration::Builder::getName() { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline void Enumeration::Builder::setName(::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers<::capnp::Text>::set( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + value); +} +inline ::capnp::Text::Builder Enumeration::Builder::initName( + unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::Text>::init( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + size); +} +inline void Enumeration::Builder::adoptName( + ::capnp::Orphan<::capnp::Text>&& value) { + ::capnp::_::PointerHelpers<::capnp::Text>::adopt( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::Text> Enumeration::Builder::disownName() { + return ::capnp::_::PointerHelpers<::capnp::Text>::disown( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} + +inline bool Enumeration::Reader::hasPathName() const { + return !_reader.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS) + .isNull(); +} +inline bool Enumeration::Builder::hasPathName() { + return !_builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::Text::Reader Enumeration::Reader::getPathName() const { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _reader.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder Enumeration::Builder::getPathName() { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); +} +inline void Enumeration::Builder::setPathName(::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers<::capnp::Text>::set( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), + value); +} +inline ::capnp::Text::Builder Enumeration::Builder::initPathName( + unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::Text>::init( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), + size); +} +inline void Enumeration::Builder::adoptPathName( + ::capnp::Orphan<::capnp::Text>&& value) { + ::capnp::_::PointerHelpers<::capnp::Text>::adopt( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::Text> Enumeration::Builder::disownPathName() { + return ::capnp::_::PointerHelpers<::capnp::Text>::disown( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); +} + +inline bool Enumeration::Reader::hasType() const { + return !_reader.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS) + .isNull(); +} +inline bool Enumeration::Builder::hasType() { + return !_builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::Text::Reader Enumeration::Reader::getType() const { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _reader.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder Enumeration::Builder::getType() { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS)); +} +inline void Enumeration::Builder::setType(::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers<::capnp::Text>::set( + _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS), + value); +} +inline ::capnp::Text::Builder Enumeration::Builder::initType( + unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::Text>::init( + _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS), + size); +} +inline void Enumeration::Builder::adoptType( + ::capnp::Orphan<::capnp::Text>&& value) { + ::capnp::_::PointerHelpers<::capnp::Text>::adopt( + _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::Text> Enumeration::Builder::disownType() { + return ::capnp::_::PointerHelpers<::capnp::Text>::disown( + _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS)); +} + +inline ::uint32_t Enumeration::Reader::getCellValNum() const { + return _reader.getDataField<::uint32_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS); +} + +inline ::uint32_t Enumeration::Builder::getCellValNum() { + return _builder.getDataField<::uint32_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS); +} +inline void Enumeration::Builder::setCellValNum(::uint32_t value) { + _builder.setDataField<::uint32_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS, value); +} + +inline bool Enumeration::Reader::getOrdered() const { + return _reader.getDataField(::capnp::bounded<32>() * ::capnp::ELEMENTS); +} + +inline bool Enumeration::Builder::getOrdered() { + return _builder.getDataField( + ::capnp::bounded<32>() * ::capnp::ELEMENTS); +} +inline void Enumeration::Builder::setOrdered(bool value) { + _builder.setDataField( + ::capnp::bounded<32>() * ::capnp::ELEMENTS, value); +} + +inline bool Enumeration::Reader::hasData() const { + return !_reader.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS) + .isNull(); +} +inline bool Enumeration::Builder::hasData() { + return !_builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::Data::Reader Enumeration::Reader::getData() const { + return ::capnp::_::PointerHelpers<::capnp::Data>::get( + _reader.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS)); +} +inline ::capnp::Data::Builder Enumeration::Builder::getData() { + return ::capnp::_::PointerHelpers<::capnp::Data>::get( + _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS)); +} +inline void Enumeration::Builder::setData(::capnp::Data::Reader value) { + ::capnp::_::PointerHelpers<::capnp::Data>::set( + _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS), + value); +} +inline ::capnp::Data::Builder Enumeration::Builder::initData( + unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::Data>::init( + _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS), + size); +} +inline void Enumeration::Builder::adoptData( + ::capnp::Orphan<::capnp::Data>&& value) { + ::capnp::_::PointerHelpers<::capnp::Data>::adopt( + _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::Data> Enumeration::Builder::disownData() { + return ::capnp::_::PointerHelpers<::capnp::Data>::disown( + _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS)); +} + +inline bool Enumeration::Reader::hasOffsets() const { + return !_reader.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS) + .isNull(); +} +inline bool Enumeration::Builder::hasOffsets() { + return !_builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::Data::Reader Enumeration::Reader::getOffsets() const { + return ::capnp::_::PointerHelpers<::capnp::Data>::get( + _reader.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS)); +} +inline ::capnp::Data::Builder Enumeration::Builder::getOffsets() { + return ::capnp::_::PointerHelpers<::capnp::Data>::get( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS)); +} +inline void Enumeration::Builder::setOffsets(::capnp::Data::Reader value) { + ::capnp::_::PointerHelpers<::capnp::Data>::set( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS), + value); +} +inline ::capnp::Data::Builder Enumeration::Builder::initOffsets( + unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::Data>::init( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS), + size); +} +inline void Enumeration::Builder::adoptOffsets( + ::capnp::Orphan<::capnp::Data>&& value) { + ::capnp::_::PointerHelpers<::capnp::Data>::adopt( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::Data> Enumeration::Builder::disownOffsets() { + return ::capnp::_::PointerHelpers<::capnp::Data>::disown( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS)); +} + inline bool AttributeBufferHeader::Reader::hasName() const { return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) .isNull(); @@ -20752,6 +21570,17 @@ inline ::capnp::Orphan<::capnp::Text> ConditionClause::Builder::disownOp() { _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS)); } +inline bool ConditionClause::Reader::getUseEnumeration() const { + return _reader.getDataField(::capnp::bounded<0>() * ::capnp::ELEMENTS); +} + +inline bool ConditionClause::Builder::getUseEnumeration() { + return _builder.getDataField(::capnp::bounded<0>() * ::capnp::ELEMENTS); +} +inline void ConditionClause::Builder::setUseEnumeration(bool value) { + _builder.setDataField(::capnp::bounded<0>() * ::capnp::ELEMENTS, value); +} + inline bool ASTNode::Reader::getIsExpression() const { return _reader.getDataField(::capnp::bounded<0>() * ::capnp::ELEMENTS); } @@ -20986,6 +21815,17 @@ inline ::capnp::Orphan<::capnp::Text> ASTNode::Builder::disownCombinationOp() { _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS)); } +inline bool ASTNode::Reader::getUseEnumeration() const { + return _reader.getDataField(::capnp::bounded<1>() * ::capnp::ELEMENTS); +} + +inline bool ASTNode::Builder::getUseEnumeration() { + return _builder.getDataField(::capnp::bounded<1>() * ::capnp::ELEMENTS); +} +inline void ASTNode::Builder::setUseEnumeration(bool value) { + _builder.setDataField(::capnp::bounded<1>() * ::capnp::ELEMENTS, value); +} + inline bool Condition::Reader::hasClauses() const { return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) .isNull(); diff --git a/tiledb/sm/serialization/query.cc b/tiledb/sm/serialization/query.cc index 4e697a28e812..397eba71aa85 100644 --- a/tiledb/sm/serialization/query.cc +++ b/tiledb/sm/serialization/query.cc @@ -776,6 +776,9 @@ static Status condition_ast_to_capnp( const std::string op_str = query_condition_op_str(node->get_op()); ensure_qc_op_string_is_valid(op_str); ast_builder->setOp(op_str); + + // Store whether this expression should skip the enumeration lookup. + ast_builder->setUseEnumeration(node->use_enumeration()); } else { // Store the boolean expression tag. ast_builder->setIsExpression(true); @@ -823,6 +826,8 @@ static void clause_to_capnp( const std::string op_str = query_condition_op_str(node->get_op()); ensure_qc_op_string_is_valid(op_str); clause_builder->setOp(op_str); + + clause_builder->setUseEnumeration(node->use_enumeration()); } Status condition_to_capnp( @@ -996,8 +1001,10 @@ tdb_unique_ptr condition_ast_from_capnp( } ensure_qc_op_is_valid(op); + auto use_enumeration = ast_reader.getUseEnumeration(); + return tdb_unique_ptr( - tdb_new(ASTNodeVal, field_name, data, size, op)); + tdb_new(ASTNodeVal, field_name, data, size, op, use_enumeration)); } // Getting and validating the query condition combination operator. @@ -1049,8 +1056,10 @@ Status condition_from_capnp( } ensure_qc_op_is_valid(op); + bool use_enumeration = clause.getUseEnumeration(); + ast_nodes.push_back(tdb_unique_ptr( - tdb_new(ASTNodeVal, field_name, data, size, op))); + tdb_new(ASTNodeVal, field_name, data, size, op, use_enumeration))); } // Constructing the tree from the list of AST nodes. diff --git a/tiledb/sm/serialization/tiledb-rest.capnp b/tiledb/sm/serialization/tiledb-rest.capnp index 1bcfb60f3bcb..6a37ed9d67eb 100644 --- a/tiledb/sm/serialization/tiledb-rest.capnp +++ b/tiledb/sm/serialization/tiledb-rest.capnp @@ -122,6 +122,12 @@ struct ArraySchema { dimensionLabels @14 :List(DimensionLabel); # Dimension labels of the array + + enumerations @15: List(Enumeration); + # Enumerations of the array + + enumerationPathMap @16: List(KV); + # Enumeration name to path map } struct DimensionLabel { @@ -167,6 +173,12 @@ struct ArraySchemaEvolution { timestampRange @2 :List(UInt64); # Timestamp range of array schema + + enumerationsToAdd @3 :List(Enumeration); + # Enumerations to be added + + enumerationsToDrop @4 :List(Text); + # Enumeration names to be dropped } struct Attribute { @@ -194,6 +206,33 @@ struct Attribute { order @7 :Text; # The prescribed order of the data stored in the attribute + + enumerationName @8 :Text; + # Name of the enumeration for this attribute, if it has one +} + +struct Enumeration { +# Enumeration of values for use by Attributes + name @0 :Text; + # Enumeration name + + pathName @1 :Text; + # Enumeration path name + + type @2 :Text; + # Type of the Enumeration values + + cellValNum @3 :UInt32; + # Enumeration number of values per cell + + ordered @4 :Bool; + # Whether the enumeration is considered orderable + + data @5 :Data; + # The contents of the enumeration values + + offsets @6 :Data; + # The contents of the enumeration offsets buffer } struct AttributeBufferHeader { @@ -534,6 +573,9 @@ struct ConditionClause { op @2 :Text; # The comparison operation + + useEnumeration @3 :Bool; + # Whether or not to use the associated attribute's Enumeration } struct ASTNode { @@ -557,6 +599,9 @@ struct ASTNode { combinationOp @5 :Text; # The combination logical operator + + useEnumeration @6 :Bool; + # Whether or not to use the associated attribute's Enumeration } struct Condition { diff --git a/tiledb/sm/serialization/win32/tiledb-rest.capnp.c++ b/tiledb/sm/serialization/win32/tiledb-rest.capnp.c++ index 08df30190b05..86e22aaddac3 100644 --- a/tiledb/sm/serialization/win32/tiledb-rest.capnp.c++ +++ b/tiledb/sm/serialization/win32/tiledb-rest.capnp.c++ @@ -635,17 +635,17 @@ const ::capnp::_::RawSchema s_facceeafd4472c68 = { 1, 2, i_facceeafd4472c68, nullptr, nullptr, { &s_facceeafd4472c68, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<274> b_d71de32f98e296fe = { +static const ::capnp::_::AlignedData<315> b_d71de32f98e296fe = { { 0, 0, 0, 0, 5, 0, 6, 0, 254, 150, 226, 152, 47, 227, 29, 215, 18, 0, 0, 0, 1, 0, 2, 0, 127, 216, 135, 181, 36, 146, 125, 181, - 13, 0, 7, 0, 0, 0, 0, 0, + 15, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 242, 0, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 0, 0, 0, 79, 3, 0, 0, + 29, 0, 0, 0, 191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, @@ -653,112 +653,126 @@ static const ::capnp::_::AlignedData<274> b_d71de32f98e296fe = { 112, 58, 65, 114, 114, 97, 121, 83, 99, 104, 101, 109, 97, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, - 60, 0, 0, 0, 3, 0, 4, 0, + 68, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 149, 1, 0, 0, 82, 0, 0, 0, + 205, 1, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 148, 1, 0, 0, 3, 0, 1, 0, - 160, 1, 0, 0, 2, 0, 1, 0, + 204, 1, 0, 0, 3, 0, 1, 0, + 216, 1, 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 157, 1, 0, 0, 90, 0, 0, 0, + 213, 1, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 156, 1, 0, 0, 3, 0, 1, 0, - 184, 1, 0, 0, 2, 0, 1, 0, + 212, 1, 0, 0, 3, 0, 1, 0, + 240, 1, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 181, 1, 0, 0, 74, 0, 0, 0, + 237, 1, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 180, 1, 0, 0, 3, 0, 1, 0, - 192, 1, 0, 0, 2, 0, 1, 0, + 236, 1, 0, 0, 3, 0, 1, 0, + 248, 1, 0, 0, 2, 0, 1, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 189, 1, 0, 0, 82, 0, 0, 0, + 245, 1, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 188, 1, 0, 0, 3, 0, 1, 0, - 200, 1, 0, 0, 2, 0, 1, 0, + 244, 1, 0, 0, 3, 0, 1, 0, + 0, 2, 0, 0, 2, 0, 1, 0, 4, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 197, 1, 0, 0, 170, 0, 0, 0, + 253, 1, 0, 0, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 200, 1, 0, 0, 3, 0, 1, 0, - 212, 1, 0, 0, 2, 0, 1, 0, + 0, 2, 0, 0, 3, 0, 1, 0, + 12, 2, 0, 0, 2, 0, 1, 0, 5, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 209, 1, 0, 0, 58, 0, 0, 0, + 9, 2, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 204, 1, 0, 0, 3, 0, 1, 0, - 216, 1, 0, 0, 2, 0, 1, 0, + 4, 2, 0, 0, 3, 0, 1, 0, + 16, 2, 0, 0, 2, 0, 1, 0, 6, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 213, 1, 0, 0, 170, 0, 0, 0, + 13, 2, 0, 0, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 216, 1, 0, 0, 3, 0, 1, 0, - 228, 1, 0, 0, 2, 0, 1, 0, + 16, 2, 0, 0, 3, 0, 1, 0, + 28, 2, 0, 0, 2, 0, 1, 0, 7, 0, 0, 0, 6, 0, 0, 0, 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 225, 1, 0, 0, 82, 0, 0, 0, + 25, 2, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 224, 1, 0, 0, 3, 0, 1, 0, - 236, 1, 0, 0, 2, 0, 1, 0, + 24, 2, 0, 0, 3, 0, 1, 0, + 36, 2, 0, 0, 2, 0, 1, 0, 8, 0, 0, 0, 7, 0, 0, 0, 0, 0, 1, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 233, 1, 0, 0, 34, 0, 0, 0, + 33, 2, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 228, 1, 0, 0, 3, 0, 1, 0, - 240, 1, 0, 0, 2, 0, 1, 0, + 28, 2, 0, 0, 3, 0, 1, 0, + 40, 2, 0, 0, 2, 0, 1, 0, 9, 0, 0, 0, 8, 0, 0, 0, 0, 0, 1, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 237, 1, 0, 0, 66, 0, 0, 0, + 37, 2, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 232, 1, 0, 0, 3, 0, 1, 0, - 4, 2, 0, 0, 2, 0, 1, 0, + 32, 2, 0, 0, 3, 0, 1, 0, + 60, 2, 0, 0, 2, 0, 1, 0, 10, 0, 0, 0, 64, 0, 0, 0, 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 2, 0, 0, 138, 0, 0, 0, + 57, 2, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 4, 2, 0, 0, 3, 0, 1, 0, - 16, 2, 0, 0, 2, 0, 1, 0, + 60, 2, 0, 0, 3, 0, 1, 0, + 72, 2, 0, 0, 2, 0, 1, 0, 11, 0, 0, 0, 9, 0, 0, 0, 0, 0, 1, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 13, 2, 0, 0, 186, 0, 0, 0, + 69, 2, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 16, 2, 0, 0, 3, 0, 1, 0, - 28, 2, 0, 0, 2, 0, 1, 0, + 72, 2, 0, 0, 3, 0, 1, 0, + 84, 2, 0, 0, 2, 0, 1, 0, 12, 0, 0, 0, 10, 0, 0, 0, 0, 0, 1, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 25, 2, 0, 0, 42, 0, 0, 0, + 81, 2, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 20, 2, 0, 0, 3, 0, 1, 0, - 32, 2, 0, 0, 2, 0, 1, 0, + 76, 2, 0, 0, 3, 0, 1, 0, + 88, 2, 0, 0, 2, 0, 1, 0, 13, 0, 0, 0, 11, 0, 0, 0, 0, 0, 1, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 2, 0, 0, 122, 0, 0, 0, + 85, 2, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 28, 2, 0, 0, 3, 0, 1, 0, - 56, 2, 0, 0, 2, 0, 1, 0, + 84, 2, 0, 0, 3, 0, 1, 0, + 112, 2, 0, 0, 2, 0, 1, 0, 14, 0, 0, 0, 12, 0, 0, 0, 0, 0, 1, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 53, 2, 0, 0, 130, 0, 0, 0, + 109, 2, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 52, 2, 0, 0, 3, 0, 1, 0, - 80, 2, 0, 0, 2, 0, 1, 0, + 108, 2, 0, 0, 3, 0, 1, 0, + 136, 2, 0, 0, 2, 0, 1, 0, + 15, 0, 0, 0, 13, 0, 0, 0, + 0, 0, 1, 0, 15, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 133, 2, 0, 0, 106, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 132, 2, 0, 0, 3, 0, 1, 0, + 160, 2, 0, 0, 2, 0, 1, 0, + 16, 0, 0, 0, 14, 0, 0, 0, + 0, 0, 1, 0, 16, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 157, 2, 0, 0, 154, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 160, 2, 0, 0, 3, 0, 1, 0, + 188, 2, 0, 0, 2, 0, 1, 0, 97, 114, 114, 97, 121, 84, 121, 112, 101, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, @@ -907,6 +921,33 @@ static const ::capnp::_::AlignedData<274> b_d71de32f98e296fe = { 222, 209, 12, 209, 98, 141, 255, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 110, 117, 109, 101, 114, 97, 116, + 105, 111, 110, 115, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 180, 185, 33, 204, 25, 47, 11, 208, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 110, 117, 109, 101, 114, 97, 116, + 105, 111, 110, 80, 97, 116, 104, 77, + 97, 112, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 151, 188, 17, 242, 43, 223, 218, 227, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, } @@ -917,13 +958,15 @@ static const ::capnp::_::RawSchema* const d_d71de32f98e296fe[] = { &s_92ad78f56de3d76a, &s_bc4583f733eac4f5, &s_ceff8d62d10cd1de, + &s_d00b2f19cc21b9b4, &s_de030f447664754c, + &s_e3dadf2bf211bc97, }; -static const uint16_t m_d71de32f98e296fe[] = {10, 0, 1, 2, 3, 4, 14, 5, 12, 6, 7, 13, 8, 11, 9}; -static const uint16_t i_d71de32f98e296fe[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}; +static const uint16_t m_d71de32f98e296fe[] = {10, 0, 1, 2, 3, 4, 14, 5, 16, 15, 12, 6, 7, 13, 8, 11, 9}; +static const uint16_t i_d71de32f98e296fe[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; const ::capnp::_::RawSchema s_d71de32f98e296fe = { - 0xd71de32f98e296fe, b_d71de32f98e296fe.words, 274, d_d71de32f98e296fe, m_d71de32f98e296fe, - 4, 15, i_d71de32f98e296fe, nullptr, nullptr, { &s_d71de32f98e296fe, nullptr, nullptr, 0, 0, nullptr } + 0xd71de32f98e296fe, b_d71de32f98e296fe.words, 315, d_d71de32f98e296fe, m_d71de32f98e296fe, + 6, 17, i_d71de32f98e296fe, nullptr, nullptr, { &s_d71de32f98e296fe, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE static const ::capnp::_::AlignedData<174> b_ceff8d62d10cd1de = { @@ -1114,17 +1157,17 @@ const ::capnp::_::RawSchema s_ceff8d62d10cd1de = { 1, 10, i_ceff8d62d10cd1de, nullptr, nullptr, { &s_ceff8d62d10cd1de, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<80> b_a1b81d67548230d4 = { +static const ::capnp::_::AlignedData<122> b_a1b81d67548230d4 = { { 0, 0, 0, 0, 5, 0, 6, 0, 212, 48, 130, 84, 103, 29, 184, 161, 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, - 3, 0, 7, 0, 0, 0, 0, 0, + 5, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 58, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 0, 0, 0, 175, 0, 0, 0, + 33, 0, 0, 0, 31, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, @@ -1133,28 +1176,42 @@ static const ::capnp::_::AlignedData<80> b_a1b81d67548230d4 = { 99, 104, 101, 109, 97, 69, 118, 111, 108, 117, 116, 105, 111, 110, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, - 12, 0, 0, 0, 3, 0, 4, 0, + 20, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 69, 0, 0, 0, 138, 0, 0, 0, + 125, 0, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 72, 0, 0, 0, 3, 0, 1, 0, - 100, 0, 0, 0, 2, 0, 1, 0, + 128, 0, 0, 0, 3, 0, 1, 0, + 156, 0, 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 97, 0, 0, 0, 130, 0, 0, 0, + 153, 0, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 96, 0, 0, 0, 3, 0, 1, 0, - 124, 0, 0, 0, 2, 0, 1, 0, + 152, 0, 0, 0, 3, 0, 1, 0, + 180, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 121, 0, 0, 0, 122, 0, 0, 0, + 177, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 120, 0, 0, 0, 3, 0, 1, 0, - 148, 0, 0, 0, 2, 0, 1, 0, + 176, 0, 0, 0, 3, 0, 1, 0, + 204, 0, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 201, 0, 0, 0, 146, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 204, 0, 0, 0, 3, 0, 1, 0, + 232, 0, 0, 0, 2, 0, 1, 0, + 4, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 229, 0, 0, 0, 154, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 232, 0, 0, 0, 3, 0, 1, 0, + 4, 1, 0, 0, 2, 0, 1, 0, 97, 116, 116, 114, 105, 98, 117, 116, 101, 115, 84, 111, 68, 114, 111, 112, 0, 0, 0, 0, 0, 0, 0, 0, @@ -1192,6 +1249,34 @@ static const ::capnp::_::AlignedData<80> b_a1b81d67548230d4 = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 110, 117, 109, 101, 114, 97, 116, + 105, 111, 110, 115, 84, 111, 65, 100, + 100, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 180, 185, 33, 204, 25, 47, 11, 208, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 110, 117, 109, 101, 114, 97, 116, + 105, 111, 110, 115, 84, 111, 68, 114, + 111, 112, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, } @@ -1200,25 +1285,26 @@ static const ::capnp::_::AlignedData<80> b_a1b81d67548230d4 = { #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_a1b81d67548230d4[] = { &s_92ad78f56de3d76a, + &s_d00b2f19cc21b9b4, }; -static const uint16_t m_a1b81d67548230d4[] = {1, 0, 2}; -static const uint16_t i_a1b81d67548230d4[] = {0, 1, 2}; +static const uint16_t m_a1b81d67548230d4[] = {1, 0, 3, 4, 2}; +static const uint16_t i_a1b81d67548230d4[] = {0, 1, 2, 3, 4}; const ::capnp::_::RawSchema s_a1b81d67548230d4 = { - 0xa1b81d67548230d4, b_a1b81d67548230d4.words, 80, d_a1b81d67548230d4, m_a1b81d67548230d4, - 1, 3, i_a1b81d67548230d4, nullptr, nullptr, { &s_a1b81d67548230d4, nullptr, nullptr, 0, 0, nullptr } + 0xa1b81d67548230d4, b_a1b81d67548230d4.words, 122, d_a1b81d67548230d4, m_a1b81d67548230d4, + 2, 5, i_a1b81d67548230d4, nullptr, nullptr, { &s_a1b81d67548230d4, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<144> b_92ad78f56de3d76a = { +static const ::capnp::_::AlignedData<160> b_92ad78f56de3d76a = { { 0, 0, 0, 0, 5, 0, 6, 0, 106, 215, 227, 109, 245, 120, 173, 146, 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, - 5, 0, 7, 0, 0, 0, 0, 0, + 6, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 226, 0, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 0, 0, 0, 199, 1, 0, 0, + 29, 0, 0, 0, 255, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, @@ -1226,63 +1312,70 @@ static const ::capnp::_::AlignedData<144> b_92ad78f56de3d76a = { 112, 58, 65, 116, 116, 114, 105, 98, 117, 116, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, - 32, 0, 0, 0, 3, 0, 4, 0, + 36, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 209, 0, 0, 0, 90, 0, 0, 0, + 237, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 208, 0, 0, 0, 3, 0, 1, 0, - 220, 0, 0, 0, 2, 0, 1, 0, + 236, 0, 0, 0, 3, 0, 1, 0, + 248, 0, 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 217, 0, 0, 0, 42, 0, 0, 0, + 245, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 212, 0, 0, 0, 3, 0, 1, 0, - 224, 0, 0, 0, 2, 0, 1, 0, + 240, 0, 0, 0, 3, 0, 1, 0, + 252, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 221, 0, 0, 0, 42, 0, 0, 0, + 249, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 216, 0, 0, 0, 3, 0, 1, 0, - 228, 0, 0, 0, 2, 0, 1, 0, + 244, 0, 0, 0, 3, 0, 1, 0, + 0, 1, 0, 0, 2, 0, 1, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 225, 0, 0, 0, 122, 0, 0, 0, + 253, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 224, 0, 0, 0, 3, 0, 1, 0, - 236, 0, 0, 0, 2, 0, 1, 0, + 252, 0, 0, 0, 3, 0, 1, 0, + 8, 1, 0, 0, 2, 0, 1, 0, 4, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 233, 0, 0, 0, 82, 0, 0, 0, + 5, 1, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 232, 0, 0, 0, 3, 0, 1, 0, - 244, 0, 0, 0, 2, 0, 1, 0, + 4, 1, 0, 0, 3, 0, 1, 0, + 16, 1, 0, 0, 2, 0, 1, 0, 5, 0, 0, 0, 32, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 241, 0, 0, 0, 74, 0, 0, 0, + 13, 1, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 240, 0, 0, 0, 3, 0, 1, 0, - 252, 0, 0, 0, 2, 0, 1, 0, + 12, 1, 0, 0, 3, 0, 1, 0, + 24, 1, 0, 0, 2, 0, 1, 0, 6, 0, 0, 0, 33, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 249, 0, 0, 0, 146, 0, 0, 0, + 21, 1, 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 252, 0, 0, 0, 3, 0, 1, 0, - 8, 1, 0, 0, 2, 0, 1, 0, + 24, 1, 0, 0, 3, 0, 1, 0, + 36, 1, 0, 0, 2, 0, 1, 0, 7, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 5, 1, 0, 0, 50, 0, 0, 0, + 33, 1, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1, 0, 0, 3, 0, 1, 0, - 12, 1, 0, 0, 2, 0, 1, 0, + 28, 1, 0, 0, 3, 0, 1, 0, + 40, 1, 0, 0, 2, 0, 1, 0, + 8, 0, 0, 0, 5, 0, 0, 0, + 0, 0, 1, 0, 8, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 37, 1, 0, 0, 130, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 36, 1, 0, 0, 3, 0, 1, 0, + 48, 1, 0, 0, 2, 0, 1, 0, 99, 101, 108, 108, 86, 97, 108, 78, 117, 109, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, @@ -1346,6 +1439,15 @@ static const ::capnp::_::AlignedData<144> b_92ad78f56de3d76a = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 114, 100, 101, 114, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 110, 117, 109, 101, 114, 97, 116, + 105, 111, 110, 78, 97, 109, 101, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -1359,11 +1461,147 @@ static const ::capnp::_::AlignedData<144> b_92ad78f56de3d76a = { static const ::capnp::_::RawSchema* const d_92ad78f56de3d76a[] = { &s_bc4583f733eac4f5, }; -static const uint16_t m_92ad78f56de3d76a[] = {0, 4, 6, 3, 1, 5, 7, 2}; -static const uint16_t i_92ad78f56de3d76a[] = {0, 1, 2, 3, 4, 5, 6, 7}; +static const uint16_t m_92ad78f56de3d76a[] = {0, 8, 4, 6, 3, 1, 5, 7, 2}; +static const uint16_t i_92ad78f56de3d76a[] = {0, 1, 2, 3, 4, 5, 6, 7, 8}; const ::capnp::_::RawSchema s_92ad78f56de3d76a = { - 0x92ad78f56de3d76a, b_92ad78f56de3d76a.words, 144, d_92ad78f56de3d76a, m_92ad78f56de3d76a, - 1, 8, i_92ad78f56de3d76a, nullptr, nullptr, { &s_92ad78f56de3d76a, nullptr, nullptr, 0, 0, nullptr } + 0x92ad78f56de3d76a, b_92ad78f56de3d76a.words, 160, d_92ad78f56de3d76a, m_92ad78f56de3d76a, + 1, 9, i_92ad78f56de3d76a, nullptr, nullptr, { &s_92ad78f56de3d76a, nullptr, nullptr, 0, 0, nullptr } +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<125> b_d00b2f19cc21b9b4 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 180, 185, 33, 204, 25, 47, 11, 208, + 18, 0, 0, 0, 1, 0, 1, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 5, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 242, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 29, 0, 0, 0, 143, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 69, 110, 117, 109, 101, 114, + 97, 116, 105, 111, 110, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 28, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 181, 0, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 176, 0, 0, 0, 3, 0, 1, 0, + 188, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 185, 0, 0, 0, 74, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 184, 0, 0, 0, 3, 0, 1, 0, + 196, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 193, 0, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 188, 0, 0, 0, 3, 0, 1, 0, + 200, 0, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 197, 0, 0, 0, 90, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 196, 0, 0, 0, 3, 0, 1, 0, + 208, 0, 0, 0, 2, 0, 1, 0, + 4, 0, 0, 0, 32, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 205, 0, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 200, 0, 0, 0, 3, 0, 1, 0, + 212, 0, 0, 0, 2, 0, 1, 0, + 5, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 5, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 209, 0, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 204, 0, 0, 0, 3, 0, 1, 0, + 216, 0, 0, 0, 2, 0, 1, 0, + 6, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 6, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 213, 0, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 208, 0, 0, 0, 3, 0, 1, 0, + 220, 0, 0, 0, 2, 0, 1, 0, + 110, 97, 109, 101, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 112, 97, 116, 104, 78, 97, 109, 101, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 121, 112, 101, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 101, 108, 108, 86, 97, 108, 78, + 117, 109, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 111, 114, 100, 101, 114, 101, 100, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 100, 97, 116, 97, 0, 0, 0, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 111, 102, 102, 115, 101, 116, 115, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; +::capnp::word const* const bp_d00b2f19cc21b9b4 = b_d00b2f19cc21b9b4.words; +#if !CAPNP_LITE +static const uint16_t m_d00b2f19cc21b9b4[] = {3, 5, 0, 6, 4, 1, 2}; +static const uint16_t i_d00b2f19cc21b9b4[] = {0, 1, 2, 3, 4, 5, 6}; +const ::capnp::_::RawSchema s_d00b2f19cc21b9b4 = { + 0xd00b2f19cc21b9b4, b_d00b2f19cc21b9b4.words, 125, nullptr, m_d00b2f19cc21b9b4, + 0, 7, i_d00b2f19cc21b9b4, nullptr, nullptr, { &s_d00b2f19cc21b9b4, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE static const ::capnp::_::AlignedData<143> b_d20a578112fa92a2 = { @@ -4271,17 +4509,17 @@ const ::capnp::_::RawSchema s_cbe1e7c13508aa2c = { 1, 4, i_cbe1e7c13508aa2c, nullptr, nullptr, { &s_cbe1e7c13508aa2c, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<65> b_dac6a7f675c57409 = { +static const ::capnp::_::AlignedData<81> b_dac6a7f675c57409 = { { 0, 0, 0, 0, 5, 0, 6, 0, 9, 116, 197, 117, 246, 167, 198, 218, - 18, 0, 0, 0, 1, 0, 0, 0, + 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 18, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 0, 0, 0, 175, 0, 0, 0, + 33, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, @@ -4290,28 +4528,35 @@ static const ::capnp::_::AlignedData<65> b_dac6a7f675c57409 = { 105, 111, 110, 67, 108, 97, 117, 115, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, - 12, 0, 0, 0, 3, 0, 4, 0, + 16, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 69, 0, 0, 0, 82, 0, 0, 0, + 97, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 68, 0, 0, 0, 3, 0, 1, 0, - 80, 0, 0, 0, 2, 0, 1, 0, + 96, 0, 0, 0, 3, 0, 1, 0, + 108, 0, 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 77, 0, 0, 0, 50, 0, 0, 0, + 105, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 72, 0, 0, 0, 3, 0, 1, 0, - 84, 0, 0, 0, 2, 0, 1, 0, + 100, 0, 0, 0, 3, 0, 1, 0, + 112, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 81, 0, 0, 0, 26, 0, 0, 0, + 109, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 76, 0, 0, 0, 3, 0, 1, 0, - 88, 0, 0, 0, 2, 0, 1, 0, + 104, 0, 0, 0, 3, 0, 1, 0, + 116, 0, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 113, 0, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 112, 0, 0, 0, 3, 0, 1, 0, + 124, 0, 0, 0, 2, 0, 1, 0, 102, 105, 101, 108, 100, 78, 97, 109, 101, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, @@ -4335,19 +4580,28 @@ static const ::capnp::_::AlignedData<65> b_dac6a7f675c57409 = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 117, 115, 101, 69, 110, 117, 109, 101, + 114, 97, 116, 105, 111, 110, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, } }; ::capnp::word const* const bp_dac6a7f675c57409 = b_dac6a7f675c57409.words; #if !CAPNP_LITE -static const uint16_t m_dac6a7f675c57409[] = {0, 2, 1}; -static const uint16_t i_dac6a7f675c57409[] = {0, 1, 2}; +static const uint16_t m_dac6a7f675c57409[] = {0, 2, 3, 1}; +static const uint16_t i_dac6a7f675c57409[] = {0, 1, 2, 3}; const ::capnp::_::RawSchema s_dac6a7f675c57409 = { - 0xdac6a7f675c57409, b_dac6a7f675c57409.words, 65, nullptr, m_dac6a7f675c57409, - 0, 3, i_dac6a7f675c57409, nullptr, nullptr, { &s_dac6a7f675c57409, nullptr, nullptr, 0, 0, nullptr } + 0xdac6a7f675c57409, b_dac6a7f675c57409.words, 81, nullptr, m_dac6a7f675c57409, + 0, 4, i_dac6a7f675c57409, nullptr, nullptr, { &s_dac6a7f675c57409, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<116> b_afc739d5c01e6496 = { +static const ::capnp::_::AlignedData<132> b_afc739d5c01e6496 = { { 0, 0, 0, 0, 5, 0, 6, 0, 150, 100, 30, 192, 213, 57, 199, 175, 18, 0, 0, 0, 1, 0, 1, 0, @@ -4357,7 +4611,7 @@ static const ::capnp::_::AlignedData<116> b_afc739d5c01e6496 = { 21, 0, 0, 0, 210, 0, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 0, 0, 0, 87, 1, 0, 0, + 29, 0, 0, 0, 143, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, @@ -4365,49 +4619,56 @@ static const ::capnp::_::AlignedData<116> b_afc739d5c01e6496 = { 112, 58, 65, 83, 84, 78, 111, 100, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, - 24, 0, 0, 0, 3, 0, 4, 0, + 28, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 153, 0, 0, 0, 106, 0, 0, 0, + 181, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 152, 0, 0, 0, 3, 0, 1, 0, - 164, 0, 0, 0, 2, 0, 1, 0, + 180, 0, 0, 0, 3, 0, 1, 0, + 192, 0, 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 161, 0, 0, 0, 82, 0, 0, 0, + 189, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 160, 0, 0, 0, 3, 0, 1, 0, - 172, 0, 0, 0, 2, 0, 1, 0, + 188, 0, 0, 0, 3, 0, 1, 0, + 200, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 169, 0, 0, 0, 50, 0, 0, 0, + 197, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 164, 0, 0, 0, 3, 0, 1, 0, - 176, 0, 0, 0, 2, 0, 1, 0, + 192, 0, 0, 0, 3, 0, 1, 0, + 204, 0, 0, 0, 2, 0, 1, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 173, 0, 0, 0, 26, 0, 0, 0, + 201, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 168, 0, 0, 0, 3, 0, 1, 0, - 180, 0, 0, 0, 2, 0, 1, 0, + 196, 0, 0, 0, 3, 0, 1, 0, + 208, 0, 0, 0, 2, 0, 1, 0, 4, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 177, 0, 0, 0, 74, 0, 0, 0, + 205, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 176, 0, 0, 0, 3, 0, 1, 0, - 204, 0, 0, 0, 2, 0, 1, 0, + 204, 0, 0, 0, 3, 0, 1, 0, + 232, 0, 0, 0, 2, 0, 1, 0, 5, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 201, 0, 0, 0, 114, 0, 0, 0, + 229, 0, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 200, 0, 0, 0, 3, 0, 1, 0, - 212, 0, 0, 0, 2, 0, 1, 0, + 228, 0, 0, 0, 3, 0, 1, 0, + 240, 0, 0, 0, 2, 0, 1, 0, + 6, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 6, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 237, 0, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 236, 0, 0, 0, 3, 0, 1, 0, + 248, 0, 0, 0, 2, 0, 1, 0, 105, 115, 69, 120, 112, 114, 101, 115, 115, 105, 111, 110, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, @@ -4462,6 +4723,15 @@ static const ::capnp::_::AlignedData<116> b_afc739d5c01e6496 = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 117, 115, 101, 69, 110, 117, 109, 101, + 114, 97, 116, 105, 111, 110, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, } }; @@ -4470,11 +4740,11 @@ static const ::capnp::_::AlignedData<116> b_afc739d5c01e6496 = { static const ::capnp::_::RawSchema* const d_afc739d5c01e6496[] = { &s_afc739d5c01e6496, }; -static const uint16_t m_afc739d5c01e6496[] = {4, 5, 1, 0, 3, 2}; -static const uint16_t i_afc739d5c01e6496[] = {0, 1, 2, 3, 4, 5}; +static const uint16_t m_afc739d5c01e6496[] = {4, 5, 1, 0, 3, 6, 2}; +static const uint16_t i_afc739d5c01e6496[] = {0, 1, 2, 3, 4, 5, 6}; const ::capnp::_::RawSchema s_afc739d5c01e6496 = { - 0xafc739d5c01e6496, b_afc739d5c01e6496.words, 116, d_afc739d5c01e6496, m_afc739d5c01e6496, - 1, 6, i_afc739d5c01e6496, nullptr, nullptr, { &s_afc739d5c01e6496, nullptr, nullptr, 0, 0, nullptr } + 0xafc739d5c01e6496, b_afc739d5c01e6496.words, 132, d_afc739d5c01e6496, m_afc739d5c01e6496, + 1, 7, i_afc739d5c01e6496, nullptr, nullptr, { &s_afc739d5c01e6496, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE static const ::capnp::_::AlignedData<73> b_eaf57cb9871fc06f = { @@ -9010,6 +9280,14 @@ constexpr ::capnp::Kind Attribute::_capnpPrivate::kind; constexpr ::capnp::_::RawSchema const* Attribute::_capnpPrivate::schema; #endif // !CAPNP_LITE +// Enumeration +constexpr uint16_t Enumeration::_capnpPrivate::dataWordSize; +constexpr uint16_t Enumeration::_capnpPrivate::pointerCount; +#if !CAPNP_LITE +constexpr ::capnp::Kind Enumeration::_capnpPrivate::kind; +constexpr ::capnp::_::RawSchema const* Enumeration::_capnpPrivate::schema; +#endif // !CAPNP_LITE + // AttributeBufferHeader constexpr uint16_t AttributeBufferHeader::_capnpPrivate::dataWordSize; constexpr uint16_t AttributeBufferHeader::_capnpPrivate::pointerCount; diff --git a/tiledb/sm/serialization/win32/tiledb-rest.capnp.h b/tiledb/sm/serialization/win32/tiledb-rest.capnp.h index 260cbaa58fb9..05ccbc0a6b37 100644 --- a/tiledb/sm/serialization/win32/tiledb-rest.capnp.h +++ b/tiledb/sm/serialization/win32/tiledb-rest.capnp.h @@ -23,6 +23,7 @@ CAPNP_DECLARE_SCHEMA(d71de32f98e296fe); CAPNP_DECLARE_SCHEMA(ceff8d62d10cd1de); CAPNP_DECLARE_SCHEMA(a1b81d67548230d4); CAPNP_DECLARE_SCHEMA(92ad78f56de3d76a); +CAPNP_DECLARE_SCHEMA(d00b2f19cc21b9b4); CAPNP_DECLARE_SCHEMA(d20a578112fa92a2); CAPNP_DECLARE_SCHEMA(95e26a84d32d8223); CAPNP_DECLARE_SCHEMA(a2a652536db09fa0); @@ -196,7 +197,7 @@ struct ArraySchema { class Pipeline; struct _capnpPrivate { - CAPNP_DECLARE_STRUCT_HEADER(d71de32f98e296fe, 2, 13) + CAPNP_DECLARE_STRUCT_HEADER(d71de32f98e296fe, 2, 15) #if !CAPNP_LITE static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; @@ -230,7 +231,7 @@ struct ArraySchemaEvolution { class Pipeline; struct _capnpPrivate { - CAPNP_DECLARE_STRUCT_HEADER(a1b81d67548230d4, 0, 3) + CAPNP_DECLARE_STRUCT_HEADER(a1b81d67548230d4, 0, 5) #if !CAPNP_LITE static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; @@ -247,7 +248,24 @@ struct Attribute { class Pipeline; struct _capnpPrivate { - CAPNP_DECLARE_STRUCT_HEADER(92ad78f56de3d76a, 1, 5) + CAPNP_DECLARE_STRUCT_HEADER(92ad78f56de3d76a, 1, 6) +#if !CAPNP_LITE + static constexpr ::capnp::_::RawBrandedSchema const* brand() { + return &schema->defaultBrand; + } +#endif // !CAPNP_LITE + }; +}; + +struct Enumeration { + Enumeration() = delete; + + class Reader; + class Builder; + class Pipeline; + + struct _capnpPrivate { + CAPNP_DECLARE_STRUCT_HEADER(d00b2f19cc21b9b4, 1, 5) #if !CAPNP_LITE static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; @@ -820,7 +838,7 @@ struct ConditionClause { class Pipeline; struct _capnpPrivate { - CAPNP_DECLARE_STRUCT_HEADER(dac6a7f675c57409, 0, 3) + CAPNP_DECLARE_STRUCT_HEADER(dac6a7f675c57409, 1, 3) #if !CAPNP_LITE static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; @@ -2441,6 +2459,18 @@ class ArraySchema::Reader { ::capnp::Kind::STRUCT>::Reader getDimensionLabels() const; + inline bool hasEnumerations() const; + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader + getEnumerations() const; + + inline bool hasEnumerationPathMap() const; + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>::Reader + getEnumerationPathMap() const; + private: ::capnp::_::StructReader _reader; template @@ -2640,6 +2670,48 @@ class ArraySchema::Builder { ::capnp::Kind::STRUCT>> disownDimensionLabels(); + inline bool hasEnumerations(); + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder + getEnumerations(); + inline void setEnumerations(::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader value); + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder + initEnumerations(unsigned int size); + inline void adoptEnumerations( + ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>&& value); + inline ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>> + disownEnumerations(); + + inline bool hasEnumerationPathMap(); + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>::Builder + getEnumerationPathMap(); + inline void setEnumerationPathMap(::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>::Reader value); + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>::Builder + initEnumerationPathMap(unsigned int size); + inline void adoptEnumerationPathMap( + ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>>&& value); + inline ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>> + disownEnumerationPathMap(); + private: ::capnp::_::StructBuilder _builder; template @@ -2883,6 +2955,16 @@ class ArraySchemaEvolution::Reader { inline ::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>::Reader getTimestampRange() const; + inline bool hasEnumerationsToAdd() const; + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader + getEnumerationsToAdd() const; + + inline bool hasEnumerationsToDrop() const; + inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Reader + getEnumerationsToDrop() const; + private: ::capnp::_::StructReader _reader; template @@ -2972,6 +3054,43 @@ class ArraySchemaEvolution::Builder { inline ::capnp::Orphan<::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>> disownTimestampRange(); + inline bool hasEnumerationsToAdd(); + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder + getEnumerationsToAdd(); + inline void setEnumerationsToAdd( + ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader value); + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder + initEnumerationsToAdd(unsigned int size); + inline void adoptEnumerationsToAdd( + ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>&& value); + inline ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>> + disownEnumerationsToAdd(); + + inline bool hasEnumerationsToDrop(); + inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Builder + getEnumerationsToDrop(); + inline void setEnumerationsToDrop( + ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Reader value); + inline void setEnumerationsToDrop( + ::kj::ArrayPtr value); + inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Builder + initEnumerationsToDrop(unsigned int size); + inline void adoptEnumerationsToDrop( + ::capnp::Orphan<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>&& + value); + inline ::capnp::Orphan<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>> + disownEnumerationsToDrop(); + private: ::capnp::_::StructBuilder _builder; template @@ -3042,6 +3161,9 @@ class Attribute::Reader { inline bool hasOrder() const; inline ::capnp::Text::Reader getOrder() const; + inline bool hasEnumerationName() const; + inline ::capnp::Text::Reader getEnumerationName() const; + private: ::capnp::_::StructReader _reader; template @@ -3131,6 +3253,13 @@ class Attribute::Builder { inline void adoptOrder(::capnp::Orphan<::capnp::Text>&& value); inline ::capnp::Orphan<::capnp::Text> disownOrder(); + inline bool hasEnumerationName(); + inline ::capnp::Text::Builder getEnumerationName(); + inline void setEnumerationName(::capnp::Text::Reader value); + inline ::capnp::Text::Builder initEnumerationName(unsigned int size); + inline void adoptEnumerationName(::capnp::Orphan<::capnp::Text>&& value); + inline ::capnp::Orphan<::capnp::Text> disownEnumerationName(); + private: ::capnp::_::StructBuilder _builder; template @@ -3163,6 +3292,153 @@ class Attribute::Pipeline { }; #endif // !CAPNP_LITE +class Enumeration::Reader { + public: + typedef Enumeration Reads; + + Reader() = default; + inline explicit Reader(::capnp::_::StructReader base) + : _reader(base) { + } + + inline ::capnp::MessageSize totalSize() const { + return _reader.totalSize().asPublic(); + } + +#if !CAPNP_LITE + inline ::kj::StringTree toString() const { + return ::capnp::_::structString(_reader, *_capnpPrivate::brand()); + } +#endif // !CAPNP_LITE + + inline bool hasName() const; + inline ::capnp::Text::Reader getName() const; + + inline bool hasPathName() const; + inline ::capnp::Text::Reader getPathName() const; + + inline bool hasType() const; + inline ::capnp::Text::Reader getType() const; + + inline ::uint32_t getCellValNum() const; + + inline bool getOrdered() const; + + inline bool hasData() const; + inline ::capnp::Data::Reader getData() const; + + inline bool hasOffsets() const; + inline ::capnp::Data::Reader getOffsets() const; + + private: + ::capnp::_::StructReader _reader; + template + friend struct ::capnp::ToDynamic_; + template + friend struct ::capnp::_::PointerHelpers; + template + friend struct ::capnp::List; + friend class ::capnp::MessageBuilder; + friend class ::capnp::Orphanage; +}; + +class Enumeration::Builder { + public: + typedef Enumeration Builds; + + Builder() = delete; // Deleted to discourage incorrect usage. + // You can explicitly initialize to nullptr instead. + inline Builder(decltype(nullptr)) { + } + inline explicit Builder(::capnp::_::StructBuilder base) + : _builder(base) { + } + inline operator Reader() const { + return Reader(_builder.asReader()); + } + inline Reader asReader() const { + return *this; + } + + inline ::capnp::MessageSize totalSize() const { + return asReader().totalSize(); + } +#if !CAPNP_LITE + inline ::kj::StringTree toString() const { + return asReader().toString(); + } +#endif // !CAPNP_LITE + + inline bool hasName(); + inline ::capnp::Text::Builder getName(); + inline void setName(::capnp::Text::Reader value); + inline ::capnp::Text::Builder initName(unsigned int size); + inline void adoptName(::capnp::Orphan<::capnp::Text>&& value); + inline ::capnp::Orphan<::capnp::Text> disownName(); + + inline bool hasPathName(); + inline ::capnp::Text::Builder getPathName(); + inline void setPathName(::capnp::Text::Reader value); + inline ::capnp::Text::Builder initPathName(unsigned int size); + inline void adoptPathName(::capnp::Orphan<::capnp::Text>&& value); + inline ::capnp::Orphan<::capnp::Text> disownPathName(); + + inline bool hasType(); + inline ::capnp::Text::Builder getType(); + inline void setType(::capnp::Text::Reader value); + inline ::capnp::Text::Builder initType(unsigned int size); + inline void adoptType(::capnp::Orphan<::capnp::Text>&& value); + inline ::capnp::Orphan<::capnp::Text> disownType(); + + inline ::uint32_t getCellValNum(); + inline void setCellValNum(::uint32_t value); + + inline bool getOrdered(); + inline void setOrdered(bool value); + + inline bool hasData(); + inline ::capnp::Data::Builder getData(); + inline void setData(::capnp::Data::Reader value); + inline ::capnp::Data::Builder initData(unsigned int size); + inline void adoptData(::capnp::Orphan<::capnp::Data>&& value); + inline ::capnp::Orphan<::capnp::Data> disownData(); + + inline bool hasOffsets(); + inline ::capnp::Data::Builder getOffsets(); + inline void setOffsets(::capnp::Data::Reader value); + inline ::capnp::Data::Builder initOffsets(unsigned int size); + inline void adoptOffsets(::capnp::Orphan<::capnp::Data>&& value); + inline ::capnp::Orphan<::capnp::Data> disownOffsets(); + + private: + ::capnp::_::StructBuilder _builder; + template + friend struct ::capnp::ToDynamic_; + friend class ::capnp::Orphanage; + template + friend struct ::capnp::_::PointerHelpers; +}; + +#if !CAPNP_LITE +class Enumeration::Pipeline { + public: + typedef Enumeration Pipelines; + + inline Pipeline(decltype(nullptr)) + : _typeless(nullptr) { + } + inline explicit Pipeline(::capnp::AnyPointer::Pipeline&& typeless) + : _typeless(kj::mv(typeless)) { + } + + private: + ::capnp::AnyPointer::Pipeline _typeless; + friend class ::capnp::PipelineHook; + template + friend struct ::capnp::ToDynamic_; +}; +#endif // !CAPNP_LITE + class AttributeBufferHeader::Reader { public: typedef AttributeBufferHeader Reads; @@ -7193,6 +7469,8 @@ class ConditionClause::Reader { inline bool hasOp() const; inline ::capnp::Text::Reader getOp() const; + inline bool getUseEnumeration() const; + private: ::capnp::_::StructReader _reader; template @@ -7253,6 +7531,9 @@ class ConditionClause::Builder { inline void adoptOp(::capnp::Orphan<::capnp::Text>&& value); inline ::capnp::Orphan<::capnp::Text> disownOp(); + inline bool getUseEnumeration(); + inline void setUseEnumeration(bool value); + private: ::capnp::_::StructBuilder _builder; template @@ -7321,6 +7602,8 @@ class ASTNode::Reader { inline bool hasCombinationOp() const; inline ::capnp::Text::Reader getCombinationOp() const; + inline bool getUseEnumeration() const; + private: ::capnp::_::StructReader _reader; template @@ -7411,6 +7694,9 @@ class ASTNode::Builder { inline void adoptCombinationOp(::capnp::Orphan<::capnp::Text>&& value); inline ::capnp::Orphan<::capnp::Text> disownCombinationOp(); + inline bool getUseEnumeration(); + inline void setUseEnumeration(bool value); + private: ::capnp::_::StructBuilder _builder; template @@ -15475,54 +15761,199 @@ ArraySchema::Builder::disownDimensionLabels() { ::capnp::POINTERS)); } -inline ::uint32_t DimensionLabel::Reader::getDimensionId() const { - return _reader.getDataField<::uint32_t>( - ::capnp::bounded<0>() * ::capnp::ELEMENTS); -} - -inline ::uint32_t DimensionLabel::Builder::getDimensionId() { - return _builder.getDataField<::uint32_t>( - ::capnp::bounded<0>() * ::capnp::ELEMENTS); -} -inline void DimensionLabel::Builder::setDimensionId(::uint32_t value) { - _builder.setDataField<::uint32_t>( - ::capnp::bounded<0>() * ::capnp::ELEMENTS, value); -} - -inline bool DimensionLabel::Reader::hasName() const { - return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) +inline bool ArraySchema::Reader::hasEnumerations() const { + return !_reader.getPointerField(::capnp::bounded<13>() * ::capnp::POINTERS) .isNull(); } -inline bool DimensionLabel::Builder::hasName() { - return !_builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) +inline bool ArraySchema::Builder::hasEnumerations() { + return !_builder.getPointerField(::capnp::bounded<13>() * ::capnp::POINTERS) .isNull(); } -inline ::capnp::Text::Reader DimensionLabel::Reader::getName() const { - return ::capnp::_::PointerHelpers<::capnp::Text>::get( - _reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); -} -inline ::capnp::Text::Builder DimensionLabel::Builder::getName() { - return ::capnp::_::PointerHelpers<::capnp::Text>::get( - _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader +ArraySchema::Reader::getEnumerations() const { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>::get(_reader + .getPointerField( + ::capnp::bounded<13>() * + ::capnp::POINTERS)); } -inline void DimensionLabel::Builder::setName(::capnp::Text::Reader value) { - ::capnp::_::PointerHelpers<::capnp::Text>::set( - _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), - value); +inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder +ArraySchema::Builder::getEnumerations() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>::get(_builder + .getPointerField( + ::capnp::bounded<13>() * + ::capnp::POINTERS)); } -inline ::capnp::Text::Builder DimensionLabel::Builder::initName( - unsigned int size) { - return ::capnp::_::PointerHelpers<::capnp::Text>::init( - _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), - size); +inline void ArraySchema::Builder::setEnumerations( + ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>:: + set(_builder.getPointerField(::capnp::bounded<13>() * ::capnp::POINTERS), + value); } -inline void DimensionLabel::Builder::adoptName( - ::capnp::Orphan<::capnp::Text>&& value) { - ::capnp::_::PointerHelpers<::capnp::Text>::adopt( - _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), - kj::mv(value)); +inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder +ArraySchema::Builder::initEnumerations(unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>:: + init( + _builder.getPointerField(::capnp::bounded<13>() * ::capnp::POINTERS), + size); } -inline ::capnp::Orphan<::capnp::Text> DimensionLabel::Builder::disownName() { +inline void ArraySchema::Builder::adoptEnumerations( + ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>&& value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>:: + adopt( + _builder.getPointerField(::capnp::bounded<13>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>> +ArraySchema::Builder::disownEnumerations() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>::disown(_builder + .getPointerField( + ::capnp::bounded<13>() * + ::capnp::POINTERS)); +} + +inline bool ArraySchema::Reader::hasEnumerationPathMap() const { + return !_reader.getPointerField(::capnp::bounded<14>() * ::capnp::POINTERS) + .isNull(); +} +inline bool ArraySchema::Builder::hasEnumerationPathMap() { + return !_builder.getPointerField(::capnp::bounded<14>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp:: + List<::tiledb::sm::serialization::capnp::KV, ::capnp::Kind::STRUCT>::Reader + ArraySchema::Reader::getEnumerationPathMap() const { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>>::get(_reader + .getPointerField( + ::capnp::bounded<14>() * + ::capnp::POINTERS)); +} +inline ::capnp:: + List<::tiledb::sm::serialization::capnp::KV, ::capnp::Kind::STRUCT>::Builder + ArraySchema::Builder::getEnumerationPathMap() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>>::get(_builder + .getPointerField( + ::capnp::bounded<14>() * + ::capnp::POINTERS)); +} +inline void ArraySchema::Builder::setEnumerationPathMap( + ::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>::Reader value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>>:: + set(_builder.getPointerField(::capnp::bounded<14>() * ::capnp::POINTERS), + value); +} +inline ::capnp:: + List<::tiledb::sm::serialization::capnp::KV, ::capnp::Kind::STRUCT>::Builder + ArraySchema::Builder::initEnumerationPathMap(unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>>:: + init( + _builder.getPointerField(::capnp::bounded<14>() * ::capnp::POINTERS), + size); +} +inline void ArraySchema::Builder::adoptEnumerationPathMap( + ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>>&& value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>>:: + adopt( + _builder.getPointerField(::capnp::bounded<14>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>> +ArraySchema::Builder::disownEnumerationPathMap() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>>::disown(_builder + .getPointerField( + ::capnp::bounded<14>() * + ::capnp::POINTERS)); +} + +inline ::uint32_t DimensionLabel::Reader::getDimensionId() const { + return _reader.getDataField<::uint32_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS); +} + +inline ::uint32_t DimensionLabel::Builder::getDimensionId() { + return _builder.getDataField<::uint32_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS); +} +inline void DimensionLabel::Builder::setDimensionId(::uint32_t value) { + _builder.setDataField<::uint32_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS, value); +} + +inline bool DimensionLabel::Reader::hasName() const { + return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) + .isNull(); +} +inline bool DimensionLabel::Builder::hasName() { + return !_builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::Text::Reader DimensionLabel::Reader::getName() const { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder DimensionLabel::Builder::getName() { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline void DimensionLabel::Builder::setName(::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers<::capnp::Text>::set( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + value); +} +inline ::capnp::Text::Builder DimensionLabel::Builder::initName( + unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::Text>::init( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + size); +} +inline void DimensionLabel::Builder::adoptName( + ::capnp::Orphan<::capnp::Text>&& value) { + ::capnp::_::PointerHelpers<::capnp::Text>::adopt( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::Text> DimensionLabel::Builder::disownName() { return ::capnp::_::PointerHelpers<::capnp::Text>::disown( _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); } @@ -15962,6 +16393,136 @@ ArraySchemaEvolution::Builder::disownTimestampRange() { _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS)); } +inline bool ArraySchemaEvolution::Reader::hasEnumerationsToAdd() const { + return !_reader.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS) + .isNull(); +} +inline bool ArraySchemaEvolution::Builder::hasEnumerationsToAdd() { + return !_builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader +ArraySchemaEvolution::Reader::getEnumerationsToAdd() const { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>::get(_reader + .getPointerField( + ::capnp::bounded<3>() * + ::capnp::POINTERS)); +} +inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder +ArraySchemaEvolution::Builder::getEnumerationsToAdd() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>::get(_builder + .getPointerField( + ::capnp::bounded<3>() * + ::capnp::POINTERS)); +} +inline void ArraySchemaEvolution::Builder::setEnumerationsToAdd( + ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>:: + set(_builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS), + value); +} +inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder +ArraySchemaEvolution::Builder::initEnumerationsToAdd(unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>:: + init( + _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS), + size); +} +inline void ArraySchemaEvolution::Builder::adoptEnumerationsToAdd( + ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>&& value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>:: + adopt( + _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>> +ArraySchemaEvolution::Builder::disownEnumerationsToAdd() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>::disown(_builder + .getPointerField( + ::capnp::bounded<3>() * + ::capnp::POINTERS)); +} + +inline bool ArraySchemaEvolution::Reader::hasEnumerationsToDrop() const { + return !_reader.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS) + .isNull(); +} +inline bool ArraySchemaEvolution::Builder::hasEnumerationsToDrop() { + return !_builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Reader +ArraySchemaEvolution::Reader::getEnumerationsToDrop() const { + return ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::get( + _reader.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS)); +} +inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Builder +ArraySchemaEvolution::Builder::getEnumerationsToDrop() { + return ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::get( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS)); +} +inline void ArraySchemaEvolution::Builder::setEnumerationsToDrop( + ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Reader value) { + ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::set( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS), + value); +} +inline void ArraySchemaEvolution::Builder::setEnumerationsToDrop( + ::kj::ArrayPtr value) { + ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::set( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS), + value); +} +inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Builder +ArraySchemaEvolution::Builder::initEnumerationsToDrop(unsigned int size) { + return ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::init( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS), + size); +} +inline void ArraySchemaEvolution::Builder::adoptEnumerationsToDrop( + ::capnp::Orphan<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>&& + value) { + ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::adopt( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>> +ArraySchemaEvolution::Builder::disownEnumerationsToDrop() { + return ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::disown( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS)); +} + inline ::uint32_t Attribute::Reader::getCellValNum() const { return _reader.getDataField<::uint32_t>( ::capnp::bounded<0>() * ::capnp::ELEMENTS); @@ -16207,6 +16768,263 @@ inline ::capnp::Orphan<::capnp::Text> Attribute::Builder::disownOrder() { _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS)); } +inline bool Attribute::Reader::hasEnumerationName() const { + return !_reader.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS) + .isNull(); +} +inline bool Attribute::Builder::hasEnumerationName() { + return !_builder.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::Text::Reader Attribute::Reader::getEnumerationName() const { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _reader.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder Attribute::Builder::getEnumerationName() { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _builder.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS)); +} +inline void Attribute::Builder::setEnumerationName( + ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers<::capnp::Text>::set( + _builder.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS), + value); +} +inline ::capnp::Text::Builder Attribute::Builder::initEnumerationName( + unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::Text>::init( + _builder.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS), + size); +} +inline void Attribute::Builder::adoptEnumerationName( + ::capnp::Orphan<::capnp::Text>&& value) { + ::capnp::_::PointerHelpers<::capnp::Text>::adopt( + _builder.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::Text> +Attribute::Builder::disownEnumerationName() { + return ::capnp::_::PointerHelpers<::capnp::Text>::disown( + _builder.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS)); +} + +inline bool Enumeration::Reader::hasName() const { + return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) + .isNull(); +} +inline bool Enumeration::Builder::hasName() { + return !_builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::Text::Reader Enumeration::Reader::getName() const { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder Enumeration::Builder::getName() { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline void Enumeration::Builder::setName(::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers<::capnp::Text>::set( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + value); +} +inline ::capnp::Text::Builder Enumeration::Builder::initName( + unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::Text>::init( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + size); +} +inline void Enumeration::Builder::adoptName( + ::capnp::Orphan<::capnp::Text>&& value) { + ::capnp::_::PointerHelpers<::capnp::Text>::adopt( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::Text> Enumeration::Builder::disownName() { + return ::capnp::_::PointerHelpers<::capnp::Text>::disown( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} + +inline bool Enumeration::Reader::hasPathName() const { + return !_reader.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS) + .isNull(); +} +inline bool Enumeration::Builder::hasPathName() { + return !_builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::Text::Reader Enumeration::Reader::getPathName() const { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _reader.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder Enumeration::Builder::getPathName() { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); +} +inline void Enumeration::Builder::setPathName(::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers<::capnp::Text>::set( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), + value); +} +inline ::capnp::Text::Builder Enumeration::Builder::initPathName( + unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::Text>::init( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), + size); +} +inline void Enumeration::Builder::adoptPathName( + ::capnp::Orphan<::capnp::Text>&& value) { + ::capnp::_::PointerHelpers<::capnp::Text>::adopt( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::Text> Enumeration::Builder::disownPathName() { + return ::capnp::_::PointerHelpers<::capnp::Text>::disown( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); +} + +inline bool Enumeration::Reader::hasType() const { + return !_reader.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS) + .isNull(); +} +inline bool Enumeration::Builder::hasType() { + return !_builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::Text::Reader Enumeration::Reader::getType() const { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _reader.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder Enumeration::Builder::getType() { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS)); +} +inline void Enumeration::Builder::setType(::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers<::capnp::Text>::set( + _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS), + value); +} +inline ::capnp::Text::Builder Enumeration::Builder::initType( + unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::Text>::init( + _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS), + size); +} +inline void Enumeration::Builder::adoptType( + ::capnp::Orphan<::capnp::Text>&& value) { + ::capnp::_::PointerHelpers<::capnp::Text>::adopt( + _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::Text> Enumeration::Builder::disownType() { + return ::capnp::_::PointerHelpers<::capnp::Text>::disown( + _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS)); +} + +inline ::uint32_t Enumeration::Reader::getCellValNum() const { + return _reader.getDataField<::uint32_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS); +} + +inline ::uint32_t Enumeration::Builder::getCellValNum() { + return _builder.getDataField<::uint32_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS); +} +inline void Enumeration::Builder::setCellValNum(::uint32_t value) { + _builder.setDataField<::uint32_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS, value); +} + +inline bool Enumeration::Reader::getOrdered() const { + return _reader.getDataField(::capnp::bounded<32>() * ::capnp::ELEMENTS); +} + +inline bool Enumeration::Builder::getOrdered() { + return _builder.getDataField( + ::capnp::bounded<32>() * ::capnp::ELEMENTS); +} +inline void Enumeration::Builder::setOrdered(bool value) { + _builder.setDataField( + ::capnp::bounded<32>() * ::capnp::ELEMENTS, value); +} + +inline bool Enumeration::Reader::hasData() const { + return !_reader.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS) + .isNull(); +} +inline bool Enumeration::Builder::hasData() { + return !_builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::Data::Reader Enumeration::Reader::getData() const { + return ::capnp::_::PointerHelpers<::capnp::Data>::get( + _reader.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS)); +} +inline ::capnp::Data::Builder Enumeration::Builder::getData() { + return ::capnp::_::PointerHelpers<::capnp::Data>::get( + _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS)); +} +inline void Enumeration::Builder::setData(::capnp::Data::Reader value) { + ::capnp::_::PointerHelpers<::capnp::Data>::set( + _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS), + value); +} +inline ::capnp::Data::Builder Enumeration::Builder::initData( + unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::Data>::init( + _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS), + size); +} +inline void Enumeration::Builder::adoptData( + ::capnp::Orphan<::capnp::Data>&& value) { + ::capnp::_::PointerHelpers<::capnp::Data>::adopt( + _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::Data> Enumeration::Builder::disownData() { + return ::capnp::_::PointerHelpers<::capnp::Data>::disown( + _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS)); +} + +inline bool Enumeration::Reader::hasOffsets() const { + return !_reader.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS) + .isNull(); +} +inline bool Enumeration::Builder::hasOffsets() { + return !_builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::Data::Reader Enumeration::Reader::getOffsets() const { + return ::capnp::_::PointerHelpers<::capnp::Data>::get( + _reader.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS)); +} +inline ::capnp::Data::Builder Enumeration::Builder::getOffsets() { + return ::capnp::_::PointerHelpers<::capnp::Data>::get( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS)); +} +inline void Enumeration::Builder::setOffsets(::capnp::Data::Reader value) { + ::capnp::_::PointerHelpers<::capnp::Data>::set( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS), + value); +} +inline ::capnp::Data::Builder Enumeration::Builder::initOffsets( + unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::Data>::init( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS), + size); +} +inline void Enumeration::Builder::adoptOffsets( + ::capnp::Orphan<::capnp::Data>&& value) { + ::capnp::_::PointerHelpers<::capnp::Data>::adopt( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::Data> Enumeration::Builder::disownOffsets() { + return ::capnp::_::PointerHelpers<::capnp::Data>::disown( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS)); +} + inline bool AttributeBufferHeader::Reader::hasName() const { return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) .isNull(); @@ -20752,6 +21570,17 @@ inline ::capnp::Orphan<::capnp::Text> ConditionClause::Builder::disownOp() { _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS)); } +inline bool ConditionClause::Reader::getUseEnumeration() const { + return _reader.getDataField(::capnp::bounded<0>() * ::capnp::ELEMENTS); +} + +inline bool ConditionClause::Builder::getUseEnumeration() { + return _builder.getDataField(::capnp::bounded<0>() * ::capnp::ELEMENTS); +} +inline void ConditionClause::Builder::setUseEnumeration(bool value) { + _builder.setDataField(::capnp::bounded<0>() * ::capnp::ELEMENTS, value); +} + inline bool ASTNode::Reader::getIsExpression() const { return _reader.getDataField(::capnp::bounded<0>() * ::capnp::ELEMENTS); } @@ -20986,6 +21815,17 @@ inline ::capnp::Orphan<::capnp::Text> ASTNode::Builder::disownCombinationOp() { _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS)); } +inline bool ASTNode::Reader::getUseEnumeration() const { + return _reader.getDataField(::capnp::bounded<1>() * ::capnp::ELEMENTS); +} + +inline bool ASTNode::Builder::getUseEnumeration() { + return _builder.getDataField(::capnp::bounded<1>() * ::capnp::ELEMENTS); +} +inline void ASTNode::Builder::setUseEnumeration(bool value) { + _builder.setDataField(::capnp::bounded<1>() * ::capnp::ELEMENTS, value); +} + inline bool Condition::Reader::hasClauses() const { return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) .isNull(); diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index 5b89809704a1..369ecb4a8047 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -46,6 +46,7 @@ #include "tiledb/sm/array/array_directory.h" #include "tiledb/sm/array_schema/array_schema.h" #include "tiledb/sm/array_schema/array_schema_evolution.h" +#include "tiledb/sm/array_schema/enumeration.h" #include "tiledb/sm/consolidator/consolidator.h" #include "tiledb/sm/consolidator/fragment_consolidator.h" #include "tiledb/sm/enums/array_type.h" @@ -630,6 +631,11 @@ Status StorageManagerCanonical::array_create( array_uri.join_path(constants::array_schema_dir_name); RETURN_NOT_OK(vfs()->create_dir(array_schema_dir_uri)); + // Create the enumerations directory inside the array schema directory + URI array_enumerations_uri = + array_schema_dir_uri.join_path(constants::array_enumerations_dir_name); + RETURN_NOT_OK(vfs()->create_dir(array_enumerations_uri)); + // Create commit directory URI array_commit_uri = array_uri.join_path(constants::array_commits_dir_name); RETURN_NOT_OK(vfs()->create_dir(array_commit_uri)); @@ -736,11 +742,9 @@ Status StorageManager::array_evolve_schema( auto&& array_schema = array_dir.load_array_schema_latest(encryption_key); // Evolve schema - auto&& [st1, array_schema_evolved] = - schema_evolution->evolve_schema(array_schema); - RETURN_NOT_OK(st1); + auto array_schema_evolved = schema_evolution->evolve_schema(array_schema); - Status st = store_array_schema(array_schema_evolved.value(), encryption_key); + Status st = store_array_schema(array_schema_evolved, encryption_key); if (!st.ok()) { logger_->status_no_return_value(st); return logger_->status(Status_StorageManagerError( @@ -1692,11 +1696,46 @@ Status StorageManagerCanonical::store_array_schema( URI array_schema_dir_uri = array_schema->array_uri().join_path(constants::array_schema_dir_name); RETURN_NOT_OK(vfs()->is_dir(array_schema_dir_uri, &schema_dir_exists)); + if (!schema_dir_exists) RETURN_NOT_OK(vfs()->create_dir(array_schema_dir_uri)); RETURN_NOT_OK(store_data_to_generic_tile(tile, schema_uri, encryption_key)); + // Create the `__enumerations` directory under `__schema` if it doesn't + // exist. This might happen if someone tries to add an enumeration to an + // array created before version 19. + bool enumerations_dir_exists = false; + URI array_enumerations_dir_uri = + array_schema_dir_uri.join_path(constants::array_enumerations_dir_name); + RETURN_NOT_OK( + vfs()->is_dir(array_enumerations_dir_uri, &enumerations_dir_exists)); + + if (!enumerations_dir_exists) { + RETURN_NOT_OK(vfs()->create_dir(array_enumerations_dir_uri)); + } + + // Serialize all enumerations into the `__enumerations` directory + for (auto& enmr_name : array_schema->get_loaded_enumeration_names()) { + auto enmr = array_schema->get_enumeration(enmr_name); + if (enmr == nullptr) { + return logger_->status(Status_StorageManagerError( + "Error serializing enumeration; Loaded enumeration is null")); + } + + SizeComputationSerializer enumeration_size_serializer; + enmr->serialize(enumeration_size_serializer); + + WriterTile tile{ + WriterTile::from_generic(enumeration_size_serializer.size())}; + Serializer serializer(tile.data(), tile.size()); + enmr->serialize(serializer); + + auto abs_enmr_uri = array_enumerations_dir_uri.join_path(enmr->path_name()); + RETURN_NOT_OK( + store_data_to_generic_tile(tile, abs_enmr_uri, encryption_key)); + } + return Status::Ok(); } diff --git a/tiledb/sm/tile/CMakeLists.txt b/tiledb/sm/tile/CMakeLists.txt index c23f54bec7ff..f52ea816f9b0 100644 --- a/tiledb/sm/tile/CMakeLists.txt +++ b/tiledb/sm/tile/CMakeLists.txt @@ -44,7 +44,6 @@ commence(object_library generic_tile_io) baseline buffer constants - context_resources tiledb_crypto filter_pipeline tile @@ -52,4 +51,10 @@ commence(object_library generic_tile_io) ) conclude(object_library) +# This is linked outside the object_library scope because ContextResources +# is recompiled as part of the capi_context_stub. Including context_resources +# here like this prevents many headaches revolving around duplicate symbols +# when linking executables. +target_link_libraries(compile_generic_tile_io PRIVATE context_resources) + add_test_subdirectory() diff --git a/tiledb/storage_format/uri/CMakeLists.txt b/tiledb/storage_format/uri/CMakeLists.txt index 6923c26548d7..4dbd4dd3f319 100644 --- a/tiledb/storage_format/uri/CMakeLists.txt +++ b/tiledb/storage_format/uri/CMakeLists.txt @@ -30,6 +30,6 @@ include(object_library) # `uri_format` object library # commence(object_library uri_format) - this_target_sources(parse_uri.cc) + this_target_sources(parse_uri.cc generate_uri.cc) this_target_object_libraries(baseline time uuid vfs) conclude(object_library)