Add Enumerated Data Types

This PR adds the Enumerated data types. Enumerated data types work by
adding an Enumeration to the ArraySchema, setting an enumeration name on
an attribute, and then adding the attribute to the ArraySchema.

An Enumeration object contains a short list of options and a vector of
values. An attribute that has an enumeration name set must have an
integral type that is wide enough to index all of the enumerated values.

Changes to the values of an enumeration (any of adding, renaming, or
removing) can be accomplished via ArraySchemaEvolution.

CMakeLists.txt

@@ -227,7 +227,10 @@ else()
     "$<$<CONFIG:Release,RelWithDebInfo>:-O3>"
     "$<$<CONFIG:RelWithDebInfo>:-g3;-ggdb3;-gdwarf-3>")
   if(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
-    add_compile_options("$<$<CONFIG:Debug>:-fstandalone-debug>")
+    add_compile_options(
+      "$<$<CONFIG:Debug>:-fstandalone-debug>"
+      "$<$<CONFIG:Coverage>:-fprofile-instr-generate;-fcoverage-mapping>")
+    add_link_options("$<$<CONFIG:Coverage>:--coverage;-fprofile-instr-generate;-fcoverage-mapping>")
   endif()
 
   # Disable newer Apple Clang warnings about unqualified calls to std::move
@@ -344,6 +347,7 @@ list(APPEND TILEDB_C_API_RELATIVE_HEADERS
     "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/datatype/datatype_api_external.h"
     "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/dimension/dimension_api_external.h"
     "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/dimension_label/dimension_label_api_external.h"
+    "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/enumeration/enumeration_api_experimental.h"
     "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/error/error_api_external.h"
     "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/filesystem/filesystem_api_enum.h"
     "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/filesystem/filesystem_api_external.h"

PJD_TODO.md

@@ -0,0 +1,9 @@
+Chores Left
+===
+
+* NEED HELP
+  - New REST API endpoint for loading one or more enumerations
+  - Update get_array_schema_from_rest API endpoint to have a "?include_enumerations=true|false" QS parameter.
+
+* Miscellany
+  * Document enumeration format changes in the storage format docs

examples/cpp_api/enumerations.cc

@@ -0,0 +1,160 @@
+/**
+ * @file enumerations.cc
+ *
+ * @section LICENSE
+ *
+ * The MIT License
+ *
+ * @copyright Copyright (c) 2023 TileDB, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ * @section DESCRIPTION
+ *
+ * When run, this program will create a simple 2D sparse array with an
+ * enumeration and then use a query condition to select data based on the
+ * enumerations values.
+ */
+
+#include <iostream>
+#include <tiledb/tiledb>
+#include <tiledb/tiledb_experimental>
+
+using namespace tiledb;
+
+// Name of array.
+std::string array_name("enumerations_example_array");
+
+void create_array() {
+  Context ctx;
+
+  // First some standard boiler plate for creating an array. Nothing here
+  // is important or required for Enumerations.
+  ArraySchema schema(ctx, TILEDB_SPARSE);
+  schema.set_order({{TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR}});
+
+  auto dim1 = Dimension::create<int>(ctx, "rows", {{1, 4}}, 4);
+  auto dim2 = Dimension::create<int>(ctx, "cols", {{1, 4}}, 4);
+
+  Domain dom(ctx);
+  dom.add_dimensions(dim1, dim2);
+  schema.set_domain(dom);
+
+  // The most basic enumeration only requires a name and a vector of values
+  // to use as lookups. Enumeration values can be any supported TileDB type
+  // although they are most commonly strings.
+  std::vector<std::string> values = {"red", "green", "blue"};
+  auto enmr = Enumeration::create(ctx, "colors", values);
+
+  // To use an enumeration with an attribute, we just set the enumeration
+  // name on the attribute before adding it to the schema. Attributes that
+  // use an enumeration are required to have an integral type that is wide
+  // enough to index the entire enumeration. For instance, an enumeration with
+  // 256 values can fit in a uint8_t type, but at 257 values, the attribute
+  // would require a type of int16_t at a minimum.
+  auto attr = Attribute::create<uint8_t>(ctx, "attr");
+  AttributeExperimental::set_enumeration_name(ctx, attr, "colors");
+
+  // The enumeration must be added to the schema before any attribute that
+  // references the enumeration so that the requirements of the attribute
+  // can be accurately checked.
+  ArraySchemaExperimental::add_enumeration(ctx, schema, enmr);
+
+  // Finally, we add the attribute as per normal.
+  schema.add_attribute(attr);
+
+  Array::create(array_name, schema);
+}
+
+void write_array() {
+  Context ctx;
+
+  std::vector<int> row_data = {1, 2, 2};
+  std::vector<int> col_data = {1, 4, 3};
+
+  // Attribute data for an enumeration is just numeric indices into the
+  // list of enumeration values.
+  std::vector<uint8_t> attr_data = {2, 1, 1};
+
+  // Open the array for writing and create the query.
+  Array array(ctx, array_name, TILEDB_WRITE);
+  Query query(ctx, array);
+  query.set_layout(TILEDB_UNORDERED)
+      .set_data_buffer("rows", row_data)
+      .set_data_buffer("cols", col_data)
+      .set_data_buffer("attr", attr_data);
+
+  // Write and close the array
+  query.submit();
+  array.close();
+}
+
+void read_array() {
+  Context ctx;
+
+  // This is all standard boiler plate for reading from an array. The
+  // section below will demonstrate using a QueryCondition to select
+  // rows based on the enumeration.
+  Array array(ctx, array_name, TILEDB_READ);
+  Subarray subarray(ctx, array);
+  subarray.add_range(0, 1, 4).add_range(1, 1, 4);
+
+  std::vector<int> row_data(16);
+  std::vector<int> col_data(16);
+  std::vector<uint8_t> attr_data(16);
+
+  Query query(ctx, array, TILEDB_READ);
+  query.set_subarray(subarray)
+      .set_layout(TILEDB_ROW_MAJOR)
+      .set_data_buffer("rows", row_data)
+      .set_data_buffer("cols", col_data)
+      .set_data_buffer("attr", attr_data);
+
+  // Query conditions apply against the enumeration's values instead of the
+  // integral data. Thus, we can select values here using color names instead
+  // of the integer indices.
+  QueryCondition qc(ctx);
+  qc.init("attr", "green", 5, TILEDB_EQ);
+  query.set_condition(qc);
+
+  // Finally, submit and display the query results
+  query.submit();
+  array.close();
+
+  // Print out the results.
+  auto result_num = (int)query.result_buffer_elements()["attr"].second;
+  for (int i = 0; i < result_num; i++) {
+    int r = row_data[i];
+    int c = col_data[i];
+    int a = attr_data[i];
+    std::cout << "Cell (" << r << ", " << c << ") has attr " << a << "\n";
+  }
+}
+
+int main() {
+  Context ctx;
+
+  if (Object::object(ctx, array_name).type() != Object::Type::Array) {
+    create_array();
+    write_array();
+  }
+
+  read_array();
+  return 0;
+}

format_spec/FORMAT_SPEC.md

@@ -4,7 +4,7 @@ title: Format Specification
 
 **Notes:**
 
-* The current TileDB format version number is **18** (`uint32_t`).
+* The current TileDB format version number is **20** (`uint32_t`).
 * Data written by TileDB and referenced in this document is **little-endian**
   with the following exceptions:
 

format_spec/enumeration.md

@@ -0,0 +1,27 @@
+---
+title: Enumerations
+---
+
+## Main Structure
+
+```
+my_array                                    # array folder
+   |  ...
+   |_ schema                      # ArraySchema directory named `__schema`
+         |_ enumerations          # Enumeration directory named `__enumerations`
+               |_ enumeration     # enumeration data with names `__uuid_v`
+
+
+Enumeration data is stored in a subdirectory of the [array schema][./array_schema.md]
+directory. Enumerations are stored using [Generic Tiles][./generic_tile.md].
+
+| **Field** | **Type** | **Description** |
+| :--- | :--- | :--- |
+| Version number | `uint32_t` | Enumerations version number |
+| Datatype | `uint8_t` | The datatype of the enumeration values |
+| Cell Val Num | `uint32_t` | The cell val num of the enumeration values |
+| Ordered | `bool` | Whether the enumeration values should be considered ordered |
+| Data Size | `uint64_t` | The number of bytes used to store the values |
+| Data | `uint8_t` * Data Size | The data for the enumeration values |
+| Offsets Size | `uint64_t` | The number of bytes used to store offsets if cell_var_num is TILEDB_VAR_NUM |
+| Offsets | `uint8_t` * Offsets Size | The offsets data for the enumeration if cell_var_num is TILEDB_VAR_NUM |

test/CMakeLists.txt

@@ -134,6 +134,7 @@ set(TILEDB_UNIT_TEST_SOURCES
   src/unit-capi-dense_neg.cc
   src/unit-capi-dense_vector.cc
   src/unit-capi-enum_values.cc
+  src/unit-capi-enumerations.cc
   src/unit-capi-error.cc
   src/unit-capi-filestore.cc
   src/unit-capi-fill_values.cc
@@ -172,6 +173,8 @@ set(TILEDB_UNIT_TEST_SOURCES
   src/unit-dimension.cc
   src/unit-duplicates.cc
   src/unit-empty-var-length.cc
+  src/unit-enumerations.cc
+  src/unit-enum-helpers.cc
   src/unit-filter-buffer.cc
   src/unit-filter-pipeline.cc
   src/unit-global-order.cc
@@ -213,6 +216,7 @@ if (TILEDB_CPP_API)
     src/unit-cppapi-deletes.cc
     src/unit-cppapi-dense-qc-coords-mode.cc
     src/unit-cppapi-time.cc
+    src/unit-cppapi-enumerations.cc
     src/unit-cppapi-fill_values.cc
     src/unit-cppapi-filter.cc
     src/unit-cppapi-float-scaling-filter.cc

test/src/unit-capi-array_schema.cc

@@ -959,7 +959,10 @@ int ArraySchemaFx::get_schema_file_struct(const char* path, void* data) {
   int rc = tiledb_vfs_is_dir(ctx, vfs, path, &is_dir);
   CHECK(rc == TILEDB_OK);
 
-  data_struct->path = path;
+  if (!is_dir) {
+    data_struct->path = path;
+  }
+
   return 1;
 }
 

test/src/unit-capi-enumerations.cc

@@ -0,0 +1,119 @@
+/**
+ * @file unit-capi-enumerations.cc
+ *
+ * @section LICENSE
+ *
+ * The MIT License
+ *
+ * @copyright Copyright (c) 2023 TileDB Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ * @section DESCRIPTION
+ *
+ * Tests for the various Enumerations C API errors.
+ */
+
+#include <test/support/tdb_catch.h>
+#include "tiledb/sm/c_api/tiledb.h"
+#include "tiledb/sm/c_api/tiledb_experimental.h"
+
+#include <iostream>
+
+TEST_CASE(
+    "C API: Test invalid attribute for tiledb_attribute_set_enumeration_name",
+    "[enumeration][capi][error]") {
+  tiledb_ctx_t* ctx;
+
+  auto rc = tiledb_ctx_alloc(nullptr, &ctx);
+  CHECK(rc == TILEDB_OK);
+
+  rc = tiledb_attribute_set_enumeration_name(ctx, nullptr, "enmr_name");
+  REQUIRE(rc == TILEDB_ERR);
+}
+
+TEST_CASE(
+    "C API: Test invalid attribute for tiledb_attribute_get_enumeration_name",
+    "[enumeration][capi][error]") {
+  tiledb_ctx_t* ctx;
+
+  auto rc = tiledb_ctx_alloc(nullptr, &ctx);
+  CHECK(rc == TILEDB_OK);
+
+  tiledb_string_t* name;
+  rc = tiledb_attribute_get_enumeration_name(ctx, nullptr, &name);
+  REQUIRE(rc == TILEDB_ERR);
+}
+
+TEST_CASE(
+    "C API: Test invalid array schema for tiledb_array_schema_add_enumeration",
+    "[enumeration][capi][error]") {
+  tiledb_ctx_t* ctx;
+
+  auto rc = tiledb_ctx_alloc(nullptr, &ctx);
+  CHECK(rc == TILEDB_OK);
+
+  tiledb_enumeration_t* enmr;
+  uint32_t values[5] = {1, 2, 3, 4, 5};
+  rc = tiledb_enumeration_alloc(
+      ctx,
+      "an_enumeration",
+      TILEDB_UINT32,
+      1,
+      0,
+      values,
+      sizeof(uint32_t) * 5,
+      nullptr,
+      0,
+      &enmr);
+  REQUIRE(rc == TILEDB_OK);
+
+  rc = tiledb_array_schema_add_enumeration(ctx, nullptr, enmr);
+  REQUIRE(rc == TILEDB_ERR);
+}
+
+TEST_CASE(
+    "C API: Test invalid array for tiledb_array_get_enumeration",
+    "[enumeration][capi][error]") {
+  tiledb_ctx_t* ctx;
+
+  auto rc = tiledb_ctx_alloc(nullptr, &ctx);
+  CHECK(rc == TILEDB_OK);
+
+  tiledb_enumeration_t* enmr;
+  rc = tiledb_array_get_enumeration(ctx, nullptr, "an_enumeration", &enmr);
+  REQUIRE(rc == TILEDB_ERR);
+}
+
+TEST_CASE(
+    "C API: Test invalid enumeration name for tiledb_array_get_enumeration",
+    "[enumeration][capi][error]") {
+  tiledb_ctx_t* ctx;
+
+  auto rc = tiledb_ctx_alloc(nullptr, &ctx);
+  CHECK(rc == TILEDB_OK);
+
+  tiledb_array_t* array;
+  rc = tiledb_array_alloc(ctx, "array_uri", &array);
+  REQUIRE(rc == TILEDB_OK);
+
+  tiledb_enumeration_t* enmr;
+  rc = tiledb_array_get_enumeration(ctx, array, nullptr, &enmr);
+  REQUIRE(rc == TILEDB_ERR);
+}