Merge branch 'dev' into yt/sc-48774/fragment_list_consolidation

format_spec/array_schema.md

@@ -54,6 +54,15 @@ The array schema file consists of a single [generic tile](./generic_tile.md), wi
 | Label 1 | [Dimension Label](#dimension_label) | First dimension label |
 | … | … | … |
 | Label N | [Dimension Label](#dimension_label) | Nth dimension label |
+| Num enumerations | `uint32_t` | Number of [enumerations](./enumeration.md) in the array |
+| Enumeration name length 1 | `uint32_t` | The number of characters in the enumeration 1 name |
+| Enumeration name 1 | `uint8_t[]` | The name of enumeration 1 |
+| Enumeration filename length 1 | `uint32_t` | The number of characters in the enumeration 1 file |
+| Enumeration filename 1 | `uint8_t[]` | The name of the file in the `__enumerations` subdirectory that conatins enumeration 1's data |
+| Enumeration name length N | `uint32_t` | The number of characters in the enumeration N name |
+| Enumeration name N | `uint8_t[]` | The name of enumeration N |
+| Enumeration filename length N | `uint32_t` | The number of characters in the enumeration N file |
+| Enumeration filename N | `uint8_t[]` | The name of the file in the `__enumerations` subdirectory that conatins enumeration N's data |
 | CurrentDomain | [CurrentDomain](./current_domain.md) | The array current domain |
 
 ## Domain

test/src/unit-Reader.cc

@@ -158,6 +158,7 @@ TEST_CASE_METHOD(
   uint64_t tmp_size = 0;
   Config config;
   Context context(config);
+  LocalQueryStateMachine lq_state_machine{LocalQueryState::uninitialized};
   std::unordered_map<std::string, tiledb::sm::QueryBuffer> buffers;
   buffers.emplace(
       "a", tiledb::sm::QueryBuffer(nullptr, nullptr, &tmp_size, &tmp_size));
@@ -173,7 +174,8 @@ TEST_CASE_METHOD(
       context.resources(),
       array.memory_tracker(),
       tracker_,
-      context.storage_manager(),
+      lq_state_machine,
+      CancellationSource(context.storage_manager()),
       array.opened_array(),
       config,
       nullopt,

test/src/unit-capi-incomplete.cc

@@ -1145,7 +1145,7 @@ void IncompleteFx::check_sparse_unsplittable_complete() {
 TEST_CASE_METHOD(
     IncompleteFx,
     "C API: Test incomplete read queries, dense",
-    "[capi][incomplete][dense-incomplete][serialization][rest]") {
+    "[capi][incomplete][dense-incomplete][rest]") {
   create_dense_array();
   write_dense_full();
   check_dense_incomplete();
@@ -1159,7 +1159,7 @@ TEST_CASE_METHOD(
 TEST_CASE_METHOD(
     IncompleteFx,
     "C API: Test incomplete read queries, sparse",
-    "[capi][incomplete][sparse][serialization][rest]") {
+    "[capi][incomplete][sparse][rest]") {
   create_sparse_array();
   write_sparse_full();
   check_sparse_incomplete();
@@ -1171,8 +1171,47 @@ TEST_CASE_METHOD(
 TEST_CASE_METHOD(
     IncompleteFx,
     "C API: Test incomplete read queries, dense, serialized",
-    "[capi][incomplete][dense][serialization][rest]") {
+    "[capi][incomplete][dense][rest]") {
   create_dense_array();
   write_dense_full();
   check_dense_incomplete();
 }
+
+TEST_CASE_METHOD(
+    IncompleteFx,
+    "C API: Test incomplete read queries, sparse",
+    "[capi][incomplete][sparse][retries][sc-49128][rest]") {
+  // This test is testing CURL logic and only makes sense on REST-CI
+  if (!vfs_test_setup_.is_rest()) {
+    return;
+  }
+
+  // Force retries on successful requests to test that the buffers
+  // resetting in the retry logic works well
+  tiledb_config_t* cfg;
+  tiledb_error_t* err = nullptr;
+  int rc = tiledb_config_alloc(&cfg, &err);
+  REQUIRE(rc == TILEDB_OK);
+  REQUIRE(err == nullptr);
+  rc = tiledb_config_set(cfg, "rest.retry_http_codes", "200", &err);
+  REQUIRE(rc == TILEDB_OK);
+  REQUIRE(err == nullptr);
+  rc = tiledb_config_set(cfg, "rest.retry_count", "1", &err);
+  REQUIRE(rc == TILEDB_OK);
+  REQUIRE(err == nullptr);
+  rc = tiledb_config_set(cfg, "rest.retry_initial_delay_ms", "5", &err);
+  REQUIRE(rc == TILEDB_OK);
+  REQUIRE(err == nullptr);
+
+  // Update the context with config
+  vfs_test_setup_.update_config(cfg);
+  ctx_ = vfs_test_setup_.ctx_c;
+  tiledb_config_free(&cfg);
+
+  create_sparse_array();
+  write_sparse_full();
+  check_sparse_incomplete();
+  check_sparse_until_complete();
+  check_sparse_unsplittable_overflow();
+  check_sparse_unsplittable_complete();
+}

test/src/unit-curl.cc

@@ -43,7 +43,8 @@
 
 using namespace tiledb::sm;
 
-TEST_CASE("CURL: Test curl's header parsing callback", "[curl]") {
+TEST_CASE(
+    "CURL: Test curl's header parsing callback", "[curl][redirect-caching]") {
   // Initialize data that in real life scenario would be initialized by
   // RestClient
   char res_data[] =
@@ -58,6 +59,7 @@ TEST_CASE("CURL: Test curl's header parsing callback", "[curl]") {
   std::unordered_map<std::string, std::string> redirect_meta_;
   userdata.redirect_uri_map = &redirect_meta_;
   userdata.redirect_uri_map_lock = &redirect_mtx_;
+  userdata.should_cache_redirect = true;
 
   size_t result = write_header_callback(&res_data, size, count, &userdata);
 
@@ -94,6 +96,35 @@ TEST_CASE("CURL: Test curl's header parsing callback", "[curl]") {
       "tiledb://my_username");
 }
 
+TEST_CASE(
+    "CURL: Test curl is not caching the region when requested so",
+    "[curl][redirect-caching]") {
+  // Initialize data that in real life scenario would be initialized by
+  // RestClient
+  char res_data[] =
+      "Location: https://test.url.domain/v1/arrays/testns/test_arr";
+  size_t size = 1;
+  size_t count = sizeof(res_data);
+
+  HeaderCbData userdata;
+  std::string ns_array = "testns:test_arr";
+  userdata.uri = &ns_array;
+  std::mutex redirect_mtx_;
+  std::unordered_map<std::string, std::string> redirect_meta_;
+  userdata.redirect_uri_map = &redirect_meta_;
+  userdata.redirect_uri_map_lock = &redirect_mtx_;
+  userdata.should_cache_redirect = false;
+
+  size_t result = write_header_callback(&res_data, size, count, &userdata);
+
+  // The return value should be equal to size * count
+  CHECK(result == size * count);
+  // The uri is not mutated inside the callback function
+  CHECK(*userdata.uri == ns_array);
+  // The redirect map holds a record of ns_array key and redirection url value
+  CHECK(userdata.redirect_uri_map->count(ns_array) == 0);
+}
+
 TEST_CASE(
     "RestClient: Remove trailing slash from rest_server_", "[rest-client]") {
   std::string rest_server =

test/src/unit-enumerations.cc

@@ -2398,10 +2398,18 @@ TEST_CASE_METHOD(
   auto qc1 = create_qc("attr1", (int)2, QueryConditionOp::EQ);
   qc1.set_use_enumeration(false);
 
-  Query q1(ctx_.resources(), ctx_.storage_manager(), array);
+  Query q1(
+      ctx_.resources(),
+      ctx_.cancellation_source(),
+      ctx_.storage_manager(),
+      array);
   throw_if_not_ok(q1.set_condition(qc1));
 
-  Query q2(ctx_.resources(), ctx_.storage_manager(), array);
+  Query q2(
+      ctx_.resources(),
+      ctx_.cancellation_source(),
+      ctx_.storage_manager(),
+      array);
   ser_des_query(&q1, &q2, client_side, ser_type);
 
   auto qc2 = q2.condition();
@@ -2434,10 +2442,18 @@ TEST_CASE_METHOD(
 
   throw_if_not_ok(qc1.combine(qc2, QueryConditionCombinationOp::OR, &qc3));
 
-  Query q1(ctx_.resources(), ctx_.storage_manager(), array);
+  Query q1(
+      ctx_.resources(),
+      ctx_.cancellation_source(),
+      ctx_.storage_manager(),
+      array);
   throw_if_not_ok(q1.set_condition(qc3));
 
-  Query q2(ctx_.resources(), ctx_.storage_manager(), array);
+  Query q2(
+      ctx_.resources(),
+      ctx_.cancellation_source(),
+      ctx_.storage_manager(),
+      array);
   ser_des_query(&q1, &q2, client_side, ser_type);
 
   auto qc4 = q2.condition();

tiledb/CMakeLists.txt

@@ -259,6 +259,7 @@ set(TILEDB_CORE_SOURCES
   ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/query/query.cc
   ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/query/query_condition.cc
   ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/query/query_remote_buffer_storage.cc
+  ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/query/query_state.cc
   ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/query/readers/aggregators/count_aggregator.cc
   ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/query/readers/aggregators/min_max_aggregator.cc
   ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/query/readers/aggregators/operation.cc
@@ -300,6 +301,7 @@ set(TILEDB_CORE_SOURCES
   ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/serialization/vacuum.cc
   ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/stats/global_stats.cc
   ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/stats/stats.cc
+  ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/storage_manager/cancellation_source.cc
   ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/storage_manager/context.cc
   ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/storage_manager/context_resources.cc
   ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/storage_manager/storage_manager.cc

tiledb/api/c_api/context/context_api_internal.h

@@ -36,6 +36,7 @@
 #include "../../c_api_support/handle/handle.h"
 #include "../config/config_api_internal.h"
 #include "../error/error_api_internal.h"
+#include "tiledb/sm/storage_manager/cancellation_source.h"
 #include "tiledb/sm/storage_manager/context.h"
 
 struct tiledb_ctx_handle_t
@@ -71,6 +72,10 @@ struct tiledb_ctx_handle_t
     return ctx_.storage_manager();
   }
 
+  inline tiledb::sm::CancellationSource cancellation_source() {
+    return ctx_.cancellation_source();
+  }
+
   inline tiledb::sm::RestClient& rest_client() {
     return ctx_.rest_client();
   }

tiledb/common/memory_tracker.cc

@@ -54,6 +54,8 @@ std::string memory_type_to_str(MemoryType type) {
       return "Attributes";
     case MemoryType::CONSOLIDATION_BUFFERS:
       return "ConsolidationBuffers";
+    case MemoryType::DENSE_TILE_SUBARRAY:
+      return "DenseTileSubarray";
     case MemoryType::DIMENSION_LABELS:
       return "DimensionLabels";
     case MemoryType::DIMENSIONS:

tiledb/common/memory_tracker.h

@@ -110,6 +110,7 @@ namespace tiledb::sm {
 enum class MemoryType {
   ATTRIBUTES,
   CONSOLIDATION_BUFFERS,
+  DENSE_TILE_SUBARRAY,
   DIMENSION_LABELS,
   DIMENSIONS,
   DOMAINS,

tiledb/sm/array/CMakeLists.txt

@@ -35,17 +35,8 @@ include(object_library)
 # 
 # This design eliminates cyclic dependencies in the build, as each module has
 # its own dependency chain.
-# 
-# Note that neither object library has been implemented at present. Rather,
-# class `Array` is getting segregated by functionality to prepare for this 
-# future work. 
-# 
-# Though there is currently an object library named `array`, it will soon be
-# aptly renamed `array_directory`.
-# 
-# The `array` object library will be added in the near future.
 #
-# The `array_operations` object library is very specifically blocked by 
+# Note: The `array_operations` object library is very specifically blocked by 
 # `StrategyBase::throw_if_cancelled`. At present, this function relies on 
 # `StorageManagerCanonical`, which is slated for removal. The `Query` class is 
 # undergoing maintenance to remove all dependencies on class `StorageManager`. 
@@ -54,15 +45,15 @@ include(object_library)
 #
 
 #
-# `array` object library
+# `array_directory` object library
 #
-commence(object_library array)
+commence(object_library array_directory)
     this_target_sources(array_directory.cc)
     this_target_object_libraries(
         array_schema
         baseline
         context_resources
-        fragment
+        fragment_identifier
         generic_tile_io
         vfs
     )
@@ -71,4 +62,18 @@ commence(object_library array)
     endif()
 conclude(object_library)
 
+#
+# `array` object library
+#
+commence(object_library array)
+    this_target_sources(array.cc consistency.cc)
+    this_target_object_libraries(
+        array_directory
+        array_schema_operations
+        fragment_metadata
+        metadata
+        object
+    )
+conclude(object_library)
+
 add_test_subdirectory()

tiledb/sm/array/array.h

@@ -52,7 +52,6 @@ namespace tiledb::sm {
 
 class ArraySchema;
 class ArraySchemaEvolution;
-class SchemaEvolution;
 class FragmentMetadata;
 class MemoryTracker;
 enum class QueryType : uint8_t;

tiledb/sm/array/test/CMakeLists.txt

@@ -3,7 +3,7 @@
 #
 # The MIT License
 #
-# Copyright (c) 2022 TileDB, Inc.
+# Copyright (c) 2022-2024 TileDB, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -29,7 +29,7 @@ include(unit_test)
 commence(unit_test array)
     this_target_compile_definitions(-DTILEDB_TEST_INPUTS_DIR="${CMAKE_SOURCE_DIR}/test/inputs/")
     this_target_sources(main.cc unit_array_directory.cc)
-    this_target_link_libraries(array)
+    this_target_link_libraries(array_directory)
 conclude(unit_test)
 
 commence(unit_test consistency)

tiledb/sm/array/test/compile_array_directory_main.cc

@@ -0,0 +1,45 @@
+/**
+ * @file compile_array_directory_main.cc
+ *
+ * @section LICENSE
+ *
+ * The MIT License
+ *
+ * @copyright Copyright (c) 2022-2024 TileDB, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "../array_directory.h"
+
+#include "tiledb/common/logger.h"
+#include "tiledb/sm/storage_manager/context_resources.h"
+
+using namespace tiledb::sm;
+
+int main() {
+  Config config;
+  auto logger = make_shared<Logger>(HERE(), "foo");
+  ContextResources resources(config, logger, 1, 1, "");
+
+  ArrayDirectory x(resources, URI{});
+  (void)x.loaded();
+
+  return 0;
+}