Add regression test target; Test and fix for SC-17415

.github/workflows/build-macOS11-S3.yml

@@ -100,6 +100,7 @@ jobs:
           source $GITHUB_WORKSPACE/scripts/ci/build_libtiledb.sh
 
           ./tiledb/test/tiledb_unit -d yes | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1'
+          ./tiledb/test/regression/tiledb_regression -d yes | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1'
 
           # Kill the running Minio server, OSX only because Linux runs it within
           # docker.

.github/workflows/build-ubuntu16.04-HDFS.yml

@@ -114,6 +114,7 @@ jobs:
           # Bypass Catch2 Framework stdout interception with awk on test output
           # make check | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1'
           ./tiledb/test/tiledb_unit -d yes | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1'
+          ./tiledb/test/regression/tiledb_regression -d yes | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1'
 
           pushd $GITHUB_WORKSPACE/examples/cmake_project
           mkdir build && cd build

.github/workflows/build-ubuntu20.04-AZURE.yml

@@ -121,6 +121,7 @@ jobs:
           # Bypass Catch2 Framework stdout interception with awk on test output
           # make check | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1'
           ./tiledb/test/tiledb_unit -d yes | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1'
+          ./tiledb/test/regression/tiledb_regression -d yes | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1'
 
           # Kill the running Azurite server
           kill -n 9 $AZURITE_PID

.github/workflows/build-ubuntu20.04-GCS.yml

@@ -108,6 +108,7 @@ jobs:
           # Bypass Catch2 Framework stdout interception with awk on test output
           # make check | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1'
           ./tiledb/test/tiledb_unit -d yes | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1'
+          ./tiledb/test/regression/tiledb_regression -d yes | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1'
 
           # Kill the running GCS emulator server Linux only because OSX does not
           # run the emulator

.github/workflows/build-ubuntu20.04-S3.yml

@@ -110,6 +110,7 @@ jobs:
           docker ps -a
 
           ./tiledb/test/tiledb_unit -d yes | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1'
+          ./tiledb/test/regression/tiledb_regression -d yes | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1'
 
           pushd $GITHUB_WORKSPACE/examples/cmake_project
           mkdir build && cd build

.github/workflows/build-ubuntu20.04-SERIALIZATION.yml

@@ -101,6 +101,7 @@ jobs:
           # Bypass Catch2 Framework stdout interception with awk on test output
           # make check | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1'
           ./tiledb/test/tiledb_unit -d yes | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1'
+          ./tiledb/test/regression/tiledb_regression -d yes | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1'
 
           # - bash: |
           pushd $GITHUB_WORKSPACE/examples/cmake_project

.github/workflows/build-windows.yml

@@ -223,7 +223,8 @@ jobs:
           }
 
           # CMake exits with non-0 status if there are any warnings during the build, so
-          cmake --build $env:BUILD_BUILDDIRECTORY\tiledb  -j $env:NUMBER_OF_PROCESSORS --target tiledb_unit --config Release -- /verbosity:minimal
+          cmake --build $env:BUILD_BUILDDIRECTORY\tiledb -j $env:NUMBER_OF_PROCESSORS --target tiledb_unit --config Release -- /verbosity:minimal
+          cmake --build $env:BUILD_BUILDDIRECTORY\tiledb -j $env:NUMBER_OF_PROCESSORS --target tiledb_regression --config Release -- /verbosity:minimal
 
           if ($env:TILEDB_AZURE -eq "ON") {
             if($env.TILEDB_USE_CUSTOM_NODE_JS) {

CMakeLists.txt

@@ -293,6 +293,8 @@ if (TILEDB_TESTS)
   add_dependencies(tests unit_range_subset)
   add_dependencies(tests unit_range)
   add_dependencies(tests unit_mgc_dict)
+
+  add_subdirectory(test/regression)
 endif()
 
 # Build tools

scripts/azure-linux_mac.yml

@@ -178,6 +178,7 @@ steps:
     fi
 
     make -j4 -C tiledb tiledb_unit
+    make -j4 -C tiledb tiledb_regression
 
     if [[ "$TILEDB_CI_ASAN" == "ON" ]]; then
       export ASAN_OPTIONS=detect_leaks=0 LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libasan.so.5
@@ -191,6 +192,7 @@ steps:
       # run directly the executable, cmake catches the segfault and blocks
       # the core dump
       ./tiledb/test/tiledb_unit -d yes
+      ./tiledb/test/regression/tiledb_regression -d yes
     fi
 
     # Kill the running Minio server, OSX only because Linux runs it within

scripts/ci/build_libtiledb.sh

@@ -45,3 +45,4 @@ cd $GITHUB_WORKSPACE/build
 ls -la
 
 make -j4 -C tiledb tiledb_unit
+make -j4 -C tiledb tiledb_regression

test/regression/CMakeLists.txt

@@ -0,0 +1,59 @@
+#
+# test/regression/CMakeLists.txt
+#
+#
+# The MIT License
+#
+# Copyright (c) 2022 TileDB, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+find_package(Catch_EP REQUIRED)
+
+set(SOURCES
+  targets/sc-15387.cc
+)
+
+add_executable(tiledb_regression
+  EXCLUDE_FROM_ALL
+  regression.cc
+  ${SOURCES}
+)
+
+if (NOT MSVC)
+  target_compile_options(tiledb_regression PRIVATE -Wno-deprecated-declarations)
+endif()
+
+target_link_libraries(tiledb_regression
+  PUBLIC
+    Catch2::Catch2
+    tiledb_shared
+)
+
+target_include_directories(tiledb_regression
+  PRIVATE
+    ${CMAKE_INSTALL_PREFIX}/include
+)
+
+add_test(
+  NAME "tiledb_regression"
+  COMMAND $<TARGET_FILE:tiledb_regression> --durations=yes
+  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+)

test/regression/regression.cc

@@ -0,0 +1,12 @@
+#define CATCH_CONFIG_RUNNER
+#include <catch.hpp>
+
+int main(const int argc, char** const argv) {
+  Catch::Session session;
+
+  int rc = session.applyCommandLine(argc, argv);
+  if (rc != 0)
+    return rc;
+
+  session.run();
+}

test/regression/targets/sc-15387.cc

@@ -0,0 +1,132 @@
+#include <tiledb/tiledb>
+
+#include "catch.hpp"
+
+#include <chrono>
+#include <thread>
+
+using namespace tiledb;
+
+namespace {
+void create_array(Context& ctx, std::string uri, bool use_two_dims) {
+  auto dim0 =
+      Dimension::create(ctx, "__dim_0", TILEDB_STRING_ASCII, nullptr, nullptr);
+  tiledb::FilterList dim0_filters{ctx};
+  dim0_filters.add_filter({ctx, TILEDB_FILTER_RLE});
+  dim0.set_filter_list(dim0_filters);
+
+  auto dim1 =
+      Dimension::create(ctx, "__dim_1", TILEDB_STRING_ASCII, nullptr, nullptr);
+
+  tiledb::Filter dim1_filter(ctx, TILEDB_FILTER_ZSTD);
+  int level{22};
+  dim1_filter.set_option(TILEDB_COMPRESSION_LEVEL, &level);
+  tiledb::FilterList dim1_filter_list{ctx};
+  dim1_filter_list.add_filter(dim1_filter);
+  dim1.set_filter_list(dim1_filter_list);
+
+  tiledb::FilterList attr_filter_list{ctx};
+  attr_filter_list.add_filter({ctx, TILEDB_FILTER_ZSTD});
+
+  Domain domain{ctx};
+  domain.add_dimension(dim0);
+  if (use_two_dims) {
+    domain.add_dimension(dim1);
+  }
+
+  Attribute attr{ctx, "value", TILEDB_FLOAT32, attr_filter_list};
+
+  tiledb::FilterList offsets_filters{ctx};
+  offsets_filters.add_filter({ctx, TILEDB_FILTER_DOUBLE_DELTA})
+      .add_filter({ctx, TILEDB_FILTER_BIT_WIDTH_REDUCTION})
+      .add_filter({ctx, TILEDB_FILTER_ZSTD});
+
+  ArraySchema schema{ctx, TILEDB_SPARSE};
+  schema.set_allows_dups(true);
+  schema.set_capacity(100000);
+  schema.set_cell_order(TILEDB_ROW_MAJOR);
+  schema.set_tile_order(TILEDB_COL_MAJOR);
+  schema.set_domain(domain);
+  schema.add_attribute(attr);
+
+  Array::create(uri, schema);
+}
+
+void write_array(Context& ctx, std::string uri, bool use_two_dims) {
+  {
+    std::vector<char> d0_data{'a', 'b', 'c'};
+    std::vector<uint64_t> d0_offsets{0, 1, 2};
+    std::vector<char> d1_data{'s', 't', 'u'};
+    std::vector<uint64_t> d1_offsets{0, 1, 2};
+    std::vector<float> data{4, 5, 6};
+
+    Array array{ctx, uri, TILEDB_WRITE};
+    Query query{ctx, array, TILEDB_WRITE};
+
+    query.set_layout(TILEDB_UNORDERED)
+        .set_data_buffer("__dim_0", d0_data)
+        .set_offsets_buffer("__dim_0", d0_offsets)
+        .set_data_buffer("value", data);
+
+    if (use_two_dims) {
+      query.set_data_buffer("__dim_1", d1_data)
+          .set_offsets_buffer("__dim_1", d1_offsets);
+    }
+
+    query.submit();
+    query.finalize();
+
+    REQUIRE(query.query_status() == Query::Status::COMPLETE);
+    std::this_thread::sleep_for(std::chrono::milliseconds(1));
+  }
+  {
+    std::vector<char> d0_data{'d', 'e', 'f'};
+    std::vector<uint64_t> d0_offsets{0, 1, 2};
+    std::vector<char> d1_data{'v', 'w', 'x'};
+    std::vector<uint64_t> d1_offsets{0, 1, 2};
+    std::vector<float> data{4, 5, 6};
+
+    Array array{ctx, uri, TILEDB_WRITE};
+    Query query{ctx, array, TILEDB_WRITE};
+
+    query.set_layout(TILEDB_UNORDERED)
+        .set_data_buffer("__dim_0", d0_data)
+        .set_offsets_buffer("__dim_0", d0_offsets)
+        .set_data_buffer("value", data);
+
+    if (use_two_dims) {
+      query.set_data_buffer("__dim_1", d1_data)
+          .set_offsets_buffer("__dim_1", d1_offsets);
+    }
+
+    query.submit();
+    query.finalize();
+
+    REQUIRE(query.query_status() == Query::Status::COMPLETE);
+  }
+}
+
+};  // anonymous namespace
+
+TEST_CASE("SC-15387") {
+  Context ctx;
+
+  std::string uri{"foo1"};
+
+  auto object = tiledb::Object::object(ctx, uri);
+  if (object.type() == tiledb::Object::Type::Array)
+    tiledb::Object::remove(ctx, uri);
+
+  auto use_two_dims = GENERATE(false, true);
+
+  create_array(ctx, uri, use_two_dims);
+
+  write_array(ctx, uri, use_two_dims);
+
+  // use_two_dims
+  // - false: segfaults if pre-increment used in comparators.h
+  // - true: expected to segfault before fix
+  tiledb::Array::consolidate(ctx, uri);
+
+  // tiledb::Object::remove(ctx, uri);
+};

tiledb/sm/misc/comparators.h

@@ -300,7 +300,7 @@ class GlobalCmp : protected CellCmpBase {
       }
     } else {  // COL_MAJOR
       assert(tile_order_ == Layout::COL_MAJOR);
-      for (unsigned d = dim_num_ - 1;; --d) {
+      for (int32_t d = static_cast<int32_t>(dim_num_) - 1; d >= 0; d--) {
         // Not applicable to var-sized dimensions
         if (domain_.dimension_ptr(d)->var_size())
           continue;
@@ -312,9 +312,6 @@ class GlobalCmp : protected CellCmpBase {
         if (res == 1)
           return false;
         // else same tile on dimension d --> continue
-
-        if (d == 0)
-          break;
       }
     }
 
@@ -331,17 +328,14 @@ class GlobalCmp : protected CellCmpBase {
       }
     } else {  // COL_MAJOR
       assert(cell_order_ == Layout::COL_MAJOR);
-      for (unsigned d = dim_num_ - 1;; --d) {
+      for (int32_t d = static_cast<int32_t>(dim_num_) - 1; d >= 0; d--) {
         auto res = cell_order_cmp_RC(d, a, b);
 
         if (res == -1)
           return true;
         if (res == 1)
           return false;
         // else same tile on dimension d --> continue
-
-        if (d == 0)
-          break;
       }
     }