diff --git a/.github/workflows/build-macOS11-S3.yml b/.github/workflows/build-macOS11-S3.yml index 4ee7b99f5d4..737a8509605 100644 --- a/.github/workflows/build-macOS11-S3.yml +++ b/.github/workflows/build-macOS11-S3.yml @@ -100,6 +100,7 @@ jobs: source $GITHUB_WORKSPACE/scripts/ci/build_libtiledb.sh ./tiledb/test/tiledb_unit -d yes | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1' + ./tiledb/test/regression/tiledb_regression -d yes | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1' # Kill the running Minio server, OSX only because Linux runs it within # docker. diff --git a/.github/workflows/build-ubuntu16.04-HDFS.yml b/.github/workflows/build-ubuntu16.04-HDFS.yml index 0db41db2c40..a75c05f4372 100644 --- a/.github/workflows/build-ubuntu16.04-HDFS.yml +++ b/.github/workflows/build-ubuntu16.04-HDFS.yml @@ -114,6 +114,7 @@ jobs: # Bypass Catch2 Framework stdout interception with awk on test output # make check | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1' ./tiledb/test/tiledb_unit -d yes | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1' + ./tiledb/test/regression/tiledb_regression -d yes | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1' pushd $GITHUB_WORKSPACE/examples/cmake_project mkdir build && cd build diff --git a/.github/workflows/build-ubuntu20.04-AZURE.yml b/.github/workflows/build-ubuntu20.04-AZURE.yml index ded4feb5da3..8f6c65b9db5 100644 --- a/.github/workflows/build-ubuntu20.04-AZURE.yml +++ b/.github/workflows/build-ubuntu20.04-AZURE.yml @@ -121,6 +121,7 @@ jobs: # Bypass Catch2 Framework stdout interception with awk on test output # make check | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1' ./tiledb/test/tiledb_unit -d yes | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1' + ./tiledb/test/regression/tiledb_regression -d yes | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1' # Kill the running Azurite server kill -n 9 $AZURITE_PID diff --git a/.github/workflows/build-ubuntu20.04-GCS.yml b/.github/workflows/build-ubuntu20.04-GCS.yml index 54eca121e79..99d697f9ede 100644 --- a/.github/workflows/build-ubuntu20.04-GCS.yml +++ b/.github/workflows/build-ubuntu20.04-GCS.yml @@ -108,6 +108,7 @@ jobs: # Bypass Catch2 Framework stdout interception with awk on test output # make check | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1' ./tiledb/test/tiledb_unit -d yes | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1' + ./tiledb/test/regression/tiledb_regression -d yes | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1' # Kill the running GCS emulator server Linux only because OSX does not # run the emulator diff --git a/.github/workflows/build-ubuntu20.04-S3.yml b/.github/workflows/build-ubuntu20.04-S3.yml index 651a9b56d5d..88236eb3930 100644 --- a/.github/workflows/build-ubuntu20.04-S3.yml +++ b/.github/workflows/build-ubuntu20.04-S3.yml @@ -110,6 +110,7 @@ jobs: docker ps -a ./tiledb/test/tiledb_unit -d yes | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1' + ./tiledb/test/regression/tiledb_regression -d yes | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1' pushd $GITHUB_WORKSPACE/examples/cmake_project mkdir build && cd build diff --git a/.github/workflows/build-ubuntu20.04-SERIALIZATION.yml b/.github/workflows/build-ubuntu20.04-SERIALIZATION.yml index 9ce2751fc1b..261adafb128 100644 --- a/.github/workflows/build-ubuntu20.04-SERIALIZATION.yml +++ b/.github/workflows/build-ubuntu20.04-SERIALIZATION.yml @@ -101,6 +101,7 @@ jobs: # Bypass Catch2 Framework stdout interception with awk on test output # make check | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1' ./tiledb/test/tiledb_unit -d yes | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1' + ./tiledb/test/regression/tiledb_regression -d yes | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1' # - bash: | pushd $GITHUB_WORKSPACE/examples/cmake_project diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index 2c857831df6..5ce94aca629 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -223,7 +223,8 @@ jobs: } # CMake exits with non-0 status if there are any warnings during the build, so - cmake --build $env:BUILD_BUILDDIRECTORY\tiledb -j $env:NUMBER_OF_PROCESSORS --target tiledb_unit --config Release -- /verbosity:minimal + cmake --build $env:BUILD_BUILDDIRECTORY\tiledb -j $env:NUMBER_OF_PROCESSORS --target tiledb_unit --config Release -- /verbosity:minimal + cmake --build $env:BUILD_BUILDDIRECTORY\tiledb -j $env:NUMBER_OF_PROCESSORS --target tiledb_regression --config Release -- /verbosity:minimal if ($env:TILEDB_AZURE -eq "ON") { if($env.TILEDB_USE_CUSTOM_NODE_JS) { diff --git a/CMakeLists.txt b/CMakeLists.txt index 8e17be8c87c..e113e2fae20 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -293,6 +293,8 @@ if (TILEDB_TESTS) add_dependencies(tests unit_range_subset) add_dependencies(tests unit_range) add_dependencies(tests unit_mgc_dict) + + add_subdirectory(test/regression) endif() # Build tools diff --git a/scripts/azure-linux_mac.yml b/scripts/azure-linux_mac.yml index c0e4284a83b..506c1b36b39 100755 --- a/scripts/azure-linux_mac.yml +++ b/scripts/azure-linux_mac.yml @@ -178,6 +178,7 @@ steps: fi make -j4 -C tiledb tiledb_unit + make -j4 -C tiledb tiledb_regression if [[ "$TILEDB_CI_ASAN" == "ON" ]]; then export ASAN_OPTIONS=detect_leaks=0 LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libasan.so.5 @@ -191,6 +192,7 @@ steps: # run directly the executable, cmake catches the segfault and blocks # the core dump ./tiledb/test/tiledb_unit -d yes + ./tiledb/test/regression/tiledb_regression -d yes fi # Kill the running Minio server, OSX only because Linux runs it within diff --git a/scripts/ci/build_libtiledb.sh b/scripts/ci/build_libtiledb.sh index fb616aef10d..3844ac06204 100644 --- a/scripts/ci/build_libtiledb.sh +++ b/scripts/ci/build_libtiledb.sh @@ -45,3 +45,4 @@ cd $GITHUB_WORKSPACE/build ls -la make -j4 -C tiledb tiledb_unit +make -j4 -C tiledb tiledb_regression diff --git a/test/regression/CMakeLists.txt b/test/regression/CMakeLists.txt new file mode 100644 index 00000000000..7ec089c12a5 --- /dev/null +++ b/test/regression/CMakeLists.txt @@ -0,0 +1,59 @@ +# +# test/regression/CMakeLists.txt +# +# +# The MIT License +# +# Copyright (c) 2022 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +find_package(Catch_EP REQUIRED) + +set(SOURCES + targets/sc-15387.cc +) + +add_executable(tiledb_regression + EXCLUDE_FROM_ALL + regression.cc + ${SOURCES} +) + +if (NOT MSVC) + target_compile_options(tiledb_regression PRIVATE -Wno-deprecated-declarations) +endif() + +target_link_libraries(tiledb_regression + PUBLIC + Catch2::Catch2 + tiledb_shared +) + +target_include_directories(tiledb_regression + PRIVATE + ${CMAKE_INSTALL_PREFIX}/include +) + +add_test( + NAME "tiledb_regression" + COMMAND $ --durations=yes + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} +) diff --git a/test/regression/regression.cc b/test/regression/regression.cc new file mode 100644 index 00000000000..6ba668926f9 --- /dev/null +++ b/test/regression/regression.cc @@ -0,0 +1,12 @@ +#define CATCH_CONFIG_RUNNER +#include + +int main(const int argc, char** const argv) { + Catch::Session session; + + int rc = session.applyCommandLine(argc, argv); + if (rc != 0) + return rc; + + session.run(); +} \ No newline at end of file diff --git a/test/regression/targets/sc-15387.cc b/test/regression/targets/sc-15387.cc new file mode 100644 index 00000000000..4dafc9a5f39 --- /dev/null +++ b/test/regression/targets/sc-15387.cc @@ -0,0 +1,132 @@ +#include + +#include "catch.hpp" + +#include +#include + +using namespace tiledb; + +namespace { +void create_array(Context& ctx, std::string uri, bool use_two_dims) { + auto dim0 = + Dimension::create(ctx, "__dim_0", TILEDB_STRING_ASCII, nullptr, nullptr); + tiledb::FilterList dim0_filters{ctx}; + dim0_filters.add_filter({ctx, TILEDB_FILTER_RLE}); + dim0.set_filter_list(dim0_filters); + + auto dim1 = + Dimension::create(ctx, "__dim_1", TILEDB_STRING_ASCII, nullptr, nullptr); + + tiledb::Filter dim1_filter(ctx, TILEDB_FILTER_ZSTD); + int level{22}; + dim1_filter.set_option(TILEDB_COMPRESSION_LEVEL, &level); + tiledb::FilterList dim1_filter_list{ctx}; + dim1_filter_list.add_filter(dim1_filter); + dim1.set_filter_list(dim1_filter_list); + + tiledb::FilterList attr_filter_list{ctx}; + attr_filter_list.add_filter({ctx, TILEDB_FILTER_ZSTD}); + + Domain domain{ctx}; + domain.add_dimension(dim0); + if (use_two_dims) { + domain.add_dimension(dim1); + } + + Attribute attr{ctx, "value", TILEDB_FLOAT32, attr_filter_list}; + + tiledb::FilterList offsets_filters{ctx}; + offsets_filters.add_filter({ctx, TILEDB_FILTER_DOUBLE_DELTA}) + .add_filter({ctx, TILEDB_FILTER_BIT_WIDTH_REDUCTION}) + .add_filter({ctx, TILEDB_FILTER_ZSTD}); + + ArraySchema schema{ctx, TILEDB_SPARSE}; + schema.set_allows_dups(true); + schema.set_capacity(100000); + schema.set_cell_order(TILEDB_ROW_MAJOR); + schema.set_tile_order(TILEDB_COL_MAJOR); + schema.set_domain(domain); + schema.add_attribute(attr); + + Array::create(uri, schema); +} + +void write_array(Context& ctx, std::string uri, bool use_two_dims) { + { + std::vector d0_data{'a', 'b', 'c'}; + std::vector d0_offsets{0, 1, 2}; + std::vector d1_data{'s', 't', 'u'}; + std::vector d1_offsets{0, 1, 2}; + std::vector data{4, 5, 6}; + + Array array{ctx, uri, TILEDB_WRITE}; + Query query{ctx, array, TILEDB_WRITE}; + + query.set_layout(TILEDB_UNORDERED) + .set_data_buffer("__dim_0", d0_data) + .set_offsets_buffer("__dim_0", d0_offsets) + .set_data_buffer("value", data); + + if (use_two_dims) { + query.set_data_buffer("__dim_1", d1_data) + .set_offsets_buffer("__dim_1", d1_offsets); + } + + query.submit(); + query.finalize(); + + REQUIRE(query.query_status() == Query::Status::COMPLETE); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + { + std::vector d0_data{'d', 'e', 'f'}; + std::vector d0_offsets{0, 1, 2}; + std::vector d1_data{'v', 'w', 'x'}; + std::vector d1_offsets{0, 1, 2}; + std::vector data{4, 5, 6}; + + Array array{ctx, uri, TILEDB_WRITE}; + Query query{ctx, array, TILEDB_WRITE}; + + query.set_layout(TILEDB_UNORDERED) + .set_data_buffer("__dim_0", d0_data) + .set_offsets_buffer("__dim_0", d0_offsets) + .set_data_buffer("value", data); + + if (use_two_dims) { + query.set_data_buffer("__dim_1", d1_data) + .set_offsets_buffer("__dim_1", d1_offsets); + } + + query.submit(); + query.finalize(); + + REQUIRE(query.query_status() == Query::Status::COMPLETE); + } +} + +}; // anonymous namespace + +TEST_CASE("SC-15387") { + Context ctx; + + std::string uri{"foo1"}; + + auto object = tiledb::Object::object(ctx, uri); + if (object.type() == tiledb::Object::Type::Array) + tiledb::Object::remove(ctx, uri); + + auto use_two_dims = GENERATE(false, true); + + create_array(ctx, uri, use_two_dims); + + write_array(ctx, uri, use_two_dims); + + // use_two_dims + // - false: segfaults if pre-increment used in comparators.h + // - true: expected to segfault before fix + tiledb::Array::consolidate(ctx, uri); + + // tiledb::Object::remove(ctx, uri); +}; diff --git a/tiledb/sm/misc/comparators.h b/tiledb/sm/misc/comparators.h index a12946afd35..1163aab6205 100644 --- a/tiledb/sm/misc/comparators.h +++ b/tiledb/sm/misc/comparators.h @@ -300,7 +300,7 @@ class GlobalCmp : protected CellCmpBase { } } else { // COL_MAJOR assert(tile_order_ == Layout::COL_MAJOR); - for (unsigned d = dim_num_ - 1;; --d) { + for (int32_t d = static_cast(dim_num_) - 1; d >= 0; d--) { // Not applicable to var-sized dimensions if (domain_.dimension_ptr(d)->var_size()) continue; @@ -312,9 +312,6 @@ class GlobalCmp : protected CellCmpBase { if (res == 1) return false; // else same tile on dimension d --> continue - - if (d == 0) - break; } } @@ -331,7 +328,7 @@ class GlobalCmp : protected CellCmpBase { } } else { // COL_MAJOR assert(cell_order_ == Layout::COL_MAJOR); - for (unsigned d = dim_num_ - 1;; --d) { + for (int32_t d = static_cast(dim_num_) - 1; d >= 0; d--) { auto res = cell_order_cmp_RC(d, a, b); if (res == -1) @@ -339,9 +336,6 @@ class GlobalCmp : protected CellCmpBase { if (res == 1) return false; // else same tile on dimension d --> continue - - if (d == 0) - break; } }