From 6d6dce5e6f762b2f3ba1e4fdf3e3978499abc6b6 Mon Sep 17 00:00:00 2001 From: Stavros Papadopoulos Date: Wed, 19 Feb 2020 18:29:02 -0500 Subject: [PATCH] Refactoring towards addressing #93. (#1509) Mainly refactored fragment non-empty domain in FragmentMetadata and the way the tile MBRs are computed in Worker. --- test/src/unit-ReadCellSlabIter.cc | 99 +++- test/src/unit-Reader.cc | 80 ++- test/src/unit-TileDomain.cc | 49 +- tiledb/sm/array_schema/dimension.cc | 543 ++++++++++++++++++- tiledb/sm/array_schema/dimension.h | 146 ++++- tiledb/sm/array_schema/domain.cc | 128 ++--- tiledb/sm/array_schema/domain.h | 80 +-- tiledb/sm/array_schema/tile_domain.h | 35 +- tiledb/sm/fragment/fragment_metadata.cc | 352 +++++------- tiledb/sm/fragment/fragment_metadata.h | 48 +- tiledb/sm/{query => misc}/types.h | 70 +++ tiledb/sm/misc/utils.cc | 101 ++-- tiledb/sm/misc/utils.h | 41 +- tiledb/sm/query/read_cell_slab_iter.cc | 33 +- tiledb/sm/query/read_cell_slab_iter.h | 3 +- tiledb/sm/query/reader.cc | 32 +- tiledb/sm/query/reader.h | 8 +- tiledb/sm/query/result_space_tile.h | 84 ++- tiledb/sm/query/result_tile.cc | 3 +- tiledb/sm/query/result_tile.h | 3 +- tiledb/sm/query/writer.cc | 87 +-- tiledb/sm/query/writer.h | 16 +- tiledb/sm/storage_manager/storage_manager.cc | 137 ++--- tiledb/sm/storage_manager/storage_manager.h | 15 - tiledb/sm/subarray/subarray.cc | 37 ++ tiledb/sm/subarray/subarray.h | 10 + 26 files changed, 1370 insertions(+), 870 deletions(-) rename tiledb/sm/{query => misc}/types.h (67%) diff --git a/test/src/unit-ReadCellSlabIter.cc b/test/src/unit-ReadCellSlabIter.cc index 0cd2d9073b6..9143e8181dd 100644 --- a/test/src/unit-ReadCellSlabIter.cc +++ b/test/src/unit-ReadCellSlabIter.cc @@ -71,8 +71,9 @@ struct ReadCellSlabIterFx { template void create_result_space_tiles( const Domain* dom, + const NDRange& dom_ndrange, Layout layout, - const std::vector>& domain_slices, + const std::vector& domain_slices, const std::vector>& tile_coords, std::map>* result_space_tiles); }; @@ -141,8 +142,9 @@ void ReadCellSlabIterFx::check_iter( template void ReadCellSlabIterFx::create_result_space_tiles( const Domain* dom, + const NDRange& dom_ndrange, Layout layout, - const std::vector>& domain_slices, + const std::vector& domain_slices, const std::vector>& tile_coords, std::map>* result_space_tiles) { auto dim_num = dom->dim_num(); @@ -154,12 +156,12 @@ void ReadCellSlabIterFx::create_result_space_tiles( (unsigned)(domain_slices.size() - i), dim_num, domain, - &(domain_slices[i][0]), + domain_slices[i], tile_extents, layout); } TileDomain array_tile_domain( - UINT32_MAX, dim_num, domain, domain, tile_extents, layout); + UINT32_MAX, dim_num, domain, dom_ndrange, tile_extents, layout); Reader::compute_result_space_tiles( dom, tile_coords, @@ -218,11 +220,16 @@ TEST_CASE_METHOD( subarray.compute_tile_coords(); // Create result space tiles - std::vector> domain_slices = {{1, 100}}; + std::vector slice = {1, 100}; + NDRange ds = {Range(&slice[0], 2 * sizeof(uint64_t))}; + std::vector domain_slices = {ds}; const auto& tile_coords = subarray.tile_coords(); std::map> result_space_tiles; + auto dom = array_->array_->array_schema()->domain(); + auto dom_ndrange = dom->domain_ndrange(); create_result_space_tiles( - array_->array_->array_schema()->domain(), + dom, + dom_ndrange, subarray_layout, domain_slices, tile_coords, @@ -274,11 +281,16 @@ TEST_CASE_METHOD( subarray.compute_tile_coords(); // Create result space tiles - std::vector> domain_slices = {{20, 30}}; + std::vector slice = {20, 30}; + NDRange ds = {Range(&slice[0], 2 * sizeof(uint64_t))}; + std::vector domain_slices = {ds}; const auto& tile_coords = subarray.tile_coords(); std::map> result_space_tiles; + auto dom = array_->array_->array_schema()->domain(); + auto dom_ndrange = dom->domain_ndrange(); create_result_space_tiles( - array_->array_->array_schema()->domain(), + dom, + dom_ndrange, subarray_layout, domain_slices, tile_coords, @@ -330,11 +342,19 @@ TEST_CASE_METHOD( subarray.compute_tile_coords(); // Create result space tiles - std::vector> domain_slices = {{5, 12}, {4, 15}}; + std::vector slice_1 = {5, 12}; + std::vector slice_2 = {4, 15}; + auto size = 2 * sizeof(uint64_t); + NDRange ds1 = {Range(&slice_1[0], size)}; + NDRange ds2 = {Range(&slice_2[0], size)}; + std::vector domain_slices = {ds1, ds2}; const auto& tile_coords = subarray.tile_coords(); std::map> result_space_tiles; + auto dom = array_->array_->array_schema()->domain(); + auto dom_ndrange = dom->domain_ndrange(); create_result_space_tiles( - array_->array_->array_schema()->domain(), + dom, + dom_ndrange, subarray_layout, domain_slices, tile_coords, @@ -392,12 +412,21 @@ TEST_CASE_METHOD( subarray.compute_tile_coords(); // Create result space tiles - std::vector> domain_slices = {{3, 12}}; + std::vector slice = {3, 12}; + auto size = 2 * sizeof(uint64_t); + NDRange ds = {Range(&slice[0], size)}; + std::vector domain_slices = {ds}; const auto& tile_coords = subarray.tile_coords(); std::map> result_space_tiles; auto dom = array_->array_->array_schema()->domain(); + auto dom_ndrange = dom->domain_ndrange(); create_result_space_tiles( - dom, subarray_layout, domain_slices, tile_coords, &result_space_tiles); + dom, + dom_ndrange, + subarray_layout, + domain_slices, + tile_coords, + &result_space_tiles); // Create result coordinates std::vector result_coords; @@ -607,11 +636,17 @@ TEST_CASE_METHOD( subarray.compute_tile_coords(); // Create result space tiles - std::vector> domain_slices = {{1, 6, 1, 6}}; + std::vector slice = {1, 6, 1, 6}; + auto size = 2 * sizeof(uint64_t); + NDRange ds = {Range(&slice[0], size), Range(&slice[2], size)}; + std::vector domain_slices = {ds}; const auto& tile_coords = subarray.tile_coords(); std::map> result_space_tiles; + auto dom = array_->array_->array_schema()->domain(); + auto dom_ndrange = dom->domain_ndrange(); create_result_space_tiles( - array_->array_->array_schema()->domain(), + dom, + dom_ndrange, tile_domain_layout, domain_slices, tile_coords, @@ -774,11 +809,17 @@ TEST_CASE_METHOD( subarray.compute_tile_coords(); // Create result space tiles - std::vector> domain_slices = {{6, 6, 6, 6}}; + std::vector slice = {6, 6, 6, 6}; + auto size = 2 * sizeof(uint64_t); + NDRange ds = {Range(&slice[0], size), Range(&slice[2], size)}; + std::vector domain_slices = {ds}; const auto& tile_coords = subarray.tile_coords(); std::map> result_space_tiles; + auto dom = array_->array_->array_schema()->domain(); + auto dom_ndrange = dom->domain_ndrange(); create_result_space_tiles( - array_->array_->array_schema()->domain(), + dom, + dom_ndrange, tile_domain_layout, domain_slices, tile_coords, @@ -954,11 +995,17 @@ TEST_CASE_METHOD( subarray.compute_tile_coords(); // Create result space tiles - std::vector> domain_slices = {{3, 6, 5, 6}}; + std::vector slice = {3, 6, 5, 6}; + auto size = 2 * sizeof(uint64_t); + NDRange ds = {Range(&slice[0], size), Range(&slice[2], size)}; + std::vector domain_slices = {ds}; const auto& tile_coords = subarray.tile_coords(); std::map> result_space_tiles; + auto dom = array_->array_->array_schema()->domain(); + auto dom_ndrange = dom->domain_ndrange(); create_result_space_tiles( - array_->array_->array_schema()->domain(), + dom, + dom_ndrange, tile_domain_layout, domain_slices, tile_coords, @@ -1178,13 +1225,23 @@ TEST_CASE_METHOD( subarray.compute_tile_coords(); // Create result space tiles - std::vector> domain_slices = {{3, 5, 2, 4}, - {2, 3, 1, 6}}; + std::vector slice_1 = {3, 5, 2, 4}; + std::vector slice_2 = {2, 3, 1, 6}; + auto size = 2 * sizeof(uint64_t); + NDRange ds1 = {Range(&slice_1[0], size), Range(&slice_1[2], size)}; + NDRange ds2 = {Range(&slice_2[0], size), Range(&slice_2[2], size)}; + std::vector domain_slices = {ds1, ds2}; const auto& tile_coords = subarray.tile_coords(); std::map> result_space_tiles; auto dom = array_->array_->array_schema()->domain(); + auto dom_ndrange = dom->domain_ndrange(); create_result_space_tiles( - dom, tile_domain_layout, domain_slices, tile_coords, &result_space_tiles); + dom, + dom_ndrange, + tile_domain_layout, + domain_slices, + tile_coords, + &result_space_tiles); // Create result coordinates std::vector result_coords; diff --git a/test/src/unit-Reader.cc b/test/src/unit-Reader.cc index f8477b3a3f7..184a896963c 100644 --- a/test/src/unit-Reader.cc +++ b/test/src/unit-Reader.cc @@ -32,6 +32,7 @@ #include "test/src/helpers.h" #include "tiledb/sm/c_api/tiledb_struct_def.h" +#include "tiledb/sm/misc/types.h" #include "tiledb/sm/query/reader.h" #ifdef _WIN32 @@ -144,15 +145,25 @@ TEST_CASE_METHOD( std::vector domain_slice_1 = {3, 4, 1, 12}; std::vector domain_slice_2 = {4, 5, 2, 4}; std::vector domain_slice_3 = {5, 7, 1, 9}; + + auto size = 2 * sizeof(int32_t); + NDRange ds1 = {Range(&domain_slice_1[0], size), + Range(&domain_slice_1[2], size)}; + NDRange ds2 = {Range(&domain_slice_2[0], size), + Range(&domain_slice_2[2], size)}; + NDRange ds3 = {Range(&domain_slice_3[0], size), + Range(&domain_slice_3[2], size)}; + NDRange dsd = {Range(&domain[0], size), Range(&domain[2], size)}; + std::vector> frag_tile_domains; frag_tile_domains.emplace_back(TileDomain( - 3, dim_num, &domain[0], &domain_slice_3[0], &tile_extents[0], layout)); + 3, dim_num, &domain[0], ds3, &tile_extents[0], layout)); frag_tile_domains.emplace_back(TileDomain( - 2, dim_num, &domain[0], &domain_slice_2[0], &tile_extents[0], layout)); + 2, dim_num, &domain[0], ds2, &tile_extents[0], layout)); frag_tile_domains.emplace_back(TileDomain( - 1, dim_num, &domain[0], &domain_slice_1[0], &tile_extents[0], layout)); + 1, dim_num, &domain[0], ds1, &tile_extents[0], layout)); TileDomain array_tile_domain( - UINT32_MAX, dim_num, &domain[0], &domain[0], &tile_extents[0], layout); + UINT32_MAX, dim_num, &domain[0], dsd, &tile_extents[0], layout); Dimension d1("d1", Datatype::INT32); Dimension d2("d2", Datatype::INT32); @@ -181,46 +192,29 @@ TEST_CASE_METHOD( ResultTile result_tile_2_0_3(3, 0, &dom); ResultTile result_tile_3_0_3(3, 2, &dom); - // Initialize frag domains - typedef std::pair FragDomain; - std::vector frag_domain_1_0 = {FragDomain(2, &domain_slice_2[0]), - FragDomain(1, &domain_slice_1[0])}; - std::vector frag_domain_1_2 = {FragDomain(1, &domain_slice_1[0])}; - std::vector frag_domain_2_0 = {FragDomain(3, &domain_slice_3[0])}; - std::vector frag_domain_3_0 = {FragDomain(3, &domain_slice_3[0])}; - std::vector frag_domain_2_2 = {}; - std::vector frag_domain_3_2 = {}; - - // Initialize result tiles - std::map result_tiles_1_0 = { - std::pair(1, result_tile_1_0_1), - std::pair(2, result_tile_1_0_2), - }; - std::map result_tiles_1_2 = { - std::pair(1, result_tile_1_2_1), - }; - std::map result_tiles_2_0 = { - std::pair(3, result_tile_2_0_3), - }; - std::map result_tiles_3_0 = { - std::pair(3, result_tile_3_0_3), - }; - std::map result_tiles_2_2 = {}; - std::map result_tiles_3_2 = {}; - // Initialize result_space_tiles - ResultSpaceTile rst_1_0 = { - {3, 1}, frag_domain_1_0, result_tiles_1_0}; - ResultSpaceTile rst_1_2 = { - {3, 11}, frag_domain_1_2, result_tiles_1_2}; - ResultSpaceTile rst_2_0 = { - {5, 1}, frag_domain_2_0, result_tiles_2_0}; - ResultSpaceTile rst_2_2 = { - {5, 11}, frag_domain_2_2, result_tiles_2_2}; - ResultSpaceTile rst_3_0 = { - {7, 1}, frag_domain_3_0, result_tiles_3_0}; - ResultSpaceTile rst_3_2 = { - {7, 11}, frag_domain_3_2, result_tiles_3_2}; + ResultSpaceTile rst_1_0; + rst_1_0.set_start_coords({3, 1}); + rst_1_0.append_frag_domain(2, ds2); + rst_1_0.append_frag_domain(1, ds1); + rst_1_0.set_result_tile(1, result_tile_1_0_1); + rst_1_0.set_result_tile(2, result_tile_1_0_2); + ResultSpaceTile rst_1_2; + rst_1_2.set_start_coords({3, 11}); + rst_1_2.append_frag_domain(1, ds1); + rst_1_2.set_result_tile(1, result_tile_1_2_1); + ResultSpaceTile rst_2_0; + rst_2_0.set_start_coords({5, 1}); + rst_2_0.append_frag_domain(3, ds3); + rst_2_0.set_result_tile(3, result_tile_2_0_3); + ResultSpaceTile rst_2_2; + rst_2_2.set_start_coords({5, 11}); + ResultSpaceTile rst_3_0; + rst_3_0.set_start_coords({7, 1}); + rst_3_0.append_frag_domain(3, ds3); + rst_3_0.set_result_tile(3, result_tile_3_0_3); + ResultSpaceTile rst_3_2; + rst_3_2.set_start_coords({7, 11}); // Prepare correct space tiles map std::map> c_result_space_tiles; diff --git a/test/src/unit-TileDomain.cc b/test/src/unit-TileDomain.cc index 6c40975ec53..5cc9111021c 100644 --- a/test/src/unit-TileDomain.cc +++ b/test/src/unit-TileDomain.cc @@ -43,8 +43,10 @@ TEST_CASE("TileDomain: Test 1D", "[TileDomain][1d]") { int32_t tile_extent = 10; Layout layout = Layout::ROW_MAJOR; + NDRange ds = {Range(&domain_slice[0], 2 * sizeof(int32_t))}; + TileDomain tile_domain( - 0, dim_num, &domain[0], &domain_slice[0], &tile_extent, layout); + 0, dim_num, &domain[0], ds, &tile_extent, layout); const auto& td = tile_domain.tile_domain(); CHECK(td.size() == 2); CHECK(td[0] == 1); @@ -72,8 +74,11 @@ TEST_CASE( std::vector tile_extents = {2, 5}; Layout layout = Layout::ROW_MAJOR; + auto size = 2 * (sizeof(int32_t)); + NDRange ds = {Range(&domain_slice[0], size), Range(&domain_slice[2], size)}; + TileDomain tile_domain( - 0, dim_num, &domain[0], &domain_slice[0], &tile_extents[0], layout); + 0, dim_num, &domain[0], ds, &tile_extents[0], layout); const auto& td = tile_domain.tile_domain(); CHECK(td.size() == 4); CHECK(td[0] == 0); @@ -110,8 +115,11 @@ TEST_CASE( std::vector tile_extents = {2, 5}; Layout layout = Layout::ROW_MAJOR; + auto size = 2 * (sizeof(int32_t)); + NDRange ds = {Range(&domain_slice[0], size), Range(&domain_slice[2], size)}; + TileDomain tile_domain( - 0, dim_num, &domain[0], &domain_slice[0], &tile_extents[0], layout); + 0, dim_num, &domain[0], ds, &tile_extents[0], layout); const auto& td = tile_domain.tile_domain(); CHECK(td.size() == 4); CHECK(td[0] == 1); @@ -143,8 +151,11 @@ TEST_CASE( std::vector tile_extents = {2, 5}; Layout layout = Layout::COL_MAJOR; + auto size = 2 * (sizeof(int32_t)); + NDRange ds = {Range(&domain_slice[0], size), Range(&domain_slice[2], size)}; + TileDomain tile_domain( - 0, dim_num, &domain[0], &domain_slice[0], &tile_extents[0], layout); + 0, dim_num, &domain[0], ds, &tile_extents[0], layout); const auto& td = tile_domain.tile_domain(); CHECK(td.size() == 4); CHECK(td[0] == 0); @@ -176,8 +187,11 @@ TEST_CASE( std::vector tile_extents = {2, 5}; Layout layout = Layout::COL_MAJOR; + auto size = 2 * (sizeof(int32_t)); + NDRange ds = {Range(&domain_slice[0], size), Range(&domain_slice[2], size)}; + TileDomain tile_domain( - 0, dim_num, &domain[0], &domain_slice[0], &tile_extents[0], layout); + 0, dim_num, &domain[0], ds, &tile_extents[0], layout); const auto& td = tile_domain.tile_domain(); CHECK(td.size() == 4); CHECK(td[0] == 1); @@ -208,8 +222,11 @@ TEST_CASE( std::vector tile_extents = {2, 5}; Layout layout = Layout::COL_MAJOR; + auto size = 2 * (sizeof(int32_t)); + NDRange ds = {Range(&domain_slice[0], size), Range(&domain_slice[2], size)}; + TileDomain tile_domain( - 0, dim_num, &domain[0], &domain_slice[0], &tile_extents[0], layout); + 0, dim_num, &domain[0], ds, &tile_extents[0], layout); int32_t tile_coords[] = {0, 0}; auto tile_subarray = tile_domain.tile_subarray(tile_coords); @@ -235,8 +252,11 @@ TEST_CASE( std::vector tile_extents = {2, 5}; Layout layout = Layout::COL_MAJOR; + auto size = 2 * (sizeof(int32_t)); + NDRange ds = {Range(&domain_slice[0], size), Range(&domain_slice[2], size)}; + TileDomain tile_domain( - 0, dim_num, &domain[0], &domain_slice[0], &tile_extents[0], layout); + 0, dim_num, &domain[0], ds, &tile_extents[0], layout); int32_t tile_coords[] = {0, 0}; auto tile_overlap = tile_domain.tile_overlap(tile_coords); @@ -267,8 +287,11 @@ TEST_CASE( std::vector tile_extents = {2, 5}; Layout layout = Layout::COL_MAJOR; + auto size = 2 * (sizeof(int32_t)); + NDRange ds = {Range(&domain_slice[0], size), Range(&domain_slice[2], size)}; + TileDomain tile_domain( - 0, dim_num, &domain[0], &domain_slice[0], &tile_extents[0], layout); + 0, dim_num, &domain[0], ds, &tile_extents[0], layout); int32_t tile_coords[] = {0, 0}; CHECK(tile_domain.in_tile_domain(tile_coords)); @@ -288,11 +311,17 @@ TEST_CASE("TileDomain: Test 2D, covers", "[TileDomain][2d][covers]") { std::vector tile_extents = {2, 5}; Layout layout = Layout::COL_MAJOR; + auto size = 2 * (sizeof(int32_t)); + NDRange ds1 = {Range(&domain_slice_1[0], size), + Range(&domain_slice_1[2], size)}; + NDRange ds2 = {Range(&domain_slice_2[0], size), + Range(&domain_slice_2[2], size)}; + TileDomain tile_domain_1( - 1, dim_num, &domain[0], &domain_slice_1[0], &tile_extents[0], layout); + 1, dim_num, &domain[0], ds1, &tile_extents[0], layout); TileDomain tile_domain_2( - 2, dim_num, &domain[0], &domain_slice_2[0], &tile_extents[0], layout); + 2, dim_num, &domain[0], ds2, &tile_extents[0], layout); int32_t tile_coords[] = {0, 0}; CHECK(!tile_domain_1.covers(tile_coords, tile_domain_2)); diff --git a/tiledb/sm/array_schema/dimension.cc b/tiledb/sm/array_schema/dimension.cc index 29a2670a454..faf719df515 100644 --- a/tiledb/sm/array_schema/dimension.cc +++ b/tiledb/sm/array_schema/dimension.cc @@ -51,7 +51,14 @@ Dimension::Dimension() { domain_ = nullptr; tile_extent_ = nullptr; type_ = Datatype::INT32; + set_compute_mbr_func(); + set_crop_range_func(); + set_expand_range_func(); + set_expand_range_v_func(); + set_expand_to_tile_func(); set_oob_func(); + set_overlap_func(); + set_tile_num_func(); set_value_in_range_func(); } @@ -60,7 +67,14 @@ Dimension::Dimension(const std::string& name, Datatype type) , type_(type) { domain_ = nullptr; tile_extent_ = nullptr; + set_compute_mbr_func(); + set_crop_range_func(); + set_expand_range_func(); + set_expand_range_v_func(); + set_expand_to_tile_func(); set_oob_func(); + set_overlap_func(); + set_tile_num_func(); set_value_in_range_func(); } @@ -198,7 +212,14 @@ Status Dimension::deserialize(ConstBuffer* buff, Datatype type) { RETURN_NOT_OK(buff->read(tile_extent_, datatype_size(type_))); } + set_compute_mbr_func(); + set_crop_range_func(); + set_expand_range_func(); + set_expand_range_v_func(); + set_expand_to_tile_func(); set_oob_func(); + set_overlap_func(); + set_tile_num_func(); set_value_in_range_func(); return Status::Ok(); @@ -238,6 +259,105 @@ bool Dimension::is_anonymous() const { utils::parse::starts_with(name_, constants::default_dim_name); } +template +void Dimension::compute_mbr( + const Dimension* dim, const Tile& tile, Range* mbr) { + assert(dim != nullptr); + assert(mbr != nullptr); + auto data = (const T*)(tile.internal_data()); + assert(data != nullptr); + auto cell_num = tile.cell_num(); + assert(cell_num > 0); + + // Initialize MBR with the first tile values + T res[] = {data[0], data[0]}; + mbr->set_range(res, sizeof(res)); + + // Expand the MBR with the rest tile values + for (uint64_t c = 1; c < cell_num; ++c) + dim->expand_range_v(&data[c], mbr); +} + +void Dimension::compute_mbr(const Tile& tile, Range* mbr) const { + assert(compute_mbr_func_ != nullptr); + compute_mbr_func_(this, tile, mbr); +} + +template +void Dimension::crop_range(const Dimension* dim, Range* range) { + assert(dim != nullptr); + assert(!range->empty()); + auto dim_dom = (const T*)dim->domain(); + auto r = (const T*)range->data(); + T res[2] = {std::max(r[0], dim_dom[0]), std::min(r[1], dim_dom[1])}; + range->set_range(res, sizeof(res)); +} + +void Dimension::crop_range(Range* range) const { + assert(crop_range_func_ != nullptr); + crop_range_func_(this, range); +} + +template +void Dimension::expand_range_v(const Dimension* dim, const void* v, Range* r) { + assert(dim != nullptr); + assert(v != nullptr); + assert(!r->empty()); + (void)dim; // Not used here + auto rt = (const T*)r->data(); + auto vt = (const T*)v; + T res[2] = {std::min(rt[0], *vt), std::max(rt[1], *vt)}; + r->set_range(res, sizeof(res)); +} + +void Dimension::expand_range_v(const void* v, Range* r) const { + assert(expand_range_v_func_ != nullptr); + expand_range_v_func_(this, v, r); +} + +template +void Dimension::expand_range(const Dimension* dim, const Range& r1, Range* r2) { + assert(dim != nullptr); + assert(!r1.empty()); + assert(!r2->empty()); + (void)dim; // Not used here + auto d1 = (const T*)r1.data(); + auto d2 = (const T*)r2->data(); + T res[2] = {std::min(d1[0], d2[0]), std::max(d1[1], d2[1])}; + r2->set_range(res, sizeof(res)); +} + +void Dimension::expand_range(const Range& r1, Range* r2) const { + assert(expand_range_func_ != nullptr); + expand_range_func_(this, r1, r2); +} + +template +void Dimension::expand_to_tile(const Dimension* dim, Range* range) { + assert(dim != nullptr); + assert(!range->empty()); + + // Applicable only to regular tiles and integral domains + if (dim->tile_extent() == nullptr || !std::is_integral::value) + return; + + auto tile_extent = *(const T*)dim->tile_extent(); + auto dim_dom = (const T*)dim->domain(); + auto r = (const T*)range->data(); + T res[2]; + + res[0] = ((r[0] - dim_dom[0]) / tile_extent * tile_extent) + dim_dom[0]; + res[1] = + ((r[1] - dim_dom[0]) / tile_extent + 1) * tile_extent - 1 + dim_dom[0]; + + range->set_range(res, sizeof(res)); +} + +void Dimension::expand_to_tile(Range* range) const { + assert(expand_to_tile_func_ != nullptr); + expand_to_tile_func_(this, range); +} + template bool Dimension::oob( const Dimension* dim, const void* coord, std::string* err_msg) { @@ -259,18 +379,60 @@ bool Dimension::oob(const void* coord, std::string* err_msg) const { return oob_func_(this, coord, err_msg); } +template +bool Dimension::overlap( + const Dimension* dim, const Range& r1, const Range& r2) { + assert(dim != nullptr); + assert(!r1.empty()); + assert(!r2.empty()); + (void)dim; // Not used here + + auto d1 = (const T*)r1.data(); + auto d2 = (const T*)r2.data(); + return !(d1[0] > d2[1] || d1[1] < d2[0]); +} + +bool Dimension::overlap(const Range& r1, const Range& r2) const { + assert(overlap_func_ != nullptr); + return overlap_func_(this, r1, r2); +} + +template +uint64_t Dimension::tile_num(const Dimension* dim, const Range& range) { + assert(dim != nullptr); + assert(!range.empty()); + + // Trivial cases + if (dim->tile_extent() == nullptr) + return 1; + if (!std::is_integral::value) + return 0; + + auto tile_extent = *(const T*)dim->tile_extent(); + auto dim_dom = (const T*)dim->domain(); + auto r = (const T*)range.data(); + uint64_t start = (r[0] - dim_dom[0]) / tile_extent; + uint64_t end = (r[1] - dim_dom[0]) / tile_extent; + return end - start + 1; +} + +uint64_t Dimension::tile_num(const Range& range) const { + assert(tile_num_func_ != nullptr); + return tile_num_func_(this, range); +} + template bool Dimension::value_in_range( - const Dimension* dim, const void* value, const void* range) { - (void)*dim; // Will be used in the future + const Dimension* dim, const void* value, const Range& range) { + (void)*dim; // Not used here assert(value != nullptr); - assert(range != nullptr); + assert(!range.empty()); auto v = (const T*)value; - auto r = (const T*)range; + auto r = (const T*)(range.data()); return *v >= r[0] && *v <= r[1]; } -bool Dimension::value_in_range(const void* value, const void* range) const { +bool Dimension::value_in_range(const void* value, const Range& range) const { assert(value_in_range_func_ != nullptr); return value_in_range_func_(this, value, range); } @@ -605,6 +767,271 @@ Status Dimension::check_tile_extent() const { return Status::Ok(); } +void Dimension::set_crop_range_func() { + switch (type_) { + case Datatype::INT32: + crop_range_func_ = crop_range; + break; + case Datatype::INT64: + crop_range_func_ = crop_range; + break; + case Datatype::INT8: + crop_range_func_ = crop_range; + break; + case Datatype::UINT8: + crop_range_func_ = crop_range; + break; + case Datatype::INT16: + crop_range_func_ = crop_range; + break; + case Datatype::UINT16: + crop_range_func_ = crop_range; + break; + case Datatype::UINT32: + crop_range_func_ = crop_range; + break; + case Datatype::UINT64: + crop_range_func_ = crop_range; + break; + case Datatype::FLOAT32: + crop_range_func_ = crop_range; + break; + case Datatype::FLOAT64: + crop_range_func_ = crop_range; + break; + case Datatype::DATETIME_YEAR: + case Datatype::DATETIME_MONTH: + case Datatype::DATETIME_WEEK: + case Datatype::DATETIME_DAY: + case Datatype::DATETIME_HR: + case Datatype::DATETIME_MIN: + case Datatype::DATETIME_SEC: + case Datatype::DATETIME_MS: + case Datatype::DATETIME_US: + case Datatype::DATETIME_NS: + case Datatype::DATETIME_PS: + case Datatype::DATETIME_FS: + case Datatype::DATETIME_AS: + crop_range_func_ = crop_range; + break; + default: + crop_range_func_ = nullptr; + break; + } +} + +void Dimension::set_compute_mbr_func() { + switch (type_) { + case Datatype::INT32: + compute_mbr_func_ = compute_mbr; + break; + case Datatype::INT64: + compute_mbr_func_ = compute_mbr; + break; + case Datatype::INT8: + compute_mbr_func_ = compute_mbr; + break; + case Datatype::UINT8: + compute_mbr_func_ = compute_mbr; + break; + case Datatype::INT16: + compute_mbr_func_ = compute_mbr; + break; + case Datatype::UINT16: + compute_mbr_func_ = compute_mbr; + break; + case Datatype::UINT32: + compute_mbr_func_ = compute_mbr; + break; + case Datatype::UINT64: + compute_mbr_func_ = compute_mbr; + break; + case Datatype::FLOAT32: + compute_mbr_func_ = compute_mbr; + break; + case Datatype::FLOAT64: + compute_mbr_func_ = compute_mbr; + break; + case Datatype::DATETIME_YEAR: + case Datatype::DATETIME_MONTH: + case Datatype::DATETIME_WEEK: + case Datatype::DATETIME_DAY: + case Datatype::DATETIME_HR: + case Datatype::DATETIME_MIN: + case Datatype::DATETIME_SEC: + case Datatype::DATETIME_MS: + case Datatype::DATETIME_US: + case Datatype::DATETIME_NS: + case Datatype::DATETIME_PS: + case Datatype::DATETIME_FS: + case Datatype::DATETIME_AS: + compute_mbr_func_ = compute_mbr; + break; + default: + compute_mbr_func_ = nullptr; + break; + } +} + +void Dimension::set_expand_range_func() { + switch (type_) { + case Datatype::INT32: + expand_range_func_ = expand_range; + break; + case Datatype::INT64: + expand_range_func_ = expand_range; + break; + case Datatype::INT8: + expand_range_func_ = expand_range; + break; + case Datatype::UINT8: + expand_range_func_ = expand_range; + break; + case Datatype::INT16: + expand_range_func_ = expand_range; + break; + case Datatype::UINT16: + expand_range_func_ = expand_range; + break; + case Datatype::UINT32: + expand_range_func_ = expand_range; + break; + case Datatype::UINT64: + expand_range_func_ = expand_range; + break; + case Datatype::FLOAT32: + expand_range_func_ = expand_range; + break; + case Datatype::FLOAT64: + expand_range_func_ = expand_range; + break; + case Datatype::DATETIME_YEAR: + case Datatype::DATETIME_MONTH: + case Datatype::DATETIME_WEEK: + case Datatype::DATETIME_DAY: + case Datatype::DATETIME_HR: + case Datatype::DATETIME_MIN: + case Datatype::DATETIME_SEC: + case Datatype::DATETIME_MS: + case Datatype::DATETIME_US: + case Datatype::DATETIME_NS: + case Datatype::DATETIME_PS: + case Datatype::DATETIME_FS: + case Datatype::DATETIME_AS: + expand_range_func_ = expand_range; + break; + default: + expand_range_func_ = nullptr; + break; + } +} + +void Dimension::set_expand_range_v_func() { + switch (type_) { + case Datatype::INT32: + expand_range_v_func_ = expand_range_v; + break; + case Datatype::INT64: + expand_range_v_func_ = expand_range_v; + break; + case Datatype::INT8: + expand_range_v_func_ = expand_range_v; + break; + case Datatype::UINT8: + expand_range_v_func_ = expand_range_v; + break; + case Datatype::INT16: + expand_range_v_func_ = expand_range_v; + break; + case Datatype::UINT16: + expand_range_v_func_ = expand_range_v; + break; + case Datatype::UINT32: + expand_range_v_func_ = expand_range_v; + break; + case Datatype::UINT64: + expand_range_v_func_ = expand_range_v; + break; + case Datatype::FLOAT32: + expand_range_v_func_ = expand_range_v; + break; + case Datatype::FLOAT64: + expand_range_v_func_ = expand_range_v; + break; + case Datatype::DATETIME_YEAR: + case Datatype::DATETIME_MONTH: + case Datatype::DATETIME_WEEK: + case Datatype::DATETIME_DAY: + case Datatype::DATETIME_HR: + case Datatype::DATETIME_MIN: + case Datatype::DATETIME_SEC: + case Datatype::DATETIME_MS: + case Datatype::DATETIME_US: + case Datatype::DATETIME_NS: + case Datatype::DATETIME_PS: + case Datatype::DATETIME_FS: + case Datatype::DATETIME_AS: + expand_range_v_func_ = expand_range_v; + break; + default: + expand_range_v_func_ = nullptr; + break; + } +} + +void Dimension::set_expand_to_tile_func() { + switch (type_) { + case Datatype::INT32: + expand_to_tile_func_ = expand_to_tile; + break; + case Datatype::INT64: + expand_to_tile_func_ = expand_to_tile; + break; + case Datatype::INT8: + expand_to_tile_func_ = expand_to_tile; + break; + case Datatype::UINT8: + expand_to_tile_func_ = expand_to_tile; + break; + case Datatype::INT16: + expand_to_tile_func_ = expand_to_tile; + break; + case Datatype::UINT16: + expand_to_tile_func_ = expand_to_tile; + break; + case Datatype::UINT32: + expand_to_tile_func_ = expand_to_tile; + break; + case Datatype::UINT64: + expand_to_tile_func_ = expand_to_tile; + break; + case Datatype::FLOAT32: + expand_to_tile_func_ = expand_to_tile; + break; + case Datatype::FLOAT64: + expand_to_tile_func_ = expand_to_tile; + break; + case Datatype::DATETIME_YEAR: + case Datatype::DATETIME_MONTH: + case Datatype::DATETIME_WEEK: + case Datatype::DATETIME_DAY: + case Datatype::DATETIME_HR: + case Datatype::DATETIME_MIN: + case Datatype::DATETIME_SEC: + case Datatype::DATETIME_MS: + case Datatype::DATETIME_US: + case Datatype::DATETIME_NS: + case Datatype::DATETIME_PS: + case Datatype::DATETIME_FS: + case Datatype::DATETIME_AS: + expand_to_tile_func_ = expand_to_tile; + break; + default: + expand_to_tile_func_ = nullptr; + break; + } +} + void Dimension::set_oob_func() { switch (type_) { case Datatype::INT32: @@ -658,6 +1085,112 @@ void Dimension::set_oob_func() { } } +void Dimension::set_overlap_func() { + switch (type_) { + case Datatype::INT32: + overlap_func_ = overlap; + break; + case Datatype::INT64: + overlap_func_ = overlap; + break; + case Datatype::INT8: + overlap_func_ = overlap; + break; + case Datatype::UINT8: + overlap_func_ = overlap; + break; + case Datatype::INT16: + overlap_func_ = overlap; + break; + case Datatype::UINT16: + overlap_func_ = overlap; + break; + case Datatype::UINT32: + overlap_func_ = overlap; + break; + case Datatype::UINT64: + overlap_func_ = overlap; + break; + case Datatype::FLOAT32: + overlap_func_ = overlap; + break; + case Datatype::FLOAT64: + overlap_func_ = overlap; + break; + case Datatype::DATETIME_YEAR: + case Datatype::DATETIME_MONTH: + case Datatype::DATETIME_WEEK: + case Datatype::DATETIME_DAY: + case Datatype::DATETIME_HR: + case Datatype::DATETIME_MIN: + case Datatype::DATETIME_SEC: + case Datatype::DATETIME_MS: + case Datatype::DATETIME_US: + case Datatype::DATETIME_NS: + case Datatype::DATETIME_PS: + case Datatype::DATETIME_FS: + case Datatype::DATETIME_AS: + overlap_func_ = overlap; + break; + default: + overlap_func_ = nullptr; + break; + } +} + +void Dimension::set_tile_num_func() { + switch (type_) { + case Datatype::INT32: + tile_num_func_ = tile_num; + break; + case Datatype::INT64: + tile_num_func_ = tile_num; + break; + case Datatype::INT8: + tile_num_func_ = tile_num; + break; + case Datatype::UINT8: + tile_num_func_ = tile_num; + break; + case Datatype::INT16: + tile_num_func_ = tile_num; + break; + case Datatype::UINT16: + tile_num_func_ = tile_num; + break; + case Datatype::UINT32: + tile_num_func_ = tile_num; + break; + case Datatype::UINT64: + tile_num_func_ = tile_num; + break; + case Datatype::FLOAT32: + tile_num_func_ = tile_num; + break; + case Datatype::FLOAT64: + tile_num_func_ = tile_num; + break; + case Datatype::DATETIME_YEAR: + case Datatype::DATETIME_MONTH: + case Datatype::DATETIME_WEEK: + case Datatype::DATETIME_DAY: + case Datatype::DATETIME_HR: + case Datatype::DATETIME_MIN: + case Datatype::DATETIME_SEC: + case Datatype::DATETIME_MS: + case Datatype::DATETIME_US: + case Datatype::DATETIME_NS: + case Datatype::DATETIME_PS: + case Datatype::DATETIME_FS: + case Datatype::DATETIME_AS: + tile_num_func_ = tile_num; + break; + default: + tile_num_func_ = nullptr; + break; + } +} + void Dimension::set_value_in_range_func() { switch (type_) { case Datatype::INT32: diff --git a/tiledb/sm/array_schema/dimension.h b/tiledb/sm/array_schema/dimension.h index a71894f2ea2..9d70616eb37 100644 --- a/tiledb/sm/array_schema/dimension.h +++ b/tiledb/sm/array_schema/dimension.h @@ -37,6 +37,8 @@ #include "tiledb/sm/misc/logger.h" #include "tiledb/sm/misc/status.h" +#include "tiledb/sm/misc/types.h" +#include "tiledb/sm/tile/tile.h" namespace tiledb { namespace sm { @@ -113,6 +115,59 @@ class Dimension { /** Returns true if this is an anonymous (unlabled) dimension **/ bool is_anonymous() const; + /** + * Computed the minimum bounding range of the values stored in + * `tile`. + */ + void compute_mbr(const Tile& tile, Range* mbr) const; + + /** + * Computed the minimum bounding range of the values stored in + * `tile`. + */ + template + static void compute_mbr(const Dimension* dim, const Tile& tile, Range* mbr); + + /** + * Crops the input 1D range such that it does not exceed the + * dimension domain. + */ + void crop_range(Range* range) const; + + /** + * Crops the input 1D range such that it does not exceed the + * input dimension domain. + */ + template + static void crop_range(const Dimension* dim, Range* range); + + /** Expand 1D range `r` using value `v`. */ + void expand_range_v(const void* v, Range* r) const; + + /** Expand 1D range `r` using value `v`. */ + template + static void expand_range_v(const Dimension* dim, const void* v, Range* r); + + /** Expand 1D range `r2` using 1D range `r1`. */ + void expand_range(const Range& r1, Range* r2) const; + + /** Expand 1D range `r2` using 1D range `r1`. */ + template + static void expand_range(const Dimension* dim, const Range& r1, Range* r2); + + /** + * Expands the input 1D range to coincide with the dimension tiles. + * It is a noop if the tile extents are null and for real domains. + */ + void expand_to_tile(Range* range) const; + + /** + * Expands the input 1D range to coincide with the dimension tiles. + * It is a noop if the tile extents are null and for real domains. + */ + template + static void expand_to_tile(const Dimension* dim, Range* range); + /** * Returns true if the input coordinate is out-of-bounds with respect * to the dimension domain. @@ -140,13 +195,27 @@ class Dimension { static bool oob( const Dimension* dim, const void* coord, std::string* err_msg); + /** Return true if the input 1D ranges overlap. */ + bool overlap(const Range& r1, const Range& r2) const; + + /** Return true if the input 1D ranges overlap. */ + template + static bool overlap(const Dimension* dim, const Range& r1, const Range& r2); + + /** Return the number of tiles the input range intersects. */ + uint64_t tile_num(const Range& range) const; + + /** Return the number of tiles the input range intersects. */ + template + static uint64_t tile_num(const Dimension* dim, const Range& range); + /** Returns `true` if `value` is within the 1D `range`. */ template static bool value_in_range( - const Dimension* dim, const void* value, const void* range); + const Dimension* dim, const void* value, const Range& range); /** Returns `true` if `value` is within the 1D `range`. */ - bool value_in_range(const void* value, const void* range) const; + bool value_in_range(const void* value, const Range& range) const; /** * Serializes the object members into a binary buffer. @@ -203,6 +272,39 @@ class Dimension { /** The dimension type. */ Datatype type_; + /** + * Stores the appropriate templated compute_mbr() function based on the + * dimension datatype. + */ + std::function + compute_mbr_func_; + + /** + * Stores the appropriate templated crop_range() function based on the + * dimension datatype. + */ + std::function crop_range_func_; + + /** + * Stores the appropriate templated expand_range() function based on the + * dimension datatype. + */ + std::function + expand_range_v_func_; + + /** + * Stores the appropriate templated expand_range() function based on the + * dimension datatype. + */ + std::function + expand_range_func_; + + /** + * Stores the appropriate templated expand_to_tile() function based on the + * dimension datatype. + */ + std::function expand_to_tile_func_; + /** * Stores the appropriate templated oob() function based on the * dimension datatype. @@ -210,11 +312,24 @@ class Dimension { std::function oob_func_; + /** + * Stores the appropriate templated overlap() function based on the + * dimension datatype. + */ + std::function + overlap_func_; + + /** + * Stores the appropriate templated tile_num() function based on the + * dimension datatype. + */ + std::function tile_num_func_; + /** * Stores the appropriate templated value_in_range() function based on the * dimension datatype. */ - std::function + std::function value_in_range_func_; /* ********************************* */ @@ -298,10 +413,31 @@ class Dimension { template Status check_tile_extent() const; - /** Sets the templated oob function. */ + /** Sets the templated compute_mbr() function. */ + void set_compute_mbr_func(); + + /** Sets the templated crop_range() function. */ + void set_crop_range_func(); + + /** Sets the templated expand_range() function. */ + void set_expand_range_func(); + + /** Sets the templated expand_range_v() function. */ + void set_expand_range_v_func(); + + /** Sets the templated expand_to_tile() function. */ + void set_expand_to_tile_func(); + + /** Sets the templated oob() function. */ void set_oob_func(); - /** Sets the templated value_in_range function. */ + /** Sets the templated overlap() function. */ + void set_overlap_func(); + + /** Sets the templated tile_num() function. */ + void set_tile_num_func(); + + /** Sets the templated value_in_range() function. */ void set_value_in_range_func(); }; diff --git a/tiledb/sm/array_schema/domain.cc b/tiledb/sm/array_schema/domain.cc index a9133f73158..38454cce58f 100644 --- a/tiledb/sm/array_schema/domain.cc +++ b/tiledb/sm/array_schema/domain.cc @@ -518,50 +518,9 @@ int Domain::cell_order_cmp( return 0; } -void Domain::crop_domain(void* domain) const { - switch (type_) { - case Datatype::INT32: - crop_domain(static_cast(domain)); - break; - case Datatype::INT64: - crop_domain(static_cast(domain)); - break; - case Datatype::INT8: - crop_domain(static_cast(domain)); - break; - case Datatype::UINT8: - crop_domain(static_cast(domain)); - break; - case Datatype::INT16: - crop_domain(static_cast(domain)); - break; - case Datatype::UINT16: - crop_domain(static_cast(domain)); - break; - case Datatype::UINT32: - crop_domain(static_cast(domain)); - break; - case Datatype::UINT64: - crop_domain(static_cast(domain)); - break; - case Datatype::DATETIME_YEAR: - case Datatype::DATETIME_MONTH: - case Datatype::DATETIME_WEEK: - case Datatype::DATETIME_DAY: - case Datatype::DATETIME_HR: - case Datatype::DATETIME_MIN: - case Datatype::DATETIME_SEC: - case Datatype::DATETIME_MS: - case Datatype::DATETIME_US: - case Datatype::DATETIME_NS: - case Datatype::DATETIME_PS: - case Datatype::DATETIME_FS: - case Datatype::DATETIME_AS: - crop_domain(static_cast(domain)); - break; - default: // Non-applicable to non-integer domains - break; - } +void Domain::crop_ndrange(NDRange* ndrange) const { + for (unsigned d = 0; d < dim_num_; ++d) + dimensions_[d]->crop_range(&(*ndrange)[d]); } // ===== FORMAT ===== @@ -603,6 +562,16 @@ const void* Domain::domain(unsigned int i) const { return dimensions_[i]->domain(); } +NDRange Domain::domain_ndrange() const { + NDRange ret(dim_num_); + for (unsigned d = 0; d < dim_num_; ++d) { + Range r(dimensions_[d]->domain(), 2 * dimensions_[d]->coord_size()); + ret[d] = std::move(r); + } + + return ret; +} + const Dimension* Domain::dimension(unsigned int i) const { if (i > dim_num_) return nullptr; @@ -631,6 +600,16 @@ void Domain::dump(FILE* out) const { } } +void Domain::expand_ndrange(const NDRange& r1, NDRange* r2) const { + for (unsigned d = 0; d < dim_num_; ++d) + dimensions_[d]->expand_range(r1[d], &(*r2)[d]); +} + +void Domain::expand_to_tiles(NDRange* ndrange) const { + for (unsigned d = 0; d < dim_num_; ++d) + dimensions_[d]->expand_to_tile(&(*ndrange)[d]); +} + void Domain::expand_domain(void* domain) const { switch (type_) { case Datatype::INT32: @@ -935,56 +914,21 @@ const void* Domain::tile_extents() const { return tile_extents_; } -uint64_t Domain::tile_num(const void* range) const { - switch (type_) { - case Datatype::INT32: - return tile_num(static_cast(range)); - case Datatype::INT64: - return tile_num(static_cast(range)); - case Datatype::INT8: - return tile_num(static_cast(range)); - case Datatype::UINT8: - return tile_num(static_cast(range)); - case Datatype::INT16: - return tile_num(static_cast(range)); - case Datatype::UINT16: - return tile_num(static_cast(range)); - case Datatype::UINT32: - return tile_num(static_cast(range)); - case Datatype::UINT64: - return tile_num(static_cast(range)); - case Datatype::DATETIME_YEAR: - case Datatype::DATETIME_MONTH: - case Datatype::DATETIME_WEEK: - case Datatype::DATETIME_DAY: - case Datatype::DATETIME_HR: - case Datatype::DATETIME_MIN: - case Datatype::DATETIME_SEC: - case Datatype::DATETIME_MS: - case Datatype::DATETIME_US: - case Datatype::DATETIME_NS: - case Datatype::DATETIME_PS: - case Datatype::DATETIME_FS: - case Datatype::DATETIME_AS: - return tile_num(static_cast(range)); - case Datatype::FLOAT32: - case Datatype::FLOAT64: - // Operation not supported for float domains - case Datatype::CHAR: - case Datatype::STRING_ASCII: - case Datatype::STRING_UTF8: - case Datatype::STRING_UTF16: - case Datatype::STRING_UTF32: - case Datatype::STRING_UCS2: - case Datatype::STRING_UCS4: - case Datatype::ANY: - // Not supported domain types - assert(false); - return 0; +uint64_t Domain::tile_num(const NDRange& ndrange) const { + uint64_t ret = 1; + for (unsigned d = 0; d < dim_num_; ++d) + ret *= dimensions_[d]->tile_num(ndrange[d]); + + return ret; +} + +bool Domain::overlap(const NDRange& r1, const NDRange& r2) const { + for (unsigned d = 0; d < dim_num_; ++d) { + if (!dimensions_[d]->overlap(r1[d], r2[d])) + return false; } - assert(false); - return 0; + return true; } template diff --git a/tiledb/sm/array_schema/domain.h b/tiledb/sm/array_schema/domain.h index 954c714e985..a2a84930c4e 100644 --- a/tiledb/sm/array_schema/domain.h +++ b/tiledb/sm/array_schema/domain.h @@ -34,6 +34,7 @@ #define TILEDB_DOMAIN_H #include "tiledb/sm/misc/status.h" +#include "tiledb/sm/misc/types.h" #include @@ -271,40 +272,9 @@ class Domain { Layout cell_order() const; /** - * Crops the input domain such that it does not exceed the array domain. - * - * @param domain The domain to be cropped. - * @return void + * Crops the input ND range such that it does not exceed the array domain. */ - void crop_domain(void* domain) const; - - /** - * Crops the input domain such that it does not exceed the array domain. - * - * @param domain The domain to be cropped. - * @return void - */ - template < - class T, - typename std::enable_if::value, T>::type* = nullptr> - void crop_domain(T* domain) const { - auto array_domain = static_cast(domain_); - - for (unsigned int i = 0; i < dim_num_; ++i) { - if (domain[2 * i] < array_domain[2 * i]) - domain[2 * i] = array_domain[2 * i]; - if (domain[2 * i + 1] > array_domain[2 * i + 1]) - domain[2 * i + 1] = array_domain[2 * i + 1]; - } - } - - /** No-op for float/double domains. */ - template < - class T, - typename std::enable_if::value, T>::type* = nullptr> - void crop_domain(T* domain) const { - (void)domain; - } + void crop_ndrange(NDRange* ndrange) const; /** Returns the tile order. */ Layout tile_order() const; @@ -318,6 +288,9 @@ class Domain { /** returns the domain along the i-th dimension (nullptr upon error). */ const void* domain(unsigned int i) const; + /** Returns the domain as a N-dimensional range object. */ + NDRange domain_ndrange() const; + /** Returns the i-th dimensions (nullptr upon error). */ const Dimension* dimension(unsigned int i) const; @@ -327,6 +300,17 @@ class Domain { /** Dumps the domain in ASCII format in the selected output. */ void dump(FILE* out) const; + /** Expands ND range `r2` using ND range `r1`. */ + void expand_ndrange(const NDRange& r1, NDRange* r2) const; + + /** + * Expands the input domain such that it coincides with the boundaries of + * the array's regular tiles (i.e., it maps it on the regular tile grid). + * If the array has no regular tile grid or real domain, the function + * does not do anything. + */ + void expand_to_tiles(NDRange* ndrange) const; + /** * Expands the input domain such that it coincides with the boundaries of * the array's regular tiles (i.e., it maps it on the regular tile grid). @@ -600,33 +584,13 @@ class Domain { const void* tile_extent(unsigned int i) const; /** - * Returns the number of tiles contained in the input range. - * - * @note Applicable only to integer domains. - */ - uint64_t tile_num(const void* range) const; - - /** - * Returns the number of tiles contained in the input range. - * - * @note Applicable only to integer domains. + * Returns the number of tiles contained in the input ND range. + * Returns 0 if even a single dimension has non-integral type. */ - template < - class T, - typename std::enable_if::value, T>::type* = nullptr> - uint64_t tile_num(const T* range) const { - // For easy reference - auto tile_extents = static_cast(tile_extents_); - auto domain = static_cast(domain_); + uint64_t tile_num(const NDRange& ndrange) const; - uint64_t ret = 1; - for (unsigned int i = 0; i < dim_num_; ++i) { - uint64_t start = (range[2 * i] - domain[2 * i]) / tile_extents[i]; - uint64_t end = (range[2 * i + 1] - domain[2 * i]) / tile_extents[i]; - ret *= (end - start + 1); - } - return ret; - } + /** Returns true if the two ND ranges overlap. */ + bool overlap(const NDRange& r1, const NDRange& r2) const; /** * Checks the tile order of the input coordinates on the given dimension. diff --git a/tiledb/sm/array_schema/tile_domain.h b/tiledb/sm/array_schema/tile_domain.h index edc997b1e52..5c5862462a7 100644 --- a/tiledb/sm/array_schema/tile_domain.h +++ b/tiledb/sm/array_schema/tile_domain.h @@ -37,6 +37,7 @@ #include #include "tiledb/sm/enums/layout.h" +#include "tiledb/sm/misc/types.h" namespace tiledb { namespace sm { @@ -80,7 +81,7 @@ class TileDomain { unsigned id, unsigned dim_num, const T* domain, - const T* domain_slice, + const std::reference_wrapper& domain_slice, const T* tile_extents, Layout layout) : id_(id) @@ -90,7 +91,7 @@ class TileDomain { , tile_extents_(tile_extents) , layout_(layout) { assert(layout == Layout::ROW_MAJOR || layout == Layout::COL_MAJOR); - compute_tile_domain(domain, domain_slice, tile_extents); + compute_tile_domain(domain, domain_slice.get(), tile_extents); if (layout == Layout::ROW_MAJOR) compute_tile_offsets_row(); else @@ -185,10 +186,10 @@ class TileDomain { // Get overlap ret.resize(2 * dim_num_); auto tile_subarray = this->tile_subarray(tile_coords); - for (unsigned i = 0; i < dim_num_; ++i) { - ret[2 * i] = std::max(tile_subarray[2 * i], domain_slice_[2 * i]); - ret[2 * i + 1] = - std::min(tile_subarray[2 * i + 1], domain_slice_[2 * i + 1]); + for (unsigned d = 0; d < dim_num_; ++d) { + auto ds = (const T*)domain_slice_.get()[d].data(); + ret[2 * d] = std::max(tile_subarray[2 * d], ds[0]); + ret[2 * d + 1] = std::min(tile_subarray[2 * d + 1], ds[1]); } return ret; @@ -245,7 +246,8 @@ class TileDomain { return tile_domain_; } - const T* domain_slice() const { + /** Returns the domain slice. */ + std::reference_wrapper domain_slice() const { return domain_slice_; } @@ -267,7 +269,7 @@ class TileDomain { const T* domain_; /** The domain slice from which the tile domain is constructed. */ - const T* domain_slice_; + std::reference_wrapper domain_slice_; /** The tile extents. */ const T* tile_extents_; @@ -293,17 +295,14 @@ class TileDomain { * `tile_extents`. */ void compute_tile_domain( - const T* domain, const T* domain_slice, const T* tile_extents) { + const T* domain, const NDRange& domain_slice, const T* tile_extents) { tile_domain_.resize(2 * dim_num_); - for (unsigned i = 0; i < dim_num_; ++i) { - assert(domain_slice[2 * i] <= domain_slice[2 * i + 1]); - assert( - domain_slice[2 * i] >= domain[2 * i] && - domain_slice[2 * i + 1] <= domain[2 * i + 1]); - tile_domain_[2 * i] = - (domain_slice[2 * i] - domain[2 * i]) / tile_extents[i]; - tile_domain_[2 * i + 1] = - (domain_slice[2 * i + 1] - domain[2 * i]) / tile_extents[i]; + for (unsigned d = 0; d < dim_num_; ++d) { + auto ds = (const T*)domain_slice[d].data(); + assert(ds[0] <= ds[1]); + assert(ds[0] >= domain[2 * d] && ds[1] <= domain[2 * d + 1]); + tile_domain_[2 * d] = (ds[0] - domain[2 * d]) / tile_extents[d]; + tile_domain_[2 * d + 1] = (ds[1] - domain[2 * d]) / tile_extents[d]; } } diff --git a/tiledb/sm/fragment/fragment_metadata.cc b/tiledb/sm/fragment/fragment_metadata.cc index 0861aeae5f1..ec470a417bf 100644 --- a/tiledb/sm/fragment/fragment_metadata.cc +++ b/tiledb/sm/fragment/fragment_metadata.cc @@ -67,9 +67,7 @@ FragmentMetadata::FragmentMetadata( , dense_(dense) , fragment_uri_(fragment_uri) , timestamp_range_(timestamp_range) { - domain_ = nullptr; meta_file_size_ = 0; - non_empty_domain_ = nullptr; version_ = constants::format_version; tile_index_base_ = 0; sparse_tile_num_ = 0; @@ -86,16 +84,9 @@ FragmentMetadata::FragmentMetadata( } FragmentMetadata::~FragmentMetadata() { - std::free(domain_); - std::free(non_empty_domain_); - auto mbr_num = (uint64_t)mbrs_.size(); for (uint64_t i = 0; i < mbr_num; ++i) std::free(mbrs_[i]); - - auto bounding_coords_num = (uint64_t)bounding_coords_.size(); - for (uint64_t i = 0; i < bounding_coords_num; ++i) - std::free(bounding_coords_[i]); } /* ****************************** */ @@ -106,61 +97,26 @@ const URI& FragmentMetadata::array_uri() const { return array_schema_->array_uri(); } -Status FragmentMetadata::set_mbr(uint64_t tile, const void* mbr) { - switch (array_schema_->coords_type()) { - case Datatype::INT8: - return set_mbr(tile, static_cast(mbr)); - case Datatype::UINT8: - return set_mbr(tile, static_cast(mbr)); - case Datatype::INT16: - return set_mbr(tile, static_cast(mbr)); - case Datatype::UINT16: - return set_mbr(tile, static_cast(mbr)); - case Datatype::INT32: - return set_mbr(tile, static_cast(mbr)); - case Datatype::UINT32: - return set_mbr(tile, static_cast(mbr)); - case Datatype::INT64: - return set_mbr(tile, static_cast(mbr)); - case Datatype::UINT64: - return set_mbr(tile, static_cast(mbr)); - case Datatype::FLOAT32: - return set_mbr(tile, static_cast(mbr)); - case Datatype::FLOAT64: - return set_mbr(tile, static_cast(mbr)); - case Datatype::DATETIME_YEAR: - case Datatype::DATETIME_MONTH: - case Datatype::DATETIME_WEEK: - case Datatype::DATETIME_DAY: - case Datatype::DATETIME_HR: - case Datatype::DATETIME_MIN: - case Datatype::DATETIME_SEC: - case Datatype::DATETIME_MS: - case Datatype::DATETIME_US: - case Datatype::DATETIME_NS: - case Datatype::DATETIME_PS: - case Datatype::DATETIME_FS: - case Datatype::DATETIME_AS: - return set_mbr(tile, static_cast(mbr)); - default: - return LOG_STATUS(Status::FragmentMetadataError( - "Cannot append mbr; Unsupported coordinates type")); - } -} - -template -Status FragmentMetadata::set_mbr(uint64_t tile, const void* mbr) { +Status FragmentMetadata::set_mbr(uint64_t tile, const NDRange& mbr) { // For easy reference uint64_t mbr_size = 2 * array_schema_->coords_size(); tile += tile_index_base_; - // Copy and set MBR - void* new_mbr = std::malloc(mbr_size); - std::memcpy(new_mbr, mbr, mbr_size); + // Copy MBR + auto new_mbr = (unsigned char*)std::malloc(mbr_size); + auto dim_num = array_schema_->dim_num(); + uint64_t offset = 0; + for (unsigned d = 0; d < dim_num; ++d) { + std::memcpy(&new_mbr[offset], mbr[d].data(), mbr[d].size()); + offset += mbr[d].size(); + } + + // Set MBR assert(tile < mbrs_.size()); mbrs_[tile] = new_mbr; - return expand_non_empty_domain(static_cast(mbr)); + // Expand non-empty domain + return expand_non_empty_domain(mbr); } void FragmentMetadata::set_tile_index_base(uint64_t tile_base) { @@ -303,7 +259,7 @@ bool FragmentMetadata::dense() const { return dense_; } -const void* FragmentMetadata::domain() const { +const NDRange& FragmentMetadata::domain() const { return domain_; } @@ -335,8 +291,16 @@ Status FragmentMetadata::get_tile_overlap( const EncryptionKey& encryption_key, const std::vector& range, TileOverlap* tile_overlap) { + auto dim_num = array_schema_->dim_num(); + NDRange ndrange(dim_num); + for (unsigned d = 0; d < dim_num; ++d) { + auto r_size = 2 * array_schema_->dimension(d)->coord_size(); + Range r(range[d], r_size); + ndrange[d] = std::move(r); + } + // Return if the range does not overlap the non-empty domain of the fragment - if (!utils::geometry::overlap(range, (const T*)non_empty_domain_)) + if (!array_schema_->domain()->overlap(ndrange, non_empty_domain_)) return Status::Ok(); // Handle version > 2 @@ -362,31 +326,37 @@ Status FragmentMetadata::get_tile_overlap( Status FragmentMetadata::init(const void* non_empty_domain) { // For easy reference - auto num = array_schema_->attribute_num() + array_schema_->dim_num() + 1; + auto dim_num = array_schema_->dim_num(); + auto num = array_schema_->attribute_num() + dim_num + 1; auto domain = array_schema_->domain(); // Sanity check assert(non_empty_domain != nullptr); - assert(non_empty_domain_ == nullptr); - assert(domain_ == nullptr); + assert(non_empty_domain_.empty()); + assert(domain_.empty()); // Set non-empty domain for dense arrays (for sparse it will be calculated // via the MBRs) - uint64_t domain_size = 2 * array_schema_->coords_size(); if (dense_) { // Set non-empty domain - non_empty_domain_ = std::malloc(domain_size); - std::memcpy(non_empty_domain_, non_empty_domain, domain_size); + auto dom_ptr = (const unsigned char*)non_empty_domain; + non_empty_domain_.resize(dim_num); + for (unsigned d = 0; d < dim_num; ++d) { + auto r_size = 2 * array_schema_->dimension(d)->coord_size(); + Range r(dom_ptr, r_size); + non_empty_domain_[d] = std::move(r); + dom_ptr += r_size; + } + // The following is needed in case the fragment is a result of // dense consolidation, as the consolidator may have expanded // the fragment domain beyond the array domain to include // integral space tiles - domain->crop_domain(non_empty_domain_); + domain->crop_ndrange(&non_empty_domain_); // Set expanded domain - domain_ = std::malloc(domain_size); - std::memcpy(domain_, non_empty_domain_, domain_size); - domain->expand_domain(domain_); + domain_ = non_empty_domain_; + domain->expand_to_tiles(&domain_); } // Set last tile cell number @@ -502,19 +472,10 @@ Status FragmentMetadata::store(const EncryptionKey& encryption_key) { return !st.ok() ? st : st2; } -const void* FragmentMetadata::non_empty_domain() const { +const NDRange& FragmentMetadata::non_empty_domain() { return non_empty_domain_; } -std::vector FragmentMetadata::non_empty_domain_vec() const { - std::vector ret; - ret.reserve(array_schema_->dim_num()); - auto dom = (unsigned char*)non_empty_domain_; - for (unsigned d = 0; d < array_schema_->dim_num(); ++d) - ret.emplace_back(&dom[2 * array_schema_->dimension(d)->coord_size()]); - return ret; -} - Status FragmentMetadata::set_num_tiles(uint64_t num_tiles) { auto num = array_schema_->attribute_num() + 1 + array_schema_->dim_num(); @@ -528,7 +489,6 @@ Status FragmentMetadata::set_num_tiles(uint64_t num_tiles) { if (!dense_) { mbrs_.resize(num_tiles, nullptr); sparse_tile_num_ = num_tiles; - bounding_coords_.resize(num_tiles, nullptr); } return Status::Ok(); @@ -712,7 +672,7 @@ Status FragmentMetadata::get_footer_offset_and_size_v5_or_higher( *size = 0; *size += sizeof(uint32_t); // version *size += sizeof(char); // dense - *size += sizeof(char); // null non-empty domain + *size += sizeof(uint64_t); // non-empty domain size *size += domain_size; // non-empty domain *size += sizeof(uint64_t); // sparse tile num *size += sizeof(uint64_t); // last tile cell num @@ -735,7 +695,15 @@ std::vector FragmentMetadata::compute_overlapping_tile_ids( assert(dense_); std::vector tids; auto dim_num = array_schema_->dim_num(); - auto metadata_domain = static_cast(domain_); + + // TODO: fix + std::vector temp(2 * array_schema_->coords_size()); + uint8_t offset = 0; + for (unsigned d = 0; d < dim_num; ++d) { + std::memcpy(&temp[offset], domain_[d].data(), domain_[d].size()); + offset += domain_[d].size(); + } + auto metadata_domain = (const T*)&temp[0]; // Check if there is any overlap if (!utils::geometry::overlap(subarray, metadata_domain, dim_num)) @@ -773,7 +741,15 @@ FragmentMetadata::compute_overlapping_tile_ids_cov(const T* subarray) const { assert(dense_); std::vector> tids; auto dim_num = array_schema_->dim_num(); - auto metadata_domain = static_cast(domain_); + + // TODO: fix + std::vector temp(2 * array_schema_->coords_size()); + uint8_t offset = 0; + for (unsigned d = 0; d < dim_num; ++d) { + std::memcpy(&temp[offset], domain_[d].data(), domain_[d].size()); + offset += domain_[d].size(); + } + auto metadata_domain = (const T*)&temp[0]; // Check if there is any overlap if (!utils::geometry::overlap(subarray, metadata_domain, dim_num)) @@ -822,46 +798,31 @@ void FragmentMetadata::get_subarray_tile_domain( const T* subarray, T* subarray_tile_domain) const { // For easy reference auto dim_num = array_schema_->dim_num(); - auto domain = static_cast(domain_); auto tile_extents = static_cast(array_schema_->domain()->tile_extents()); // Calculate subarray in tile domain - for (unsigned int i = 0; i < dim_num; ++i) { - auto overlap = std::max(subarray[2 * i], domain[2 * i]); - subarray_tile_domain[2 * i] = (overlap - domain[2 * i]) / tile_extents[i]; + for (unsigned int d = 0; d < dim_num; ++d) { + auto domain = (const T*)domain_[d].data(); + auto overlap = std::max(subarray[2 * d], domain[0]); + subarray_tile_domain[2 * d] = (overlap - domain[0]) / tile_extents[d]; - overlap = std::min(subarray[2 * i + 1], domain[2 * i + 1]); - subarray_tile_domain[2 * i + 1] = - (overlap - domain[2 * i]) / tile_extents[i]; + overlap = std::min(subarray[2 * d + 1], domain[1]); + subarray_tile_domain[2 * d + 1] = (overlap - domain[0]) / tile_extents[d]; } } -template -Status FragmentMetadata::expand_non_empty_domain(const T* mbr) { +Status FragmentMetadata::expand_non_empty_domain(const NDRange& mbr) { std::lock_guard lock(mtx_); - if (non_empty_domain_ == nullptr) { - auto domain_size = 2 * array_schema_->coords_size(); - non_empty_domain_ = std::malloc(domain_size); - if (non_empty_domain_ == nullptr) - return LOG_STATUS(Status::FragmentMetadataError( - "Cannot expand non-empty domain; Memory allocation failed")); - - std::memcpy(non_empty_domain_, mbr, domain_size); + // Case the non-empty domain is not initialized yet + if (non_empty_domain_.empty()) { + non_empty_domain_ = mbr; return Status::Ok(); } - auto dim_num = array_schema_->dim_num(); - auto coords = new T[dim_num]; - for (unsigned i = 0; i < dim_num; ++i) - coords[i] = mbr[2 * i]; - auto non_empty_domain = static_cast(non_empty_domain_); - utils::geometry::expand_mbr(non_empty_domain, coords, dim_num); - for (unsigned i = 0; i < dim_num; ++i) - coords[i] = mbr[2 * i + 1]; - utils::geometry::expand_mbr(non_empty_domain, coords, dim_num); - delete[] coords; + // Expand existing non-empty domain + array_schema_->domain()->expand_ndrange(mbr, &non_empty_domain_); return Status::Ok(); } @@ -956,30 +917,18 @@ Status FragmentMetadata::load_tile_var_sizes( // bounding_coords_num (uint64_t) // bounding_coords_#1 (void*) bounding_coords_#2 (void*) ... Status FragmentMetadata::load_bounding_coords(ConstBuffer* buff) { - uint64_t bounding_coords_size = 2 * array_schema_->coords_size(); - // Get number of bounding coordinates uint64_t bounding_coords_num = 0; - Status st = buff->read(&bounding_coords_num, sizeof(uint64_t)); - if (!st.ok()) { - return LOG_STATUS(Status::FragmentMetadataError( - "Cannot load fragment metadata; Reading number of " - "bounding coordinates failed")); - } + RETURN_NOT_OK(buff->read(&bounding_coords_num, sizeof(uint64_t))); + // Get bounding coordinates - void* bounding_coords; - bounding_coords_.resize(bounding_coords_num, nullptr); + uint64_t bounding_coords_size = 2 * array_schema_->coords_size(); + bounding_coords_.resize(bounding_coords_num); for (uint64_t i = 0; i < bounding_coords_num; ++i) { - bounding_coords = std::malloc(bounding_coords_size); - st = buff->read(bounding_coords, bounding_coords_size); - if (!st.ok()) { - std::free(bounding_coords); - return LOG_STATUS( - Status::FragmentMetadataError("Cannot load fragment metadata; " - "Reading bounding coordinates failed")); - } - bounding_coords_[i] = bounding_coords; + bounding_coords_[i].resize(bounding_coords_size); + RETURN_NOT_OK(buff->read(&bounding_coords_[i][0], bounding_coords_size)); } + return Status::Ok(); } @@ -1114,44 +1063,39 @@ Status FragmentMetadata::load_mbrs(ConstBuffer* buff) { } Status FragmentMetadata::load_non_empty_domain(ConstBuffer* buff) { - if (version_ <= 2) - return load_non_empty_domain_v2(buff); - return load_non_empty_domain_v3(buff); + if (version_ <= 2 || version_ >= 5) + return load_non_empty_domain_v1_v2_and_v5(buff); + // version_ 3 or 4 + return load_non_empty_domain_v3_v4(buff); } // ===== FORMAT ===== // non_empty_domain_size (uint64_t) // non_empty_domain (void*) -Status FragmentMetadata::load_non_empty_domain_v2(ConstBuffer* buff) { +Status FragmentMetadata::load_non_empty_domain_v1_v2_and_v5(ConstBuffer* buff) { // Get domain size uint64_t domain_size = 0; - Status st = buff->read(&domain_size, sizeof(uint64_t)); - if (!st.ok()) { - return LOG_STATUS(Status::FragmentMetadataError( - "Cannot load fragment metadata; Reading domain size failed")); - } + RETURN_NOT_OK(buff->read(&domain_size, sizeof(uint64_t))); // Get non-empty domain - if (domain_size == 0) { - non_empty_domain_ = nullptr; - } else { - non_empty_domain_ = std::malloc(domain_size); - st = buff->read(non_empty_domain_, domain_size); - if (!st.ok()) { - std::free(non_empty_domain_); - // non_empty_domain_ = nullptr; - return LOG_STATUS(Status::FragmentMetadataError( - "Cannot load fragment metadata; Reading domain failed")); + if (domain_size != 0) { + auto dim_num = array_schema_->dim_num(); + std::vector temp(domain_size); + RETURN_NOT_OK(buff->read(&temp[0], domain_size)); + non_empty_domain_.resize(dim_num); + uint64_t offset = 0; + for (unsigned d = 0; d < dim_num; ++d) { + auto coord_size = array_schema_->dimension(d)->coord_size(); + Range r(&temp[offset], 2 * coord_size); + non_empty_domain_[d] = std::move(r); + offset += 2 * coord_size; } } // Get expanded domain - if (non_empty_domain_ == nullptr) { - domain_ = nullptr; - } else { - domain_ = std::malloc(domain_size); - std::memcpy(domain_, non_empty_domain_, domain_size); - array_schema_->domain()->expand_domain(domain_); + if (!non_empty_domain_.empty()) { + domain_ = non_empty_domain_; + array_schema_->domain()->expand_to_tiles(&domain_); } return Status::Ok(); @@ -1160,37 +1104,31 @@ Status FragmentMetadata::load_non_empty_domain_v2(ConstBuffer* buff) { // ===== FORMAT ===== // null non_empty_domain (char) // non_empty_domain (domain_size) -Status FragmentMetadata::load_non_empty_domain_v3(ConstBuffer* buff) { +Status FragmentMetadata::load_non_empty_domain_v3_v4(ConstBuffer* buff) { // Get null non-empty domain bool null_non_empty_domain = false; - Status st = buff->read(&null_non_empty_domain, sizeof(char)); - if (!st.ok()) { - return LOG_STATUS(Status::FragmentMetadataError( - "Cannot load fragment metadata; Reading domain size failed")); - } + RETURN_NOT_OK(buff->read(&null_non_empty_domain, sizeof(char))); // Get non-empty domain - auto domain_size = 2 * array_schema_->coords_size(); - non_empty_domain_ = std::malloc(domain_size); - st = buff->read(non_empty_domain_, domain_size); - if (!st.ok()) { - std::free(non_empty_domain_); - non_empty_domain_ = nullptr; - return LOG_STATUS(Status::FragmentMetadataError( - "Cannot load fragment metadata; Reading domain failed")); - } - if (null_non_empty_domain) { - std::free(non_empty_domain_); - non_empty_domain_ = nullptr; + if (!null_non_empty_domain) { + auto dim_num = array_schema_->dim_num(); + auto domain_size = 2 * array_schema_->coords_size(); + std::vector temp(domain_size); + RETURN_NOT_OK(buff->read(&temp[0], domain_size)); + non_empty_domain_.resize(dim_num); + uint64_t offset = 0; + for (unsigned d = 0; d < dim_num; ++d) { + auto coord_size = array_schema_->dimension(d)->coord_size(); + Range r(&temp[offset], 2 * coord_size); + non_empty_domain_[d] = std::move(r); + offset += 2 * coord_size; + } } // Get expanded domain - if (non_empty_domain_ == nullptr) { - domain_ = nullptr; - } else { - domain_ = std::malloc(domain_size); - std::memcpy(domain_, non_empty_domain_, domain_size); - array_schema_->domain()->expand_domain(domain_); + if (!non_empty_domain_.empty()) { + domain_ = non_empty_domain_; + array_schema_->domain()->expand_to_tiles(&domain_); } return Status::Ok(); @@ -1686,34 +1624,21 @@ Status FragmentMetadata::write_rtree(Buffer* buff) { } // ===== FORMAT ===== -// non_empty_domain_size(uint64_t) non_empty_domain(void*) +// non_empty_domain_size (uint64_t) +// non_empty_domain (uint8_t[]) Status FragmentMetadata::write_non_empty_domain(Buffer* buff) { - uint64_t domain_size = 2 * array_schema_->coords_size(); - bool null_non_empty_domain = (non_empty_domain_ == nullptr); - - // Write null non-empty domain - Status st = buff->write(&null_non_empty_domain, sizeof(char)); - if (!st.ok()) { - return LOG_STATUS(Status::FragmentMetadataError( - "Cannot serialize fragment metadata; Writing non-empty domain failed")); - } + // Write non-empty domain size + uint64_t domain_size = 0; + for (const auto& r : non_empty_domain_) + domain_size += r.size(); + RETURN_NOT_OK(buff->write(&domain_size, sizeof(uint64_t))); // Write non-empty domain - if (non_empty_domain_ != nullptr) { - st = buff->write(non_empty_domain_, domain_size); - if (!st.ok()) { - return LOG_STATUS( - Status::FragmentMetadataError("Cannot serialize fragment metadata; " - "Writing non-empty domain failed")); - } - } else { // Write some dummy values - std::vector d(domain_size, 0); - st = buff->write(&d[0], domain_size); - if (!st.ok()) { - return LOG_STATUS( - Status::FragmentMetadataError("Cannot serialize fragment metadata; " - "Writing non-empty domain failed")); - } + if (!non_empty_domain_.empty()) { + auto dim_num = array_schema_->dim_num(); + for (unsigned d = 0; d < dim_num; ++d) + RETURN_NOT_OK(buff->write( + non_empty_domain_[d].data(), non_empty_domain_[d].size())); } return Status::Ok(); @@ -1919,27 +1844,6 @@ void FragmentMetadata::clean_up() { } // Explicit template instantiations -template Status FragmentMetadata::set_mbr( - uint64_t tile, const void* mbr); -template Status FragmentMetadata::set_mbr( - uint64_t tile, const void* mbr); -template Status FragmentMetadata::set_mbr( - uint64_t tile, const void* mbr); -template Status FragmentMetadata::set_mbr( - uint64_t tile, const void* mbr); -template Status FragmentMetadata::set_mbr( - uint64_t tile, const void* mbr); -template Status FragmentMetadata::set_mbr( - uint64_t tile, const void* mbr); -template Status FragmentMetadata::set_mbr( - uint64_t tile, const void* mbr); -template Status FragmentMetadata::set_mbr( - uint64_t tile, const void* mbr); -template Status FragmentMetadata::set_mbr( - uint64_t tile, const void* mbr); -template Status FragmentMetadata::set_mbr( - uint64_t tile, const void* mbr); - template Status FragmentMetadata::add_max_buffer_sizes( const EncryptionKey& encryption_key, const int8_t* subarray, diff --git a/tiledb/sm/fragment/fragment_metadata.h b/tiledb/sm/fragment/fragment_metadata.h index e5fb3304177..fa71f1cd0f1 100644 --- a/tiledb/sm/fragment/fragment_metadata.h +++ b/tiledb/sm/fragment/fragment_metadata.h @@ -39,6 +39,7 @@ #include #include "tiledb/sm/misc/status.h" +#include "tiledb/sm/misc/types.h" #include "tiledb/sm/misc/uri.h" #include "tiledb/sm/rtree/rtree.h" @@ -168,7 +169,7 @@ class FragmentMetadata { bool dense() const; /** Returns the (expanded) domain in which the fragment is constrained. */ - const void* domain() const; + const NDRange& domain() const; /** Returns the format version of this fragment. */ uint32_t format_version() const; @@ -212,10 +213,7 @@ class FragmentMetadata { Status store(const EncryptionKey& encryption_key); /** Returns the non-empty domain in which the fragment is constrained. */ - const void* non_empty_domain() const; - - /** Returns the non-empty domain in which the fragment is constrained. */ - std::vector non_empty_domain_vec() const; + const NDRange& non_empty_domain(); /** * Simply sets the number of cells for the last tile. @@ -233,19 +231,7 @@ class FragmentMetadata { * @param mbr The MBR to be set. * @return Status */ - Status set_mbr(uint64_t tile, const void* mbr); - - /** - * Sets the input tile's MBR in the fragment metadata. It also expands the - * non-empty domain of the fragment. - * - * @tparam T The coordinates type. - * @param tile The tile index whose MBR will be set. - * @param mbr The MBR to be set. - * @return Status - */ - template - Status set_mbr(uint64_t tile, const void* mbr); + Status set_mbr(uint64_t tile, const NDRange& mbr); /** * Resizes the per-tile metadata vectors for the given number of tiles. This @@ -467,7 +453,7 @@ class FragmentMetadata { std::unordered_map idx_map_; /** A vector storing the first and last coordinates of each tile. */ - std::vector bounding_coords_; + std::vector> bounding_coords_; /** True if the fragment is dense, and false if it is sparse. */ bool dense_; @@ -478,7 +464,7 @@ class FragmentMetadata { * boundaries (if there is a tile grid imposed by tile extents). Note that the * type of the domain must be the same as the type of the array coordinates. */ - void* domain_; + NDRange domain_; /** Stores the size of each attribute file. */ std::vector file_sizes_; @@ -514,11 +500,8 @@ class FragmentMetadata { /** The offsets of the next variable tile for each attribute. */ std::vector next_tile_var_offsets_; - /** - * The non-empty domain in which the fragment is constrained. Note that the - * type of the domain must be the same as the type of the array coordinates. - */ - void* non_empty_domain_; + /** The non-empty domain of the fragment. */ + NDRange non_empty_domain_; /** An RTree for the MBRs. */ RTree rtree_; @@ -608,13 +591,8 @@ class FragmentMetadata { /** * Expands the non-empty domain using the input MBR. - * - * @tparam T The coordinates type. - * @param mbr The MBR to expand the non-empty domain with. - * @return Status */ - template - Status expand_non_empty_domain(const T* mbr); + Status expand_non_empty_domain(const NDRange& mbr); /** Loads the R-tree from storage. */ Status load_rtree(const EncryptionKey& encryption_key); @@ -715,15 +693,15 @@ class FragmentMetadata { /** * Loads the non-empty domain from the input buffer, - * for format versions <= 2. + * for format versions <= 2 and >= 5. */ - Status load_non_empty_domain_v2(ConstBuffer* buff); + Status load_non_empty_domain_v1_v2_and_v5(ConstBuffer* buff); /** * Loads the non-empty domain from the input buffer, - * for format versions >= 3. + * for format versions 3 and 4. */ - Status load_non_empty_domain_v3(ConstBuffer* buff); + Status load_non_empty_domain_v3_v4(ConstBuffer* buff); /** * Loads the tile offsets for the input attribute from the input buffer. diff --git a/tiledb/sm/query/types.h b/tiledb/sm/misc/types.h similarity index 67% rename from tiledb/sm/query/types.h rename to tiledb/sm/misc/types.h index e1757fba510..19ec073330c 100644 --- a/tiledb/sm/query/types.h +++ b/tiledb/sm/misc/types.h @@ -32,6 +32,10 @@ #ifndef TILEDB_TYPES_H #define TILEDB_TYPES_H + +#include +#include + namespace tiledb { namespace sm { @@ -39,6 +43,72 @@ namespace sm { /* TYPE DEFINITIONS */ /* ********************************* */ +/** + * Defines a 1D range (low, high), flattened in a sequence of bytes. + * If the range consists of var-sized values (e.g., strings), then + * the format is: + * + * low_nbytes (uint32) | low | high_nbytes (uint32) | high + */ +class Range { + public: + /** Default constructor. */ + Range() = default; + + /** Constructor setting a range. */ + Range(const void* range, uint64_t range_size) { + set_range(range, range_size); + } + + /** Copy constructor. */ + Range(const Range& range) = default; + + /** Move constructor. */ + Range(Range&& range) = default; + + /** Destructor. */ + ~Range() = default; + + /** Copy-assign operator.*/ + Range& operator=(const Range& range) = default; + + /** Move-assign operator. */ + Range& operator=(Range&& range) = default; + + /** Sets a range. */ + void set_range(const void* range, uint64_t range_size) { + range_.resize(range_size); + std::memcpy(&range_[0], range, range_size); + } + + /** Returns the pointer to the range flattened bytes. */ + const void* data() const { + return &range_[0]; + } + + /** Returns true if the range is empty. */ + bool empty() const { + return range_.empty(); + } + + /** Returns the range size in bytes. */ + uint64_t size() const { + return range_.size(); + } + + /** Equality operator. */ + bool operator==(const Range& r) const { + return range_ == r.range_; + } + + private: + /** The range as a flat byte vector.*/ + std::vector range_; +}; + +/** An N-dimensional range, consisting of a vector of 1D ranges. */ +typedef std::vector NDRange; + /** Contains the buffer(s) and buffer size(s) for some attribute / dimension. */ struct QueryBuffer { /** diff --git a/tiledb/sm/misc/utils.cc b/tiledb/sm/misc/utils.cc index 484f05bf16a..7c79bb46a25 100644 --- a/tiledb/sm/misc/utils.cc +++ b/tiledb/sm/misc/utils.cc @@ -659,6 +659,19 @@ inline bool rect_in_rect( return true; } +template +inline bool rect_in_rect( + const T* rect_a, const NDRange& rect_b, unsigned int dim_num) { + for (unsigned d = 0; d < dim_num; ++d) { + auto rb = (const T*)rect_b[d].data(); + if (rect_a[2 * d] < rb[0] || rect_a[2 * d] > rb[1] || + rect_a[2 * d + 1] < rb[0] || rect_a[2 * d + 1] > rb[1]) + return false; + } + + return true; +} + template void compute_mbr_union( unsigned dim_num, const T* mbrs, uint64_t mbr_num, T* mbr_union) { @@ -675,33 +688,6 @@ void compute_mbr_union( expand_mbr_with_mbr(mbr_union, &mbrs[i * 2 * dim_num], dim_num); } -template -void expand_mbr(T* mbr, const T* coords, unsigned int dim_num) { - for (unsigned int i = 0; i < dim_num; ++i) { - // Update lower bound on dimension i - if (mbr[2 * i] > coords[i]) - mbr[2 * i] = coords[i]; - - // Update upper bound on dimension i - if (mbr[2 * i + 1] < coords[i]) - mbr[2 * i + 1] = coords[i]; - } -} - -template -void expand_mbr(const std::vector& coords, const uint64_t pos, T* mbr) { - auto dim_num = (unsigned)coords.size(); - for (unsigned int d = 0; d < dim_num; ++d) { - // Update lower bound on dimension i - if (mbr[2 * d] > coords[d][pos]) - mbr[2 * d] = coords[d][pos]; - - // Update upper bound on dimension i - if (mbr[2 * d + 1] < coords[d][pos]) - mbr[2 * d + 1] = coords[d][pos]; - } -} - template void expand_mbr_with_mbr(T* mbr_a, const T* mbr_b, unsigned int dim_num) { for (unsigned int i = 0; i < dim_num; ++i) { @@ -935,47 +921,26 @@ template bool rect_in_rect( template bool rect_in_rect( const uint64_t* rect_a, const uint64_t* rect_b, unsigned int dim_num); -template void expand_mbr( - int* mbr, const int* coords, unsigned int dim_num); -template void expand_mbr( - int64_t* mbr, const int64_t* coords, unsigned int dim_num); -template void expand_mbr( - float* mbr, const float* coords, unsigned int dim_num); -template void expand_mbr( - double* mbr, const double* coords, unsigned int dim_num); -template void expand_mbr( - int8_t* mbr, const int8_t* coords, unsigned int dim_num); -template void expand_mbr( - uint8_t* mbr, const uint8_t* coords, unsigned int dim_num); -template void expand_mbr( - int16_t* mbr, const int16_t* coords, unsigned int dim_num); -template void expand_mbr( - uint16_t* mbr, const uint16_t* coords, unsigned int dim_num); -template void expand_mbr( - uint32_t* mbr, const uint32_t* coords, unsigned int dim_num); -template void expand_mbr( - uint64_t* mbr, const uint64_t* coords, unsigned int dim_num); - -template void expand_mbr( - const std::vector& coords, const uint64_t pos, int8_t* mbr); -template void expand_mbr( - const std::vector& coords, const uint64_t pos, uint8_t* mbr); -template void expand_mbr( - const std::vector& coords, const uint64_t pos, int16_t* mbr); -template void expand_mbr( - const std::vector& coords, const uint64_t pos, uint16_t* mbr); -template void expand_mbr( - const std::vector& coords, const uint64_t pos, int32_t* mbr); -template void expand_mbr( - const std::vector& coords, const uint64_t pos, uint32_t* mbr); -template void expand_mbr( - const std::vector& coords, const uint64_t pos, int64_t* mbr); -template void expand_mbr( - const std::vector& coords, const uint64_t pos, uint64_t* mbr); -template void expand_mbr( - const std::vector& coords, const uint64_t pos, float* mbr); -template void expand_mbr( - const std::vector& coords, const uint64_t pos, double* mbr); +template bool rect_in_rect( + const int* rect_a, const NDRange& rect_b, unsigned int dim_num); +template bool rect_in_rect( + const int64_t* rect_a, const NDRange& rect_b, unsigned int dim_num); +template bool rect_in_rect( + const float* react_a, const NDRange& rect_b, unsigned int dim_num); +template bool rect_in_rect( + const double* rect_a, const NDRange& rect_b, unsigned int dim_num); +template bool rect_in_rect( + const int8_t* rect_a, const NDRange& rect_b, unsigned int dim_num); +template bool rect_in_rect( + const uint8_t* rect_a, const NDRange& rect_b, unsigned int dim_num); +template bool rect_in_rect( + const int16_t* rect_a, const NDRange& rect_b, unsigned int dim_num); +template bool rect_in_rect( + const uint16_t* rect_a, const NDRange& rect_b, unsigned int dim_num); +template bool rect_in_rect( + const uint32_t* rect_a, const NDRange& rect_b, unsigned int dim_num); +template bool rect_in_rect( + const uint64_t* rect_a, const NDRange& rect_b, unsigned int dim_num); template void expand_mbr_with_mbr( int* mbr_a, const int* mbr_b, unsigned int dim_num); diff --git a/tiledb/sm/misc/utils.h b/tiledb/sm/misc/utils.h index c0bc9a19a29..3ccd7323f4d 100644 --- a/tiledb/sm/misc/utils.h +++ b/tiledb/sm/misc/utils.h @@ -40,6 +40,7 @@ #include #include "tiledb/sm/misc/status.h" +#include "tiledb/sm/misc/types.h" namespace tiledb { namespace sm { @@ -255,6 +256,18 @@ bool coords_in_rect( template bool rect_in_rect(const T* rect_a, const T* rect_b, unsigned int dim_num); +/** + * Checks if `rect_a` is inside `rect_b`. + * + * @tparam T The domain type. + * @param rect_a The first rectangle. + * @param rect_b The second rectangle. + * @param dim_num The number of dimensions. + * @return `true` if `rect_a` is inside `rect_b` and `false` otherwise. + */ +template +bool rect_in_rect(const T* rect_a, const NDRange& rect_b, unsigned int dim_num); + /** * Computes the union of a set of MBRs (rectangles). * @@ -269,34 +282,6 @@ template void compute_mbr_union( unsigned dim_num, const T* mbrs, uint64_t mbr_num, T* mbr_union); -/** - * Expands the input MBR so that it encompasses the input coordinates. - * - * @tparam T The type of the MBR and coordinates. - * @param mbr The input MBR to be expanded. - * @param coords The input coordinates to expand the MBR. - * @param dim_num The number of dimensions of the MBR and coordinates. - * @return void - */ -template -void expand_mbr(T* mbr, const T* coords, unsigned int dim_num); - -/** - * Expands the input MBR with the input coordinates. The input coordinates - * are given as a vector of values, one per dimension, and a position in - * the vectors along the dimensions. The input MBR will be expanded using - * the values along the dimensions located at the input position. - * - * @tparam T The type of the MBR and coordinates. - * @param coords A vector of coordinate buffers, one per dimension. - * @param pos The position of the values in the coordinate buffers that - * will be used to expand the MBR with. - * @param mbr The input MBR to be expanded. - * @return void - */ -template -void expand_mbr(const std::vector& coords, const uint64_t pos, T* mbr); - /** * Expands `mbr_a` so that it encompasses `mbr_b`. * diff --git a/tiledb/sm/query/read_cell_slab_iter.cc b/tiledb/sm/query/read_cell_slab_iter.cc index 09c0de95414..51a788e7799 100644 --- a/tiledb/sm/query/read_cell_slab_iter.cc +++ b/tiledb/sm/query/read_cell_slab_iter.cc @@ -163,7 +163,7 @@ void ReadCellSlabIter::compute_cell_slab_start( template void ReadCellSlabIter::compute_cell_slab_overlap( const CellSlab& cell_slab, - const T* frag_domain, + const NDRange& frag_domain, std::vector* slab_overlap, uint64_t* overlap_length, unsigned* overlap_type) { @@ -175,17 +175,15 @@ void ReadCellSlabIter::compute_cell_slab_overlap( slab_end = slab_start + cell_slab.length_ - 1; // Check if there is any overlap - for (unsigned i = 0; i < dim_num; ++i) { - if (i == slab_dim) { - if (slab_end < frag_domain[2 * i] || - slab_start > frag_domain[2 * i + 1]) { + for (unsigned d = 0; d < dim_num; ++d) { + auto dom = (const T*)frag_domain[d].data(); + if (d == slab_dim) { + if (slab_end < dom[0] || slab_start > dom[1]) { *overlap_type = 0; *overlap_length = 0; return; } - } else if ( - cell_slab.coords_[i] < frag_domain[2 * i] || - cell_slab.coords_[i] > frag_domain[2 * i + 1]) { + } else if (cell_slab.coords_[d] < dom[0] || cell_slab.coords_[d] > dom[1]) { *overlap_type = 0; *overlap_length = 0; return; @@ -193,8 +191,9 @@ void ReadCellSlabIter::compute_cell_slab_overlap( } // There is some overlap - T overlap_start = std::max(slab_start, frag_domain[2 * slab_dim]); - T overlap_end = std::min(slab_end, frag_domain[2 * slab_dim + 1]); + auto dom = (const T*)frag_domain[slab_dim].data(); + T overlap_start = std::max(slab_start, dom[0]); + T overlap_end = std::min(slab_end, dom[1]); *slab_overlap = cell_slab.coords_; (*slab_overlap)[slab_dim] = overlap_start; *overlap_length = overlap_end - overlap_start + 1; @@ -279,8 +278,7 @@ void ReadCellSlabIter::compute_result_cell_slabs_dense( const CellSlab& cell_slab, ResultSpaceTile* result_space_tile) { std::list> to_process; to_process.push_back(cell_slab); - const auto& frag_domains = result_space_tile->frag_domains_; - auto& result_tiles = result_space_tile->result_tiles_; + const auto& frag_domains = result_space_tile->frag_domains(); auto dim_num = domain_->dim_num(); std::vector slab_overlap; slab_overlap.resize(dim_num); @@ -295,7 +293,7 @@ void ReadCellSlabIter::compute_result_cell_slabs_dense( for (const auto& fd : frag_domains) { for (auto pit = to_process.begin(); pit != to_process.end();) { compute_cell_slab_overlap( - *pit, fd.second, &slab_overlap, &overlap_length, &overlap_type); + *pit, fd.second.get(), &slab_overlap, &overlap_length, &overlap_type); // No overlap if (overlap_type == 0) { @@ -305,10 +303,9 @@ void ReadCellSlabIter::compute_result_cell_slabs_dense( // Compute new result cell slab compute_cell_slab_start( - &slab_overlap[0], result_space_tile->start_coords_, &start); - auto tit = result_tiles.find(fd.first); - assert(tit != result_tiles.end()); - result_cell_slabs.emplace_back(&(tit->second), start, overlap_length); + &slab_overlap[0], result_space_tile->start_coords(), &start); + auto tile = result_space_tile->result_tile(fd.first); + result_cell_slabs.emplace_back(tile, start, overlap_length); // If it is partial overlap, we need to create up to two new cell slabs // and re-insert to the head of `to_process` (so that the rest of the @@ -353,7 +350,7 @@ void ReadCellSlabIter::compute_result_cell_slabs_empty( uint64_t start; for (auto pit = to_process.begin(); pit != to_process.end(); ++pit) { compute_cell_slab_start( - &pit->coords_[0], result_space_tile.start_coords_, &start); + &pit->coords_[0], result_space_tile.start_coords(), &start); result_cell_slabs->emplace_back(nullptr, start, pit->length_); } } diff --git a/tiledb/sm/query/read_cell_slab_iter.h b/tiledb/sm/query/read_cell_slab_iter.h index 46fb48a0a5a..d6bd4337b2a 100644 --- a/tiledb/sm/query/read_cell_slab_iter.h +++ b/tiledb/sm/query/read_cell_slab_iter.h @@ -34,6 +34,7 @@ #define TILEDB_READ_CELL_SLAB_ITER_H #include "tiledb/sm/array_schema/domain.h" +#include "tiledb/sm/misc/types.h" #include "tiledb/sm/query/result_cell_slab.h" #include "tiledb/sm/query/result_coords.h" #include "tiledb/sm/query/result_space_tile.h" @@ -228,7 +229,7 @@ class ReadCellSlabIter { */ void compute_cell_slab_overlap( const CellSlab& cell_slab, - const T* frag_domain, + const NDRange& frag_domain, std::vector* slab_overlap, uint64_t* overlap_length, unsigned* overlap_type); diff --git a/tiledb/sm/query/reader.cc b/tiledb/sm/query/reader.cc index 1b005fe2684..443d338154d 100644 --- a/tiledb/sm/query/reader.cc +++ b/tiledb/sm/query/reader.cc @@ -558,7 +558,7 @@ void Reader::compute_result_space_tiles( // Create result space tile and insert into the map auto r = result_space_tiles->emplace(coords, ResultSpaceTile()); auto& result_space_tile = r.first->second; - result_space_tile.start_coords_ = start_coords; + result_space_tile.set_start_coords(start_coords); // Add fragment info to the result space tile for (unsigned f = 0; f < fragment_num; ++f) { @@ -583,10 +583,10 @@ void Reader::compute_result_space_tiles( // Include this fragment in the space tile auto frag_domain = frag_tile_domains[f].domain_slice(); auto frag_idx = frag_tile_domains[f].id(); - result_space_tile.frag_domains_.emplace_back(frag_idx, frag_domain); + result_space_tile.append_frag_domain(frag_idx, frag_domain); auto tile_idx = frag_tile_domains[f].tile_pos(coords); ResultTile result_tile(frag_idx, tile_idx, domain); - result_space_tile.result_tiles_[frag_idx] = result_tile; + result_space_tile.set_result_tile(frag_idx, result_tile); } } } @@ -649,22 +649,22 @@ Status Reader::compute_result_cell_slabs( Status Reader::compute_range_result_coords( unsigned frag_idx, ResultTile* tile, - const std::vector& range, + const NDRange& ndrange, std::vector* result_coords) const { auto coords_num = tile->cell_num(); auto fragment_num = fragment_metadata_.size(); for (uint64_t pos = 0; pos < coords_num; ++pos) { // Check if the coordinates are in the range - if (!tile->coord_in_rect(pos, range)) + if (!tile->coord_in_rect(pos, ndrange)) continue; // Check if the coordinates are overwritten by a future dense fragment bool overwritten = false; for (unsigned f = frag_idx + 1; !overwritten && f < fragment_num; ++f) { if (fragment_metadata_[f]->dense()) { - overwritten = tile->coord_in_rect( - pos, fragment_metadata_[f]->non_empty_domain_vec()); + overwritten = + tile->coord_in_rect(pos, fragment_metadata_[f]->non_empty_domain()); if (overwritten) break; } @@ -769,9 +769,9 @@ Status Reader::compute_range_result_coords( if (!sparse_tile_overwritten(f, t->first)) RETURN_NOT_OK(get_all_result_coords(&tile, range_result_coords)); } else { // Partial overlap - auto range = subarray.range(range_idx); + auto ndrange = subarray.ndrange(range_idx); RETURN_NOT_OK(compute_range_result_coords( - f, &tile, range, range_result_coords)); + f, &tile, ndrange, range_result_coords)); } ++t; } @@ -1139,6 +1139,7 @@ void Reader::compute_result_space_tiles( // For easy reference auto dim_num = array_schema_->dim_num(); auto domain = (const T*)array_schema_->domain()->domain(); + auto domain_ndrange = array_schema_->domain()->domain_ndrange(); auto tile_extents = (const T*)array_schema_->domain()->tile_extents(); auto tile_order = array_schema_->tile_order(); @@ -1148,10 +1149,13 @@ void Reader::compute_result_space_tiles( if (fragment_num > 0) { for (int i = fragment_num - 1; i >= 0; --i) { if (fragment_metadata_[i]->dense()) { - auto non_empty_domain = - (const T*)fragment_metadata_[i]->non_empty_domain(); frag_tile_domains.emplace_back( - i, dim_num, domain, non_empty_domain, tile_extents, tile_order); + i, + dim_num, + domain, + fragment_metadata_[i]->non_empty_domain(), + tile_extents, + tile_order); } } } @@ -1159,7 +1163,7 @@ void Reader::compute_result_space_tiles( // Get tile coords and array domain const auto& tile_coords = subarray.tile_coords(); TileDomain array_tile_domain( - UINT32_MAX, dim_num, domain, domain, tile_extents, tile_order); + UINT32_MAX, dim_num, domain, domain_ndrange, tile_extents, tile_order); // Compute result space tiles compute_result_space_tiles( @@ -1986,7 +1990,7 @@ bool Reader::sparse_tile_overwritten( for (unsigned f = frag_idx + 1; f < fragment_num; ++f) { if (fragment_metadata_[f]->dense() && utils::geometry::rect_in_rect( - mbr, (const T*)fragment_metadata_[f]->non_empty_domain(), dim_num)) + mbr, fragment_metadata_[f]->non_empty_domain(), dim_num)) return true; } diff --git a/tiledb/sm/query/reader.h b/tiledb/sm/query/reader.h index 3f756ab4f2d..c270eb928a0 100644 --- a/tiledb/sm/query/reader.h +++ b/tiledb/sm/query/reader.h @@ -40,11 +40,11 @@ #include "tiledb/sm/array_schema/tile_domain.h" #include "tiledb/sm/misc/status.h" +#include "tiledb/sm/misc/types.h" #include "tiledb/sm/misc/uri.h" #include "tiledb/sm/query/result_cell_slab.h" #include "tiledb/sm/query/result_coords.h" #include "tiledb/sm/query/result_space_tile.h" -#include "tiledb/sm/query/types.h" #include "tiledb/sm/query/write_cell_slab_iter.h" #include "tiledb/sm/subarray/subarray_partitioner.h" @@ -557,8 +557,8 @@ class Reader { * from the input result tile. * * @param frag_idx The id of the fragment that the result tile belongs to. - * @param The result tile. - * @param range An N-dimensional range (where N is equal to the number + * @param tile The result tile. + * @param ndrange An N-dimensional range (where N is equal to the number * of dimensions of the array). * @param result_coords The overlapping coordinates to retrieve. * @return Status @@ -566,7 +566,7 @@ class Reader { Status compute_range_result_coords( unsigned frag_idx, ResultTile* tile, - const std::vector& range, + const NDRange& ndrange, std::vector* result_coords) const; /** diff --git a/tiledb/sm/query/result_space_tile.h b/tiledb/sm/query/result_space_tile.h index f74e058661e..3cab949b9ef 100644 --- a/tiledb/sm/query/result_space_tile.h +++ b/tiledb/sm/query/result_space_tile.h @@ -33,10 +33,13 @@ #ifndef TILEDB_RESULT_SPACE_TILE_H #define TILEDB_RESULT_SPACE_TILE_H +#include +#include #include #include #include +#include "tiledb/sm/misc/types.h" #include "tiledb/sm/query/result_tile.h" namespace tiledb { @@ -48,24 +51,8 @@ namespace sm { * @tparam The datatype of the array domain. */ template -struct ResultSpaceTile { - /** The (global) coordinates of the first cell in the space tile. */ - std::vector start_coords_; - - /** - * A vector of pairs `(fragment id, fragment domain)`, sorted on - * fragment id in descending order. Note that only fragments - * with domains that intersect this space tile will be included - * in this vector. - */ - std::vector> frag_domains_; - - /** - * The (dense) result tiles for this space tile, as a map - * `(fragment id) -> (result tile)`. - */ - std::map result_tiles_; - +class ResultSpaceTile { + public: /** Default constructor. */ ResultSpaceTile() = default; @@ -85,19 +72,78 @@ struct ResultSpaceTile { /** Default move-assign operator. */ ResultSpaceTile& operator=(ResultSpaceTile&& result_space_tile) = default; + /** Returns the fragment domains. */ + const std::vector>>& + frag_domains() const { + return frag_domains_; + } + + /** Returns the result tiles. */ + const std::map& result_tiles() const { + return result_tiles_; + } + + /** Returns the start coordinates. */ + const std::vector& start_coords() const { + return start_coords_; + } + + /** Sets the start coordinates. */ + void set_start_coords(const std::vector& start_coords) { + start_coords_ = start_coords; + } + + /** Appends a fragment domain. */ + void append_frag_domain( + unsigned frag_idx, const std::reference_wrapper& dom) { + frag_domains_.emplace_back(frag_idx, dom); + } + + /** Sets the input result tile for the given fragment. */ + void set_result_tile(unsigned frag_idx, const ResultTile& result_tile) { + assert(result_tiles_.count(frag_idx) == 0); + result_tiles_[frag_idx] = result_tile; + } + + /** Returns the result tile for the input fragment. */ + ResultTile* result_tile(unsigned frag_idx) { + auto it = result_tiles_.find(frag_idx); + assert(it != result_tiles_.end()); + return &(it->second); + } + /** Equality operator (mainly for debugging purposes). */ bool operator==(const ResultSpaceTile& rst) const { if (frag_domains_.size() != rst.frag_domains_.size()) return false; for (size_t i = 0; i < frag_domains_.size(); ++i) { if (!(frag_domains_[i].first == rst.frag_domains_[i].first && - frag_domains_[i].second == rst.frag_domains_[i].second)) + frag_domains_[i].second.get() == rst.frag_domains_[i].second.get())) return false; } return start_coords_ == rst.start_coords_ && result_tiles_ == rst.result_tiles_; } + + private: + /** The (global) coordinates of the first cell in the space tile. */ + std::vector start_coords_; + + /** + * A vector of pairs `(fragment id, fragment domain)`, sorted on + * fragment id in descending order. Note that only fragments + * with domains that intersect this space tile will be included + * in this vector. + */ + std::vector>> + frag_domains_; + + /** + * The (dense) result tiles for this space tile, as a map + * `(fragment id) -> (result tile)`. + */ + std::map result_tiles_; }; } // namespace sm diff --git a/tiledb/sm/query/result_tile.cc b/tiledb/sm/query/result_tile.cc index e1601904c67..61d567c7cf8 100644 --- a/tiledb/sm/query/result_tile.cc +++ b/tiledb/sm/query/result_tile.cc @@ -147,8 +147,7 @@ const void* ResultTile::coord(uint64_t pos, unsigned dim_idx) const { return nullptr; } -bool ResultTile::coord_in_rect( - uint64_t pos, const std::vector& rect) const { +bool ResultTile::coord_in_rect(uint64_t pos, const NDRange& rect) const { auto dim_num = domain_->dim_num(); for (unsigned d = 0; d < dim_num; ++d) { if (!domain_->dimension(d)->value_in_range(coord(pos, d), rect[d])) diff --git a/tiledb/sm/query/result_tile.h b/tiledb/sm/query/result_tile.h index 2387a6022ce..968ab8f48f2 100644 --- a/tiledb/sm/query/result_tile.h +++ b/tiledb/sm/query/result_tile.h @@ -38,6 +38,7 @@ #include #include "tiledb/sm/misc/constants.h" +#include "tiledb/sm/misc/types.h" #include "tiledb/sm/tile/tile.h" namespace tiledb { @@ -115,7 +116,7 @@ class ResultTile { * Returns true if the coordinates at position `pos` are inside * the input multi-dimensional rectangle. */ - bool coord_in_rect(uint64_t pos, const std::vector& rect) const; + bool coord_in_rect(uint64_t pos, const NDRange& rect) const; /** Returns the coordinate size on the input dimension. */ uint64_t coord_size(unsigned dim_idx) const; diff --git a/tiledb/sm/query/writer.cc b/tiledb/sm/query/writer.cc index 815eb85325c..6ba5a126a92 100644 --- a/tiledb/sm/query/writer.cc +++ b/tiledb/sm/query/writer.cc @@ -954,60 +954,6 @@ Status Writer::compute_coord_dups(std::set* coord_dups) const { STATS_FUNC_OUT(writer_compute_coord_dups_global); } -Status Writer::compute_coords_metadata( - const std::unordered_map>& tiles, - FragmentMetadata* meta) const { - STATS_FUNC_IN(writer_compute_coords_metadata); - - auto coords_type = array_schema_->coords_type(); - switch (coords_type) { - case Datatype::INT8: - return compute_coords_metadata(tiles, meta); - case Datatype::UINT8: - return compute_coords_metadata(tiles, meta); - case Datatype::INT16: - return compute_coords_metadata(tiles, meta); - case Datatype::UINT16: - return compute_coords_metadata(tiles, meta); - case Datatype::INT32: - return compute_coords_metadata(tiles, meta); - case Datatype::UINT32: - return compute_coords_metadata(tiles, meta); - case Datatype::INT64: - return compute_coords_metadata(tiles, meta); - case Datatype::UINT64: - return compute_coords_metadata(tiles, meta); - case Datatype::FLOAT32: - assert(!array_schema_->dense()); - return compute_coords_metadata(tiles, meta); - case Datatype::FLOAT64: - assert(!array_schema_->dense()); - return compute_coords_metadata(tiles, meta); - case Datatype::DATETIME_YEAR: - case Datatype::DATETIME_MONTH: - case Datatype::DATETIME_WEEK: - case Datatype::DATETIME_DAY: - case Datatype::DATETIME_HR: - case Datatype::DATETIME_MIN: - case Datatype::DATETIME_SEC: - case Datatype::DATETIME_MS: - case Datatype::DATETIME_US: - case Datatype::DATETIME_NS: - case Datatype::DATETIME_PS: - case Datatype::DATETIME_FS: - case Datatype::DATETIME_AS: - return compute_coords_metadata(tiles, meta); - default: - return LOG_STATUS(Status::WriterError( - "Cannot compute coordinates metadata; Unsupported domain type")); - } - - return Status::Ok(); - - STATS_FUNC_OUT(writer_compute_coords_metadata); -} - -template Status Writer::compute_coords_metadata( const std::unordered_map>& tiles, FragmentMetadata* meta) const { @@ -1028,29 +974,17 @@ Status Writer::compute_coords_metadata( // Compute MBRs auto statuses = parallel_for(0, tile_num, [&](uint64_t t) { - std::vector mbr(2 * dim_num); - std::vector data(dim_num); - uint64_t cell_num = UINT64_MAX; + NDRange mbr(dim_num); + std::vector data(dim_num); for (unsigned d = 0; d < dim_num; ++d) { - const auto& dim_name = array_schema_->dimension(d)->name(); + auto dim = array_schema_->dimension(d); + const auto& dim_name = dim->name(); auto tiles_it = tiles.find(dim_name); assert(tiles_it != tiles.end()); - data[d] = (T*)(tiles_it->second[t].internal_data()); - assert( - cell_num == UINT64_MAX || cell_num == tiles_it->second[t].cell_num()); - cell_num = tiles_it->second[t].cell_num(); - - // Initialize MBR with the first coords - mbr[2 * d] = data[d][0]; - mbr[2 * d + 1] = data[d][0]; + dim->compute_mbr(tiles_it->second[t], &mbr[d]); } - // Expand the MBR with the rest coords - assert(cell_num > 0); - for (uint64_t c = 1; c < cell_num; ++c) - utils::geometry::expand_mbr(data, c, &mbr[0]); - - meta->set_mbr(t, &mbr[0]); + meta->set_mbr(t, mbr); return Status::Ok(); }); @@ -1496,7 +1430,14 @@ Status Writer::init_tile_dense_cell_range_iters( domain->get_tile_domain(&subarray[0], &tile_domain[0]); for (unsigned i = 0; i < dim_num; ++i) tile_coords[i] = tile_domain[2 * i]; - auto tile_num = domain->tile_num(&subarray[0]); + + // TODO: fix + NDRange sub(dim_num); + for (unsigned d = 0; d < dim_num; ++d) { + Range r(&subarray[2 * d], 2 * sizeof(T)); + sub[d] = std::move(r); + } + auto tile_num = domain->tile_num(sub); // Iterate over all tiles in the tile domain iters->clear(); diff --git a/tiledb/sm/query/writer.h b/tiledb/sm/query/writer.h index c894b100f6b..7a0683e627b 100644 --- a/tiledb/sm/query/writer.h +++ b/tiledb/sm/query/writer.h @@ -39,7 +39,7 @@ #include "tiledb/sm/fragment/written_fragment_info.h" #include "tiledb/sm/misc/status.h" -#include "tiledb/sm/query/types.h" +#include "tiledb/sm/misc/types.h" #include "tiledb/sm/query/write_cell_slab_iter.h" #include "tiledb/sm/tile/tile.h" @@ -471,20 +471,6 @@ class Writer { const std::unordered_map>& tiles, FragmentMetadata* meta) const; - /** - * Computes the coordinates metadata (e.g., MBRs). - * - * @tparam T The domain type. - * @param tiles The tiles to calculate the coords metadata from. It is - * a vector of vectors, one vector of tiles per dimension. - * @param meta The fragment metadata that will store the coords metadata. - * @return Status - */ - template - Status compute_coords_metadata( - const std::unordered_map>& tiles, - FragmentMetadata* meta) const; - /** * Computes the cell ranges to be written, derived from a * dense cell range iterator for a specific tile. diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index 7d51367b966..9cfe972776e 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -541,68 +541,19 @@ Status StorageManager::array_get_non_empty_domain( if (metadata.empty()) return Status::Ok(); - // Compute domain + NDRange temp_dom = metadata[0]->non_empty_domain(); + auto metadata_num = metadata.size(); + for (size_t j = 1; j < metadata_num; ++j) { + const auto& meta_dom = metadata[j]->non_empty_domain(); + array_schema->domain()->expand_ndrange(meta_dom, &temp_dom); + } + auto dim_num = array_schema->dim_num(); - switch (array_schema->coords_type()) { - case Datatype::INT32: - array_get_non_empty_domain( - metadata, dim_num, static_cast(domain)); - break; - case Datatype::INT64: - array_get_non_empty_domain( - metadata, dim_num, static_cast(domain)); - break; - case Datatype::FLOAT32: - array_get_non_empty_domain( - metadata, dim_num, static_cast(domain)); - break; - case Datatype::FLOAT64: - array_get_non_empty_domain( - metadata, dim_num, static_cast(domain)); - break; - case Datatype::INT8: - array_get_non_empty_domain( - metadata, dim_num, static_cast(domain)); - break; - case Datatype::UINT8: - array_get_non_empty_domain( - metadata, dim_num, static_cast(domain)); - break; - case Datatype::INT16: - array_get_non_empty_domain( - metadata, dim_num, static_cast(domain)); - break; - case Datatype::UINT16: - array_get_non_empty_domain( - metadata, dim_num, static_cast(domain)); - break; - case Datatype::UINT32: - array_get_non_empty_domain( - metadata, dim_num, static_cast(domain)); - break; - case Datatype::UINT64: - array_get_non_empty_domain( - metadata, dim_num, static_cast(domain)); - break; - case Datatype::DATETIME_YEAR: - case Datatype::DATETIME_MONTH: - case Datatype::DATETIME_WEEK: - case Datatype::DATETIME_DAY: - case Datatype::DATETIME_HR: - case Datatype::DATETIME_MIN: - case Datatype::DATETIME_SEC: - case Datatype::DATETIME_MS: - case Datatype::DATETIME_US: - case Datatype::DATETIME_NS: - case Datatype::DATETIME_PS: - case Datatype::DATETIME_FS: - case Datatype::DATETIME_AS: - array_get_non_empty_domain( - metadata, dim_num, static_cast(domain)); - break; - default: - return LOG_STATUS(Status::StorageManagerError( - "Cannot get non-empty domain; Invalid coordinates type")); + auto domain_c = (unsigned char*)domain; + uint64_t offset = 0; + for (unsigned d = 0; d < dim_num; ++d) { + std::memcpy(&domain_c[offset], temp_dom[d].data(), temp_dom[d].size()); + offset += temp_dom[d].size(); } *is_empty = false; @@ -809,6 +760,7 @@ Status StorageManager::get_fragment_info( return array_close_for_reads(array_uri); uint64_t domain_size = 2 * array_schema->coords_size(); + auto dim_num = array_schema->dim_num(); for (auto meta : fragment_metadata) { const auto& uri = meta->fragment_uri(); bool sparse = !meta->dense(); @@ -817,7 +769,14 @@ Status StorageManager::get_fragment_info( non_empty_domain.resize(domain_size); // Get fragment non-empty domain - std::memcpy(&non_empty_domain[0], meta->non_empty_domain(), domain_size); + const auto meta_non_empty_domain = meta->non_empty_domain(); + auto non_empty_domain_ptr = (unsigned char*)&non_empty_domain[0]; + for (unsigned d = 0; d < dim_num; ++d) { + auto range_size = 2 * array_schema->dimension(d)->coord_size(); + std::memcpy( + non_empty_domain_ptr, meta_non_empty_domain[d].data(), range_size); + non_empty_domain_ptr += range_size; + } // Get fragment size uint64_t size; @@ -825,10 +784,7 @@ Status StorageManager::get_fragment_info( meta->fragment_size(&size), array_close_for_reads(array_uri)); // Compute expanded non-empty domain only for dense fragments - std::vector expanded_non_empty_domain; - expanded_non_empty_domain.resize(domain_size); - std::memcpy( - &expanded_non_empty_domain[0], meta->non_empty_domain(), domain_size); + auto expanded_non_empty_domain = non_empty_domain; if (!sparse) array_schema->domain()->expand_domain( (void*)&expanded_non_empty_domain[0]); @@ -887,27 +843,32 @@ Status StorageManager::get_fragment_info( } // Get fragment non-empty domain - FragmentMetadata metadata( + FragmentMetadata meta( this, array_schema, fragment_uri, timestamp_range, !sparse); - RETURN_NOT_OK(metadata.load(encryption_key)); + RETURN_NOT_OK(meta.load(encryption_key)); // This is important for format version > 2 - sparse = !metadata.dense(); + sparse = !meta.dense(); // Get fragment size uint64_t size; - RETURN_NOT_OK(metadata.fragment_size(&size)); + RETURN_NOT_OK(meta.fragment_size(&size)); uint64_t domain_size = 2 * array_schema->coords_size(); std::vector non_empty_domain; non_empty_domain.resize(domain_size); - std::memcpy(&non_empty_domain[0], metadata.non_empty_domain(), domain_size); + const auto meta_non_empty_domain = meta.non_empty_domain(); + auto non_empty_domain_ptr = (unsigned char*)&non_empty_domain[0]; + auto dim_num = array_schema->dim_num(); + for (unsigned d = 0; d < dim_num; ++d) { + auto range_size = 2 * array_schema->dimension(d)->coord_size(); + std::memcpy( + non_empty_domain_ptr, meta_non_empty_domain[d].data(), range_size); + non_empty_domain_ptr += range_size; + } // Compute expanded non-empty domain only for dense fragments - std::vector expanded_non_empty_domain; - expanded_non_empty_domain.resize(domain_size); - std::memcpy( - &expanded_non_empty_domain[0], metadata.non_empty_domain(), domain_size); + auto expanded_non_empty_domain = non_empty_domain; if (!sparse) array_schema->domain()->expand_domain((void*)&expanded_non_empty_domain[0]); @@ -1542,32 +1503,6 @@ Status StorageManager::write(const URI& uri, void* data, uint64_t size) const { /* PRIVATE METHODS */ /* ****************************** */ -template -void StorageManager::array_get_non_empty_domain( - const std::vector& metadata, - unsigned dim_num, - T* domain) { - assert(!metadata.empty()); - uint64_t domain_size = 2 * sizeof(T) * dim_num; - auto non_empty_domain = - static_cast(metadata[0]->non_empty_domain()); - std::memcpy(domain, non_empty_domain, domain_size); - - // Expand with the rest of the fragments - auto metadata_num = metadata.size(); - auto coords = new T[dim_num]; - for (size_t j = 1; j < metadata_num; ++j) { - non_empty_domain = static_cast(metadata[j]->non_empty_domain()); - for (unsigned i = 0; i < dim_num; ++i) - coords[i] = non_empty_domain[2 * i]; - utils::geometry::expand_mbr(domain, coords, dim_num); - for (unsigned i = 0; i < dim_num; ++i) - coords[i] = non_empty_domain[2 * i + 1]; - utils::geometry::expand_mbr(domain, coords, dim_num); - } - delete[] coords; -} - Status StorageManager::array_open_without_fragments( const URI& array_uri, const EncryptionKey& encryption_key, diff --git a/tiledb/sm/storage_manager/storage_manager.h b/tiledb/sm/storage_manager/storage_manager.h index f2df909121e..093dcb19bd0 100644 --- a/tiledb/sm/storage_manager/storage_manager.h +++ b/tiledb/sm/storage_manager/storage_manager.h @@ -793,21 +793,6 @@ class StorageManager { /* PRIVATE METHODS */ /* ********************************* */ - /** - * Retrieves the non-empty domain from the input fragment metadata. This is - * the union of the non-empty domains of the fragments. - * - * @param metadata The metadata of all fragments in the array. - * @param dim_num The number of dimensions in the domain. - * @param domain The domain to be retrieved. - * @return void - */ - template - void array_get_non_empty_domain( - const std::vector& metadata, - unsigned dim_num, - T* domain); - /** * This is an auxiliary function to the other `array_open*` functions. * It opens the array, retrieves an `OpenArray` instance, acquires diff --git a/tiledb/sm/subarray/subarray.cc b/tiledb/sm/subarray/subarray.cc index b0637fdc81f..45a029ee98f 100644 --- a/tiledb/sm/subarray/subarray.cc +++ b/tiledb/sm/subarray/subarray.cc @@ -720,6 +720,43 @@ std::vector Subarray::range(uint64_t range_idx) const { return ret; } +NDRange Subarray::ndrange(uint64_t range_idx) const { + NDRange ret; + uint64_t tmp_idx = range_idx; + auto dim_num = this->dim_num(); + auto cell_order = array_->array_schema()->cell_order(); + auto layout = (layout_ == Layout::UNORDERED) ? cell_order : layout_; + auto array_schema = array_->array_schema(); + + if (layout == Layout::ROW_MAJOR) { + for (unsigned d = 0; d < dim_num; ++d) { + auto coord_size = array_schema->dimension(d)->coord_size(); + ret.emplace_back( + ranges_[d].get_range(tmp_idx / range_offsets_[d]), 2 * coord_size); + tmp_idx %= range_offsets_[d]; + } + } else if (layout == Layout::COL_MAJOR) { + for (unsigned d = dim_num - 1;; --d) { + auto coord_size = array_schema->dimension(d)->coord_size(); + ret.emplace_back( + ranges_[d].get_range(tmp_idx / range_offsets_[d]), 2 * coord_size); + tmp_idx %= range_offsets_[d]; + if (d == 0) + break; + } + std::reverse(ret.begin(), ret.end()); + } else { + assert(layout == Layout::GLOBAL_ORDER); + assert(range_num() == 1); + for (unsigned d = 0; d < dim_num; ++d) { + auto coord_size = array_schema->dimension(d)->coord_size(); + ret.emplace_back(ranges_[d].get_range(0), 2 * coord_size); + } + } + + return ret; +} + const Subarray::Ranges* Subarray::ranges_for_dim(uint32_t dim_idx) const { return &ranges_[dim_idx]; } diff --git a/tiledb/sm/subarray/subarray.h b/tiledb/sm/subarray/subarray.h index 98c3a855d57..18690e1fd0d 100644 --- a/tiledb/sm/subarray/subarray.h +++ b/tiledb/sm/subarray/subarray.h @@ -37,6 +37,7 @@ #include "tiledb/sm/enums/datatype.h" #include "tiledb/sm/misc/logger.h" #include "tiledb/sm/misc/tile_overlap.h" +#include "tiledb/sm/misc/types.h" #include #include @@ -446,6 +447,15 @@ class Subarray { */ std::vector range(uint64_t range_idx) const; + /** + * Returns the multi-dimensional range with the input id, based on the + * order imposed on the the subarray ranges by the layout. If ``layout_`` + * is UNORDERED, then the range layout will be the same as the array's + * cell order, since this will lead to more beneficial tile access + * patterns upon a read query. + */ + NDRange ndrange(uint64_t range_idx) const; + /** * Returns the `Ranges` for the given dimension index. * @note Intended for serialization only