From b7514077148a4511626b73552ed5e4e52f1cf4b3 Mon Sep 17 00:00:00 2001 From: Stavros Papadopoulos Date: Wed, 11 Dec 2019 11:26:04 -0500 Subject: [PATCH] User can now set coordinate buffers separately for each dimension to write queries. --- test/src/helpers.cc | 71 ++- test/src/helpers.h | 35 +- test/src/unit-CellSlabIter.cc | 21 +- test/src/unit-ReadCellSlabIter.cc | 33 +- test/src/unit-Reader.cc | 1 + test/src/unit-Subarray.cc | 13 +- test/src/unit-SubarrayPartitioner-dense.cc | 29 +- test/src/unit-SubarrayPartitioner-error.cc | 5 +- test/src/unit-SubarrayPartitioner-sparse.cc | 69 +-- test/src/unit-capi-async.cc | 2 + test/src/unit-capi-consolidation.cc | 2 + test/src/unit-capi-dense_array.cc | 2 + test/src/unit-capi-dense_array_2.cc | 165 +++---- test/src/unit-capi-incomplete-2.cc | 2 + test/src/unit-capi-incomplete.cc | 2 + test/src/unit-capi-metadata.cc | 13 +- test/src/unit-capi-query_2.cc | 2 + test/src/unit-capi-serialized_queries.cc | 92 ++++ test/src/unit-capi-sparse_array.cc | 484 ++++++++++++++++++++ test/src/unit-capi-string.cc | 2 + test/src/unit-cppapi-metadata.cc | 13 +- tiledb/sm/array_schema/array_schema.cc | 46 +- tiledb/sm/array_schema/array_schema.h | 26 +- tiledb/sm/array_schema/dimension.cc | 5 + tiledb/sm/array_schema/dimension.h | 3 + tiledb/sm/query/query.cc | 39 +- tiledb/sm/query/query.h | 28 +- tiledb/sm/query/reader.cc | 19 +- tiledb/sm/query/reader.h | 10 +- tiledb/sm/query/types.h | 78 +--- tiledb/sm/query/writer.cc | 242 +++++++--- tiledb/sm/query/writer.h | 53 ++- tiledb/sm/rest/rest_client.cc | 57 +-- tiledb/sm/serialization/query.cc | 146 ++---- 34 files changed, 1256 insertions(+), 554 deletions(-) diff --git a/test/src/helpers.cc b/test/src/helpers.cc index aaf644ac4399..f0fe568ade3e 100644 --- a/test/src/helpers.cc +++ b/test/src/helpers.cc @@ -33,6 +33,9 @@ #include "helpers.h" #include "catch.hpp" +namespace tiledb { +namespace test { + template void check_partitions( tiledb::sm::SubarrayPartitioner& partitioner, @@ -449,7 +452,7 @@ void write_array( tiledb_ctx_t* ctx, const std::string& array_name, tiledb_layout_t layout, - const AttrBuffers& attr_buffers) { + const QueryBuffers& buffers) { // Open array tiledb_array_t* array; int rc = tiledb_array_alloc(ctx, array_name.c_str(), &array); @@ -465,7 +468,7 @@ void write_array( CHECK(rc == TILEDB_OK); // Set buffers - for (const auto& b : attr_buffers) { + for (const auto& b : buffers) { if (b.second.var_ == nullptr) { // Fixed-sized rc = tiledb_query_set_buffer( ctx, @@ -510,7 +513,7 @@ void read_array( tiledb_array_t* array, const SubarrayRanges& ranges, tiledb_layout_t layout, - const AttrBuffers& attr_buffers) { + const QueryBuffers& buffers) { // Create query tiledb_query_t* query; int rc = tiledb_query_alloc(ctx, array, TILEDB_READ, &query); @@ -529,7 +532,7 @@ void read_array( } // Set buffers - for (const auto& b : attr_buffers) { + for (const auto& b : buffers) { if (b.second.var_ == nullptr) { // Fixed-sized rc = tiledb_query_set_buffer( ctx, @@ -569,22 +572,31 @@ void read_array( template void check_subarray( tiledb::sm::Subarray& subarray, const SubarrayRanges& ranges); + template void check_subarray( tiledb::sm::Subarray& subarray, const SubarrayRanges& ranges); + template void check_subarray( tiledb::sm::Subarray& subarray, const SubarrayRanges& ranges); + template void check_subarray( tiledb::sm::Subarray& subarray, const SubarrayRanges& ranges); + template void check_subarray( tiledb::sm::Subarray& subarray, const SubarrayRanges& ranges); + template void check_subarray( tiledb::sm::Subarray& subarray, const SubarrayRanges& ranges); + template void check_subarray( tiledb::sm::Subarray& subarray, const SubarrayRanges& ranges); + template void check_subarray( tiledb::sm::Subarray& subarray, const SubarrayRanges& ranges); + template void check_subarray( tiledb::sm::Subarray& subarray, const SubarrayRanges& ranges); + template void check_subarray( tiledb::sm::Subarray& subarray, const SubarrayRanges& ranges); @@ -593,46 +605,55 @@ template void create_subarray( const SubarrayRanges& ranges, tiledb::sm::Layout layout, tiledb::sm::Subarray* subarray); + template void create_subarray( tiledb::sm::Array* array, const SubarrayRanges& ranges, tiledb::sm::Layout layout, tiledb::sm::Subarray* subarray); + template void create_subarray( tiledb::sm::Array* array, const SubarrayRanges& ranges, tiledb::sm::Layout layout, tiledb::sm::Subarray* subarray); + template void create_subarray( tiledb::sm::Array* array, const SubarrayRanges& ranges, tiledb::sm::Layout layout, tiledb::sm::Subarray* subarray); + template void create_subarray( tiledb::sm::Array* array, const SubarrayRanges& ranges, tiledb::sm::Layout layout, tiledb::sm::Subarray* subarray); + template void create_subarray( tiledb::sm::Array* array, const SubarrayRanges& ranges, tiledb::sm::Layout layout, tiledb::sm::Subarray* subarray); + template void create_subarray( tiledb::sm::Array* array, const SubarrayRanges& ranges, tiledb::sm::Layout layout, tiledb::sm::Subarray* subarray); + template void create_subarray( tiledb::sm::Array* array, const SubarrayRanges& ranges, tiledb::sm::Layout layout, tiledb::sm::Subarray* subarray); + template void create_subarray( tiledb::sm::Array* array, const SubarrayRanges& ranges, tiledb::sm::Layout layout, tiledb::sm::Subarray* subarray); + template void create_subarray( tiledb::sm::Array* array, const SubarrayRanges& ranges, @@ -643,38 +664,47 @@ template void check_partitions( tiledb::sm::SubarrayPartitioner& partitioner, const std::vector>& partitions, bool last_unsplittable); + template void check_partitions( tiledb::sm::SubarrayPartitioner& partitioner, const std::vector>& partitions, bool last_unsplittable); + template void check_partitions( tiledb::sm::SubarrayPartitioner& partitioner, const std::vector>& partitions, bool last_unsplittable); + template void check_partitions( tiledb::sm::SubarrayPartitioner& partitioner, const std::vector>& partitions, bool last_unsplittable); + template void check_partitions( tiledb::sm::SubarrayPartitioner& partitioner, const std::vector>& partitions, bool last_unsplittable); + template void check_partitions( tiledb::sm::SubarrayPartitioner& partitioner, const std::vector>& partitions, bool last_unsplittable); + template void check_partitions( tiledb::sm::SubarrayPartitioner& partitioner, const std::vector>& partitions, bool last_unsplittable); + template void check_partitions( tiledb::sm::SubarrayPartitioner& partitioner, const std::vector>& partitions, bool last_unsplittable); + template void check_partitions( tiledb::sm::SubarrayPartitioner& partitioner, const std::vector>& partitions, bool last_unsplittable); + template void check_partitions( tiledb::sm::SubarrayPartitioner& partitioner, const std::vector>& partitions, @@ -685,58 +715,71 @@ template void read_array( tiledb_array_t* array, const SubarrayRanges& ranges, tiledb_layout_t layout, - const AttrBuffers& attr_buffers); + const QueryBuffers& buffers); + template void read_array( tiledb_ctx_t* ctx, tiledb_array_t* array, const SubarrayRanges& ranges, tiledb_layout_t layout, - const AttrBuffers& attr_buffers); + const QueryBuffers& buffers); + template void read_array( tiledb_ctx_t* ctx, tiledb_array_t* array, const SubarrayRanges& ranges, tiledb_layout_t layout, - const AttrBuffers& attr_buffers); + const QueryBuffers& buffers); + template void read_array( tiledb_ctx_t* ctx, tiledb_array_t* array, const SubarrayRanges& ranges, tiledb_layout_t layout, - const AttrBuffers& attr_buffers); + const QueryBuffers& buffers); + template void read_array( tiledb_ctx_t* ctx, tiledb_array_t* array, const SubarrayRanges& ranges, tiledb_layout_t layout, - const AttrBuffers& attr_buffers); + const QueryBuffers& buffers); + template void read_array( tiledb_ctx_t* ctx, tiledb_array_t* array, const SubarrayRanges& ranges, tiledb_layout_t layout, - const AttrBuffers& attr_buffers); + const QueryBuffers& buffers); + template void read_array( tiledb_ctx_t* ctx, tiledb_array_t* array, const SubarrayRanges& ranges, tiledb_layout_t layout, - const AttrBuffers& attr_buffers); + const QueryBuffers& buffers); + template void read_array( tiledb_ctx_t* ctx, tiledb_array_t* array, const SubarrayRanges& ranges, tiledb_layout_t layout, - const AttrBuffers& attr_buffers); + const QueryBuffers& buffers); + template void read_array( tiledb_ctx_t* ctx, tiledb_array_t* array, const SubarrayRanges& ranges, tiledb_layout_t layout, - const AttrBuffers& attr_buffers); + const QueryBuffers& buffers); + template void read_array( tiledb_ctx_t* ctx, tiledb_array_t* array, const SubarrayRanges& ranges, tiledb_layout_t layout, - const AttrBuffers& attr_buffers); + const QueryBuffers& buffers); + +} // End of namespace test + +} // End of namespace tiledb diff --git a/test/src/helpers.h b/test/src/helpers.h index b8fb866f5212..7da78b622791 100644 --- a/test/src/helpers.h +++ b/test/src/helpers.h @@ -42,31 +42,38 @@ #include #include +namespace tiledb { + +namespace test { + // For easy reference typedef std::pair Compressor; template using SubarrayRanges = std::vector>; -/** Helper struct for the buffers of an attribute (fixed- or var-sized). */ -struct AttrBuffer { +/** + * Helper struct for the buffers of an attribute/dimension + * (fixed- or var-sized). + */ +struct QueryBuffer { /** - * For fixed-sized attributes, it contains the fixed-sized values. - * For var-sized attributes, it contains the offsets. + * For fixed-sized attributes/dimensions, it contains the fixed-sized values. + * For var-sized attributes/dimensions, it contains the offsets. * var buffer is nullptr. */ void* fixed_; /** Size of fixed buffer. */ uint64_t fixed_size_; /** - * For fixed-sized attributes, it is `nullptr`. - * For var-sized attributes, it contains the var-sized values. + * For fixed-sized attributes/dimensions, it is `nullptr`. + * For var-sized attributes/dimensions, it contains the var-sized values. */ void* var_; /** Size of var buffer. */ uint64_t var_size_; }; -/** Map attribute_name -> AttrBuffer */ -typedef std::map AttrBuffers; +/** Map attribute/dimension name -> QueryBuffer */ +typedef std::map QueryBuffers; /** * Checks that the input partitioner produces the input partitions @@ -297,13 +304,13 @@ int set_attribute_compression_filter( * @param ctx The TileDB context. * @param array_name The array name. * @param layout The layout to write into. - * @param attr_buffers The attribute buffers to be written. + * @param buffers The attribute/dimension buffers to be written. */ void write_array( tiledb_ctx_t* ctx, const std::string& array_name, tiledb_layout_t layout, - const AttrBuffers& attr_buffers); + const QueryBuffers& buffers); /** * Performs a single read to an array. @@ -313,7 +320,7 @@ void write_array( * @param array The input array. * @param The subarray ranges. * @param layout The query layout. - * @param attr_buffers The attribute buffers to be read. + * @param buffers The attribute/dimension buffers to be read. */ template void read_array( @@ -321,6 +328,10 @@ void read_array( tiledb_array_t* array, const SubarrayRanges& ranges, tiledb_layout_t layout, - const AttrBuffers& attr_buffers); + const QueryBuffers& buffers); + +} // End of namespace test + +} // End of namespace tiledb #endif diff --git a/test/src/unit-CellSlabIter.cc b/test/src/unit-CellSlabIter.cc index 510215bff217..6476819d4acd 100644 --- a/test/src/unit-CellSlabIter.cc +++ b/test/src/unit-CellSlabIter.cc @@ -44,6 +44,7 @@ #include using namespace tiledb::sm; +using namespace tiledb::test; /* ********************************* */ /* STRUCT DEFINITION */ @@ -151,8 +152,8 @@ TEST_CASE_METHOD( {"a", "b"}, {TILEDB_INT32, TILEDB_INT32}, {1, TILEDB_VAR_NUM}, - {::Compressor(TILEDB_FILTER_LZ4, -1), - ::Compressor(TILEDB_FILTER_LZ4, -1)}, + {tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1), + tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1)}, TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR, 2); @@ -200,8 +201,8 @@ TEST_CASE_METHOD( {"a", "b"}, {TILEDB_INT32, TILEDB_INT32}, {1, TILEDB_VAR_NUM}, - {::Compressor(TILEDB_FILTER_LZ4, -1), - ::Compressor(TILEDB_FILTER_LZ4, -1)}, + {tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1), + tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1)}, TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR, 2); @@ -254,8 +255,8 @@ TEST_CASE_METHOD( {"a", "b"}, {TILEDB_INT32, TILEDB_INT32}, {1, TILEDB_VAR_NUM}, - {::Compressor(TILEDB_FILTER_LZ4, -1), - ::Compressor(TILEDB_FILTER_LZ4, -1)}, + {tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1), + tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1)}, TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR, 2); @@ -306,8 +307,8 @@ TEST_CASE_METHOD( {"a", "b"}, {TILEDB_INT32, TILEDB_INT32}, {1, TILEDB_VAR_NUM}, - {::Compressor(TILEDB_FILTER_LZ4, -1), - ::Compressor(TILEDB_FILTER_LZ4, -1)}, + {tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1), + tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1)}, TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR, 2); @@ -355,8 +356,8 @@ TEST_CASE_METHOD( {"a", "b"}, {TILEDB_INT32, TILEDB_INT32}, {1, TILEDB_VAR_NUM}, - {::Compressor(TILEDB_FILTER_LZ4, -1), - ::Compressor(TILEDB_FILTER_LZ4, -1)}, + {tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1), + tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1)}, TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR, 2); diff --git a/test/src/unit-ReadCellSlabIter.cc b/test/src/unit-ReadCellSlabIter.cc index 54a92a50181a..3150f50d1145 100644 --- a/test/src/unit-ReadCellSlabIter.cc +++ b/test/src/unit-ReadCellSlabIter.cc @@ -44,6 +44,7 @@ #include using namespace tiledb::sm; +using namespace tiledb::test; /* ********************************* */ /* STRUCT DEFINITION */ @@ -199,8 +200,8 @@ TEST_CASE_METHOD( {"a", "b"}, {TILEDB_INT32, TILEDB_INT32}, {1, TILEDB_VAR_NUM}, - {::Compressor(TILEDB_FILTER_LZ4, -1), - ::Compressor(TILEDB_FILTER_LZ4, -1)}, + {tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1), + tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1)}, TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR, 2); @@ -257,8 +258,8 @@ TEST_CASE_METHOD( {"a", "b"}, {TILEDB_INT32, TILEDB_INT32}, {1, TILEDB_VAR_NUM}, - {::Compressor(TILEDB_FILTER_LZ4, -1), - ::Compressor(TILEDB_FILTER_LZ4, -1)}, + {tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1), + tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1)}, TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR, 2); @@ -315,8 +316,8 @@ TEST_CASE_METHOD( {"a", "b"}, {TILEDB_INT32, TILEDB_INT32}, {1, TILEDB_VAR_NUM}, - {::Compressor(TILEDB_FILTER_LZ4, -1), - ::Compressor(TILEDB_FILTER_LZ4, -1)}, + {tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1), + tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1)}, TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR, 2); @@ -379,8 +380,8 @@ TEST_CASE_METHOD( {"a", "b"}, {TILEDB_INT32, TILEDB_INT32}, {1, TILEDB_VAR_NUM}, - {::Compressor(TILEDB_FILTER_LZ4, -1), - ::Compressor(TILEDB_FILTER_LZ4, -1)}, + {tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1), + tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1)}, TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR, 2); @@ -580,8 +581,8 @@ TEST_CASE_METHOD( {"a", "b"}, {TILEDB_INT32, TILEDB_INT32}, {1, TILEDB_VAR_NUM}, - {::Compressor(TILEDB_FILTER_LZ4, -1), - ::Compressor(TILEDB_FILTER_LZ4, -1)}, + {tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1), + tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1)}, tile_order, cell_order, 2); @@ -749,8 +750,8 @@ TEST_CASE_METHOD( {"a", "b"}, {TILEDB_INT32, TILEDB_INT32}, {1, TILEDB_VAR_NUM}, - {::Compressor(TILEDB_FILTER_LZ4, -1), - ::Compressor(TILEDB_FILTER_LZ4, -1)}, + {tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1), + tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1)}, tile_order, cell_order, 2); @@ -931,8 +932,8 @@ TEST_CASE_METHOD( {"a", "b"}, {TILEDB_INT32, TILEDB_INT32}, {1, TILEDB_VAR_NUM}, - {::Compressor(TILEDB_FILTER_LZ4, -1), - ::Compressor(TILEDB_FILTER_LZ4, -1)}, + {tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1), + tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1)}, tile_order, cell_order, 2); @@ -1158,8 +1159,8 @@ TEST_CASE_METHOD( {"a", "b"}, {TILEDB_INT32, TILEDB_INT32}, {1, TILEDB_VAR_NUM}, - {::Compressor(TILEDB_FILTER_LZ4, -1), - ::Compressor(TILEDB_FILTER_LZ4, -1)}, + {tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1), + tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1)}, tile_order, cell_order, 2); diff --git a/test/src/unit-Reader.cc b/test/src/unit-Reader.cc index 82d2f6c4653a..c6b0b63fc69a 100644 --- a/test/src/unit-Reader.cc +++ b/test/src/unit-Reader.cc @@ -44,6 +44,7 @@ #include using namespace tiledb::sm; +using namespace tiledb::test; /* ********************************* */ /* STRUCT DEFINITION */ diff --git a/test/src/unit-Subarray.cc b/test/src/unit-Subarray.cc index 95b53755db64..91afde69bde4 100644 --- a/test/src/unit-Subarray.cc +++ b/test/src/unit-Subarray.cc @@ -44,6 +44,7 @@ #include using namespace tiledb::sm; +using namespace tiledb::test; /* ********************************* */ /* STRUCT DEFINITION */ @@ -120,8 +121,8 @@ TEST_CASE_METHOD( {"a", "b"}, {TILEDB_INT32, TILEDB_INT32}, {1, TILEDB_VAR_NUM}, - {::Compressor(TILEDB_FILTER_LZ4, -1), - ::Compressor(TILEDB_FILTER_LZ4, -1)}, + {tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1), + tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1)}, TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR, 2); @@ -243,8 +244,8 @@ TEST_CASE_METHOD( {"a", "b"}, {TILEDB_INT32, TILEDB_INT32}, {1, TILEDB_VAR_NUM}, - {::Compressor(TILEDB_FILTER_LZ4, -1), - ::Compressor(TILEDB_FILTER_LZ4, -1)}, + {tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1), + tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1)}, tile_order, TILEDB_ROW_MAJOR, 2); @@ -296,8 +297,8 @@ TEST_CASE_METHOD( {"a", "b"}, {TILEDB_INT32, TILEDB_INT32}, {1, TILEDB_VAR_NUM}, - {::Compressor(TILEDB_FILTER_LZ4, -1), - ::Compressor(TILEDB_FILTER_LZ4, -1)}, + {tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1), + tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1)}, TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR, 2); diff --git a/test/src/unit-SubarrayPartitioner-dense.cc b/test/src/unit-SubarrayPartitioner-dense.cc index e5cc789a70db..07d20419c9ae 100644 --- a/test/src/unit-SubarrayPartitioner-dense.cc +++ b/test/src/unit-SubarrayPartitioner-dense.cc @@ -44,6 +44,7 @@ #include using namespace tiledb::sm; +using namespace tiledb::test; /* ********************************* */ /* STRUCT DEFINITION */ @@ -163,8 +164,8 @@ void SubarrayPartitionerDenseFx::create_default_1d_array( {"a", "b"}, {TILEDB_INT32, TILEDB_INT32}, {1, TILEDB_VAR_NUM}, - {::Compressor(TILEDB_FILTER_LZ4, -1), - ::Compressor(TILEDB_FILTER_LZ4, -1)}, + {tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1), + tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1)}, tile_order, cell_order, 2); @@ -185,15 +186,15 @@ void SubarrayPartitionerDenseFx::create_default_2d_array( {"a", "b"}, {TILEDB_INT32, TILEDB_INT32}, {1, TILEDB_VAR_NUM}, - {::Compressor(TILEDB_FILTER_LZ4, -1), - ::Compressor(TILEDB_FILTER_LZ4, -1)}, + {tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1), + tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1)}, tile_order, cell_order, 2); } void SubarrayPartitionerDenseFx::write_default_1d_array() { - AttrBuffers attr_buffers; + tiledb::test::QueryBuffers buffers; std::vector a = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; uint64_t a_size = a.size() * sizeof(int); std::vector b_off = {0, @@ -210,14 +211,14 @@ void SubarrayPartitionerDenseFx::write_default_1d_array() { std::vector b_val = { 1, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 6, 6, 6, 6, 7, 8, 9, 10}; uint64_t b_val_size = b_val.size() * sizeof(int); - attr_buffers["a"] = AttrBuffer({&a[0], a_size, nullptr, 0}); - attr_buffers["b"] = - AttrBuffer({&b_off[0], b_off_size, &b_val[0], b_val_size}); - write_array(ctx_, array_name_, TILEDB_GLOBAL_ORDER, attr_buffers); + buffers["a"] = tiledb::test::QueryBuffer({&a[0], a_size, nullptr, 0}); + buffers["b"] = + tiledb::test::QueryBuffer({&b_off[0], b_off_size, &b_val[0], b_val_size}); + write_array(ctx_, array_name_, TILEDB_GLOBAL_ORDER, buffers); } void SubarrayPartitionerDenseFx::write_default_2d_array() { - AttrBuffers attr_buffers; + tiledb::test::QueryBuffers buffers; std::vector a = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; uint64_t a_size = a.size() * sizeof(int); std::vector b_off = {0, @@ -241,10 +242,10 @@ void SubarrayPartitionerDenseFx::write_default_2d_array() { 6, 6, 6, 6, 7, 7, 8, 8, 8, 9, 10, 10, 11, 12, 13, 13, 14, 15, 16, 16}; uint64_t b_val_size = b_val.size() * sizeof(int); - attr_buffers["a"] = AttrBuffer({&a[0], a_size, nullptr, 0}); - attr_buffers["b"] = - AttrBuffer({&b_off[0], b_off_size, &b_val[0], b_val_size}); - write_array(ctx_, array_name_, TILEDB_GLOBAL_ORDER, attr_buffers); + buffers["a"] = tiledb::test::QueryBuffer({&a[0], a_size, nullptr, 0}); + buffers["b"] = + tiledb::test::QueryBuffer({&b_off[0], b_off_size, &b_val[0], b_val_size}); + write_array(ctx_, array_name_, TILEDB_GLOBAL_ORDER, buffers); } template diff --git a/test/src/unit-SubarrayPartitioner-error.cc b/test/src/unit-SubarrayPartitioner-error.cc index bf46d7574949..e047954a2758 100644 --- a/test/src/unit-SubarrayPartitioner-error.cc +++ b/test/src/unit-SubarrayPartitioner-error.cc @@ -44,6 +44,7 @@ #include using namespace tiledb::sm; +using namespace tiledb::test; /* ********************************* */ /* STRUCT DEFINITION */ @@ -122,8 +123,8 @@ TEST_CASE_METHOD( {"a", "b"}, {TILEDB_INT32, TILEDB_INT32}, {1, TILEDB_VAR_NUM}, - {::Compressor(TILEDB_FILTER_LZ4, -1), - ::Compressor(TILEDB_FILTER_LZ4, -1)}, + {tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1), + tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1)}, TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR, 2); diff --git a/test/src/unit-SubarrayPartitioner-sparse.cc b/test/src/unit-SubarrayPartitioner-sparse.cc index cd30d3bf35b1..4429340a080f 100644 --- a/test/src/unit-SubarrayPartitioner-sparse.cc +++ b/test/src/unit-SubarrayPartitioner-sparse.cc @@ -44,6 +44,7 @@ #include using namespace tiledb::sm; +using namespace tiledb::test; /* ********************************* */ /* STRUCT DEFINITION */ @@ -176,8 +177,8 @@ void SubarrayPartitionerSparseFx::create_default_1d_array( {"a", "b"}, {TILEDB_INT32, TILEDB_INT32}, {1, TILEDB_VAR_NUM}, - {::Compressor(TILEDB_FILTER_LZ4, -1), - ::Compressor(TILEDB_FILTER_LZ4, -1)}, + {tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1), + tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1)}, tile_order, cell_order, 2); @@ -198,8 +199,8 @@ void SubarrayPartitionerSparseFx::create_default_1d_float_array( {"a", "b"}, {TILEDB_INT32, TILEDB_INT32}, {1, TILEDB_VAR_NUM}, - {::Compressor(TILEDB_FILTER_LZ4, -1), - ::Compressor(TILEDB_FILTER_LZ4, -1)}, + {tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1), + tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1)}, tile_order, cell_order, 2); @@ -220,15 +221,15 @@ void SubarrayPartitionerSparseFx::create_default_2d_array( {"a", "b"}, {TILEDB_INT32, TILEDB_INT32}, {1, TILEDB_VAR_NUM}, - {::Compressor(TILEDB_FILTER_LZ4, -1), - ::Compressor(TILEDB_FILTER_LZ4, -1)}, + {tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1), + tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1)}, tile_order, cell_order, 2); } void SubarrayPartitionerSparseFx::write_default_1d_array() { - AttrBuffers attr_buffers; + tiledb::test::QueryBuffers buffers; std::vector coords = {2, 4, 5, 10, 12, 18}; uint64_t coords_size = coords.size() * sizeof(uint64_t); std::vector a = {1, 2, 3, 4, 5, 6}; @@ -242,16 +243,16 @@ void SubarrayPartitionerSparseFx::write_default_1d_array() { uint64_t b_off_size = b_off.size() * sizeof(uint64_t); std::vector b_val = {1, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 6, 6, 6, 6}; uint64_t b_val_size = b_val.size() * sizeof(int); - attr_buffers[TILEDB_COORDS] = - AttrBuffer({&coords[0], coords_size, nullptr, 0}); - attr_buffers["a"] = AttrBuffer({&a[0], a_size, nullptr, 0}); - attr_buffers["b"] = - AttrBuffer({&b_off[0], b_off_size, &b_val[0], b_val_size}); - write_array(ctx_, array_name_, TILEDB_UNORDERED, attr_buffers); + buffers[TILEDB_COORDS] = + tiledb::test::QueryBuffer({&coords[0], coords_size, nullptr, 0}); + buffers["a"] = tiledb::test::QueryBuffer({&a[0], a_size, nullptr, 0}); + buffers["b"] = + tiledb::test::QueryBuffer({&b_off[0], b_off_size, &b_val[0], b_val_size}); + write_array(ctx_, array_name_, TILEDB_UNORDERED, buffers); } void SubarrayPartitionerSparseFx::write_default_1d_array_2() { - AttrBuffers attr_buffers; + tiledb::test::QueryBuffers buffers; std::vector coords = {2, 4, 5, 10, 12, 18, 25, 27, 33, 40}; uint64_t coords_size = coords.size() * sizeof(uint64_t); std::vector a = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; @@ -270,16 +271,16 @@ void SubarrayPartitionerSparseFx::write_default_1d_array_2() { std::vector b_val = { 1, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 6, 6, 6, 6, 7, 8, 9, 10}; uint64_t b_val_size = b_val.size() * sizeof(int); - attr_buffers[TILEDB_COORDS] = - AttrBuffer({&coords[0], coords_size, nullptr, 0}); - attr_buffers["a"] = AttrBuffer({&a[0], a_size, nullptr, 0}); - attr_buffers["b"] = - AttrBuffer({&b_off[0], b_off_size, &b_val[0], b_val_size}); - write_array(ctx_, array_name_, TILEDB_UNORDERED, attr_buffers); + buffers[TILEDB_COORDS] = + tiledb::test::QueryBuffer({&coords[0], coords_size, nullptr, 0}); + buffers["a"] = tiledb::test::QueryBuffer({&a[0], a_size, nullptr, 0}); + buffers["b"] = + tiledb::test::QueryBuffer({&b_off[0], b_off_size, &b_val[0], b_val_size}); + write_array(ctx_, array_name_, TILEDB_UNORDERED, buffers); } void SubarrayPartitionerSparseFx::write_default_1d_float_array() { - AttrBuffers attr_buffers; + tiledb::test::QueryBuffers buffers; std::vector coords = {2.0f, 4.0f, 5.0f, 10.0f, 12.0f, 18.0f}; uint64_t coords_size = coords.size() * sizeof(float); std::vector a = {1, 2, 3, 4, 5, 6}; @@ -293,16 +294,16 @@ void SubarrayPartitionerSparseFx::write_default_1d_float_array() { uint64_t b_off_size = b_off.size() * sizeof(uint64_t); std::vector b_val = {1, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 6, 6, 6, 6}; uint64_t b_val_size = b_val.size() * sizeof(int); - attr_buffers[TILEDB_COORDS] = - AttrBuffer({&coords[0], coords_size, nullptr, 0}); - attr_buffers["a"] = AttrBuffer({&a[0], a_size, nullptr, 0}); - attr_buffers["b"] = - AttrBuffer({&b_off[0], b_off_size, &b_val[0], b_val_size}); - write_array(ctx_, array_name_, TILEDB_UNORDERED, attr_buffers); + buffers[TILEDB_COORDS] = + tiledb::test::QueryBuffer({&coords[0], coords_size, nullptr, 0}); + buffers["a"] = tiledb::test::QueryBuffer({&a[0], a_size, nullptr, 0}); + buffers["b"] = + tiledb::test::QueryBuffer({&b_off[0], b_off_size, &b_val[0], b_val_size}); + write_array(ctx_, array_name_, TILEDB_UNORDERED, buffers); } void SubarrayPartitionerSparseFx::write_default_2d_array() { - AttrBuffers attr_buffers; + tiledb::test::QueryBuffers buffers; std::vector coords = {1, 2, 2, 5, 3, 3, 3, 9, 4, 1, 4, 7}; uint64_t coords_size = coords.size() * sizeof(uint64_t); std::vector a = {1, 2, 3, 4, 5, 6}; @@ -316,12 +317,12 @@ void SubarrayPartitionerSparseFx::write_default_2d_array() { uint64_t b_off_size = b_off.size() * sizeof(uint64_t); std::vector b_val = {1, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 6, 6, 6, 6}; uint64_t b_val_size = b_val.size() * sizeof(int); - attr_buffers[TILEDB_COORDS] = - AttrBuffer({&coords[0], coords_size, nullptr, 0}); - attr_buffers["a"] = AttrBuffer({&a[0], a_size, nullptr, 0}); - attr_buffers["b"] = - AttrBuffer({&b_off[0], b_off_size, &b_val[0], b_val_size}); - write_array(ctx_, array_name_, TILEDB_UNORDERED, attr_buffers); + buffers[TILEDB_COORDS] = + tiledb::test::QueryBuffer({&coords[0], coords_size, nullptr, 0}); + buffers["a"] = tiledb::test::QueryBuffer({&a[0], a_size, nullptr, 0}); + buffers["b"] = + tiledb::test::QueryBuffer({&b_off[0], b_off_size, &b_val[0], b_val_size}); + write_array(ctx_, array_name_, TILEDB_UNORDERED, buffers); } template diff --git a/test/src/unit-capi-async.cc b/test/src/unit-capi-async.cc index 742d9d955994..85192d85fb99 100644 --- a/test/src/unit-capi-async.cc +++ b/test/src/unit-capi-async.cc @@ -36,6 +36,8 @@ #include +using namespace tiledb::test; + /** Tests for C API async queries. */ struct AsyncFx { // Constants diff --git a/test/src/unit-capi-consolidation.cc b/test/src/unit-capi-consolidation.cc index 6a90c6e5b13a..d522dfed8714 100644 --- a/test/src/unit-capi-consolidation.cc +++ b/test/src/unit-capi-consolidation.cc @@ -39,6 +39,8 @@ #include #include +using namespace tiledb::test; + /** Tests for C API consolidation. */ struct ConsolidationFx { // Constants diff --git a/test/src/unit-capi-dense_array.cc b/test/src/unit-capi-dense_array.cc index 223877169556..ec7b781ebd79 100644 --- a/test/src/unit-capi-dense_array.cc +++ b/test/src/unit-capi-dense_array.cc @@ -53,6 +53,8 @@ #include #include +using namespace tiledb::test; + struct DenseArrayFx { // Constant parameters const char* ATTR_NAME = "a"; diff --git a/test/src/unit-capi-dense_array_2.cc b/test/src/unit-capi-dense_array_2.cc index 55c504603aae..d3b712116859 100644 --- a/test/src/unit-capi-dense_array_2.cc +++ b/test/src/unit-capi-dense_array_2.cc @@ -44,6 +44,7 @@ #include using namespace tiledb::sm; +using namespace tiledb::test; /* ********************************* */ /* STRUCT DEFINITION */ @@ -119,9 +120,9 @@ void CDenseArrayFx::create_default_array_1d() { {"a", "b", "c"}, {TILEDB_INT32, TILEDB_CHAR, TILEDB_FLOAT32}, {1, TILEDB_VAR_NUM, 2}, - {::Compressor(TILEDB_FILTER_NONE, -1), - ::Compressor(TILEDB_FILTER_ZSTD, -1), - ::Compressor(TILEDB_FILTER_LZ4, -1)}, + {tiledb::test::Compressor(TILEDB_FILTER_NONE, -1), + tiledb::test::Compressor(TILEDB_FILTER_ZSTD, -1), + tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1)}, TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR, 2); @@ -141,16 +142,16 @@ void CDenseArrayFx::create_default_array_2d() { {"a", "b", "c"}, {TILEDB_INT32, TILEDB_CHAR, TILEDB_FLOAT32}, {1, TILEDB_VAR_NUM, 2}, - {::Compressor(TILEDB_FILTER_NONE, -1), - ::Compressor(TILEDB_FILTER_ZSTD, -1), - ::Compressor(TILEDB_FILTER_LZ4, -1)}, + {tiledb::test::Compressor(TILEDB_FILTER_NONE, -1), + tiledb::test::Compressor(TILEDB_FILTER_ZSTD, -1), + tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1)}, TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR, 2); } void CDenseArrayFx::write_default_array_1d() { - AttrBuffers attr_buffers; + tiledb::test::QueryBuffers buffers; std::vector a = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; uint64_t a_size = a.size() * sizeof(int); std::vector b_off = {0, @@ -189,11 +190,11 @@ void CDenseArrayFx::write_default_array_1d() { 4.2f, 5.1f, 5.2f, 6.1f, 6.2f, 7.1f, 7.2f, 8.1f, 8.2f, 9.1f, 9.2f, 10.1f, 10.2f}; uint64_t c_size = c.size() * sizeof(float); - attr_buffers["a"] = AttrBuffer({&a[0], a_size, nullptr, 0}); - attr_buffers["b"] = - AttrBuffer({&b_off[0], b_off_size, &b_val[0], b_val_size}); - attr_buffers["c"] = AttrBuffer({&c[0], c_size, nullptr, 0}); - write_array(ctx_, array_name_, TILEDB_GLOBAL_ORDER, attr_buffers); + buffers["a"] = tiledb::test::QueryBuffer({&a[0], a_size, nullptr, 0}); + buffers["b"] = + tiledb::test::QueryBuffer({&b_off[0], b_off_size, &b_val[0], b_val_size}); + buffers["c"] = tiledb::test::QueryBuffer({&c[0], c_size, nullptr, 0}); + write_array(ctx_, array_name_, TILEDB_GLOBAL_ORDER, buffers); } void CDenseArrayFx::write_default_array_2d() { @@ -226,16 +227,16 @@ void CDenseArrayFx::write_default_array_2d() { 11.2f, 12.1f, 12.2f, 13.1f, 13.2f, 14.1f, 14.2f, 15.1f, 15.2f, 16.1f, 16.2f}; uint64_t c_size = c.size() * sizeof(float); - AttrBuffers attr_buffers; - attr_buffers["a"] = AttrBuffer({&a[0], a_size, nullptr, 0}); - attr_buffers["b"] = - AttrBuffer({&b_off[0], b_off_size, &b_val[0], b_val_size}); - attr_buffers["c"] = AttrBuffer({&c[0], c_size, nullptr, 0}); - write_array(ctx_, array_name_, TILEDB_ROW_MAJOR, attr_buffers); + tiledb::test::QueryBuffers buffers; + buffers["a"] = tiledb::test::QueryBuffer({&a[0], a_size, nullptr, 0}); + buffers["b"] = + tiledb::test::QueryBuffer({&b_off[0], b_off_size, &b_val[0], b_val_size}); + buffers["c"] = tiledb::test::QueryBuffer({&c[0], c_size, nullptr, 0}); + write_array(ctx_, array_name_, TILEDB_ROW_MAJOR, buffers); } void CDenseArrayFx::write_sparse_fragment_1d() { - AttrBuffers attr_buffers; + tiledb::test::QueryBuffers buffers; std::vector a = {102, 103, 106}; uint64_t a_size = a.size() * sizeof(int); std::vector b_off = {0, sizeof(char), 3 * sizeof(char)}; @@ -247,17 +248,17 @@ void CDenseArrayFx::write_sparse_fragment_1d() { std::vector coords = {2, 3, 6}; uint64_t coords_size = coords.size() * sizeof(uint64_t); - attr_buffers["a"] = AttrBuffer({&a[0], a_size, nullptr, 0}); - attr_buffers["b"] = - AttrBuffer({&b_off[0], b_off_size, &b_val[0], b_val_size}); - attr_buffers["c"] = AttrBuffer({&c[0], c_size, nullptr, 0}); - attr_buffers[TILEDB_COORDS] = - AttrBuffer({&coords[0], coords_size, nullptr, 0}); - write_array(ctx_, array_name_, TILEDB_GLOBAL_ORDER, attr_buffers); + buffers["a"] = tiledb::test::QueryBuffer({&a[0], a_size, nullptr, 0}); + buffers["b"] = + tiledb::test::QueryBuffer({&b_off[0], b_off_size, &b_val[0], b_val_size}); + buffers["c"] = tiledb::test::QueryBuffer({&c[0], c_size, nullptr, 0}); + buffers[TILEDB_COORDS] = + tiledb::test::QueryBuffer({&coords[0], coords_size, nullptr, 0}); + write_array(ctx_, array_name_, TILEDB_GLOBAL_ORDER, buffers); } void CDenseArrayFx::write_sparse_fragment_2d() { - AttrBuffers attr_buffers; + tiledb::test::QueryBuffers buffers; std::vector a = {102, 103, 105, 109}; uint64_t a_size = a.size() * sizeof(int); std::vector b_off = { @@ -271,13 +272,13 @@ void CDenseArrayFx::write_sparse_fragment_2d() { std::vector coords = {1, 2, 1, 3, 2, 1, 3, 1}; uint64_t coords_size = coords.size() * sizeof(uint64_t); - attr_buffers["a"] = AttrBuffer({&a[0], a_size, nullptr, 0}); - attr_buffers["b"] = - AttrBuffer({&b_off[0], b_off_size, &b_val[0], b_val_size}); - attr_buffers["c"] = AttrBuffer({&c[0], c_size, nullptr, 0}); - attr_buffers[TILEDB_COORDS] = - AttrBuffer({&coords[0], coords_size, nullptr, 0}); - write_array(ctx_, array_name_, TILEDB_UNORDERED, attr_buffers); + buffers["a"] = tiledb::test::QueryBuffer({&a[0], a_size, nullptr, 0}); + buffers["b"] = + tiledb::test::QueryBuffer({&b_off[0], b_off_size, &b_val[0], b_val_size}); + buffers["c"] = tiledb::test::QueryBuffer({&c[0], c_size, nullptr, 0}); + buffers[TILEDB_COORDS] = + tiledb::test::QueryBuffer({&coords[0], coords_size, nullptr, 0}); + write_array(ctx_, array_name_, TILEDB_UNORDERED, buffers); } /* ********************************* */ @@ -301,22 +302,22 @@ TEST_CASE_METHOD( uint64_t b_val_size = b_val.size() * sizeof(char); std::vector c(20); uint64_t c_size = c.size() * sizeof(float); - AttrBuffers attr_buffers; - attr_buffers["a"] = AttrBuffer({&a[0], a_size, nullptr, 0}); - attr_buffers["b"] = - AttrBuffer({&b_off[0], b_off_size, &b_val[0], b_val_size}); - attr_buffers["c"] = AttrBuffer({&c[0], c_size, nullptr, 0}); + tiledb::test::QueryBuffers buffers; + buffers["a"] = tiledb::test::QueryBuffer({&a[0], a_size, nullptr, 0}); + buffers["b"] = + tiledb::test::QueryBuffer({&b_off[0], b_off_size, &b_val[0], b_val_size}); + buffers["c"] = tiledb::test::QueryBuffer({&c[0], c_size, nullptr, 0}); std::vector coords(10); uint64_t coords_size = coords.size() * sizeof(uint64_t); - attr_buffers[TILEDB_COORDS] = - AttrBuffer({&coords[0], coords_size, nullptr, 0}); + buffers[TILEDB_COORDS] = + tiledb::test::QueryBuffer({&coords[0], coords_size, nullptr, 0}); // Open array open_array(ctx_, array_, TILEDB_READ); // Create subarray SubarrayRanges ranges = {{1, 10}}; - read_array(ctx_, array_, ranges, TILEDB_ROW_MAJOR, attr_buffers); + read_array(ctx_, array_, ranges, TILEDB_ROW_MAJOR, buffers); // Check results std::vector c_a = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; @@ -382,22 +383,22 @@ TEST_CASE_METHOD( uint64_t b_val_size = b_val.size() * sizeof(char); std::vector c(20); uint64_t c_size = c.size() * sizeof(float); - AttrBuffers attr_buffers; - attr_buffers["a"] = AttrBuffer({&a[0], a_size, nullptr, 0}); - attr_buffers["b"] = - AttrBuffer({&b_off[0], b_off_size, &b_val[0], b_val_size}); - attr_buffers["c"] = AttrBuffer({&c[0], c_size, nullptr, 0}); + tiledb::test::QueryBuffers buffers; + buffers["a"] = tiledb::test::QueryBuffer({&a[0], a_size, nullptr, 0}); + buffers["b"] = + tiledb::test::QueryBuffer({&b_off[0], b_off_size, &b_val[0], b_val_size}); + buffers["c"] = tiledb::test::QueryBuffer({&c[0], c_size, nullptr, 0}); std::vector coords(10); uint64_t coords_size = coords.size() * sizeof(uint64_t); - attr_buffers[TILEDB_COORDS] = - AttrBuffer({&coords[0], coords_size, nullptr, 0}); + buffers[TILEDB_COORDS] = + tiledb::test::QueryBuffer({&coords[0], coords_size, nullptr, 0}); // Open array open_array(ctx_, array_, TILEDB_READ); // Read SubarrayRanges ranges = {{1, 10}}; - read_array(ctx_, array_, ranges, TILEDB_ROW_MAJOR, attr_buffers); + read_array(ctx_, array_, ranges, TILEDB_ROW_MAJOR, buffers); // Check results std::vector c_a = {1, 102, 103, 4, 5, 106, 7, 8, 9, 10}; @@ -462,22 +463,22 @@ TEST_CASE_METHOD( uint64_t b_val_size = b_val.size() * sizeof(char); std::vector c(20); uint64_t c_size = c.size() * sizeof(float); - AttrBuffers attr_buffers; - attr_buffers["a"] = AttrBuffer({&a[0], a_size, nullptr, 0}); - attr_buffers["b"] = - AttrBuffer({&b_off[0], b_off_size, &b_val[0], b_val_size}); - attr_buffers["c"] = AttrBuffer({&c[0], c_size, nullptr, 0}); + tiledb::test::QueryBuffers buffers; + buffers["a"] = tiledb::test::QueryBuffer({&a[0], a_size, nullptr, 0}); + buffers["b"] = + tiledb::test::QueryBuffer({&b_off[0], b_off_size, &b_val[0], b_val_size}); + buffers["c"] = tiledb::test::QueryBuffer({&c[0], c_size, nullptr, 0}); std::vector coords(10); uint64_t coords_size = coords.size() * sizeof(uint64_t); - attr_buffers[TILEDB_COORDS] = - AttrBuffer({&coords[0], coords_size, nullptr, 0}); + buffers[TILEDB_COORDS] = + tiledb::test::QueryBuffer({&coords[0], coords_size, nullptr, 0}); // Open array open_array(ctx_, array_, TILEDB_READ); // Read SubarrayRanges ranges = {{1, 10}}; - read_array(ctx_, array_, ranges, TILEDB_ROW_MAJOR, attr_buffers); + read_array(ctx_, array_, ranges, TILEDB_ROW_MAJOR, buffers); // Check results std::vector c_a = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; @@ -543,15 +544,15 @@ TEST_CASE_METHOD( uint64_t b_val_size = b_val.size() * sizeof(char); std::vector c(32); uint64_t c_size = c.size() * sizeof(float); - AttrBuffers attr_buffers; - attr_buffers["a"] = AttrBuffer({&a[0], a_size, nullptr, 0}); - attr_buffers["b"] = - AttrBuffer({&b_off[0], b_off_size, &b_val[0], b_val_size}); - attr_buffers["c"] = AttrBuffer({&c[0], c_size, nullptr, 0}); + tiledb::test::QueryBuffers buffers; + buffers["a"] = tiledb::test::QueryBuffer({&a[0], a_size, nullptr, 0}); + buffers["b"] = + tiledb::test::QueryBuffer({&b_off[0], b_off_size, &b_val[0], b_val_size}); + buffers["c"] = tiledb::test::QueryBuffer({&c[0], c_size, nullptr, 0}); std::vector coords(32); uint64_t coords_size = coords.size() * sizeof(uint64_t); - attr_buffers[TILEDB_COORDS] = - AttrBuffer({&coords[0], coords_size, nullptr, 0}); + buffers[TILEDB_COORDS] = + tiledb::test::QueryBuffer({&coords[0], coords_size, nullptr, 0}); // Open array open_array(ctx_, array_, TILEDB_READ); @@ -655,7 +656,7 @@ TEST_CASE_METHOD( // Read SubarrayRanges ranges = {{1, 4}, {1, 4}}; - read_array(ctx_, array_, ranges, subarray_layout, attr_buffers); + read_array(ctx_, array_, ranges, subarray_layout, buffers); // Check results CHECK(a == c_a); @@ -731,15 +732,15 @@ TEST_CASE_METHOD( uint64_t b_val_size = b_val.size() * sizeof(char); std::vector c(32); uint64_t c_size = c.size() * sizeof(float); - AttrBuffers attr_buffers; - attr_buffers["a"] = AttrBuffer({&a[0], a_size, nullptr, 0}); - attr_buffers["b"] = - AttrBuffer({&b_off[0], b_off_size, &b_val[0], b_val_size}); - attr_buffers["c"] = AttrBuffer({&c[0], c_size, nullptr, 0}); + tiledb::test::QueryBuffers buffers; + buffers["a"] = tiledb::test::QueryBuffer({&a[0], a_size, nullptr, 0}); + buffers["b"] = + tiledb::test::QueryBuffer({&b_off[0], b_off_size, &b_val[0], b_val_size}); + buffers["c"] = tiledb::test::QueryBuffer({&c[0], c_size, nullptr, 0}); std::vector coords(32); uint64_t coords_size = coords.size() * sizeof(uint64_t); - attr_buffers[TILEDB_COORDS] = - AttrBuffer({&coords[0], coords_size, nullptr, 0}); + buffers[TILEDB_COORDS] = + tiledb::test::QueryBuffer({&coords[0], coords_size, nullptr, 0}); // Open array open_array(ctx_, array_, TILEDB_READ); @@ -843,7 +844,7 @@ TEST_CASE_METHOD( // Read from the array SubarrayRanges ranges = {{1, 4}, {1, 4}}; - read_array(ctx_, array_, ranges, subarray_layout, attr_buffers); + read_array(ctx_, array_, ranges, subarray_layout, buffers); // Check results CHECK(a == c_a); @@ -875,15 +876,15 @@ TEST_CASE_METHOD( uint64_t b_val_size = b_val.size() * sizeof(char); std::vector c(32); uint64_t c_size = c.size() * sizeof(float); - AttrBuffers attr_buffers; - attr_buffers["a"] = AttrBuffer({&a[0], a_size, nullptr, 0}); - attr_buffers["b"] = - AttrBuffer({&b_off[0], b_off_size, &b_val[0], b_val_size}); - attr_buffers["c"] = AttrBuffer({&c[0], c_size, nullptr, 0}); + tiledb::test::QueryBuffers buffers; + buffers["a"] = tiledb::test::QueryBuffer({&a[0], a_size, nullptr, 0}); + buffers["b"] = + tiledb::test::QueryBuffer({&b_off[0], b_off_size, &b_val[0], b_val_size}); + buffers["c"] = tiledb::test::QueryBuffer({&c[0], c_size, nullptr, 0}); std::vector coords(32); uint64_t coords_size = coords.size() * sizeof(uint64_t); - attr_buffers[TILEDB_COORDS] = - AttrBuffer({&coords[0], coords_size, nullptr, 0}); + buffers[TILEDB_COORDS] = + tiledb::test::QueryBuffer({&coords[0], coords_size, nullptr, 0}); // Open array open_array(ctx_, array_, TILEDB_READ); @@ -957,7 +958,7 @@ TEST_CASE_METHOD( // Read from the array SubarrayRanges ranges = {{1, 1, 2, 2, 3, 3, 4, 4}, {1, 4}}; - read_array(ctx_, array_, ranges, subarray_layout, attr_buffers); + read_array(ctx_, array_, ranges, subarray_layout, buffers); // Check results CHECK(a == c_a); diff --git a/test/src/unit-capi-incomplete-2.cc b/test/src/unit-capi-incomplete-2.cc index d642d1090d86..e0d372ed04e2 100644 --- a/test/src/unit-capi-incomplete-2.cc +++ b/test/src/unit-capi-incomplete-2.cc @@ -37,6 +37,8 @@ #include #include +using namespace tiledb::test; + /** * Tests cases where a read query is incomplete or leads to a buffer * overflow. diff --git a/test/src/unit-capi-incomplete.cc b/test/src/unit-capi-incomplete.cc index a9d3ef397be9..f6d0f493e309 100644 --- a/test/src/unit-capi-incomplete.cc +++ b/test/src/unit-capi-incomplete.cc @@ -39,6 +39,8 @@ #include #include +using namespace tiledb::test; + /** * Tests cases where a read query is incomplete or leads to a buffer * overflow. diff --git a/test/src/unit-capi-metadata.cc b/test/src/unit-capi-metadata.cc index e5e33486023e..a82a189eecf0 100644 --- a/test/src/unit-capi-metadata.cc +++ b/test/src/unit-capi-metadata.cc @@ -46,6 +46,7 @@ #include using namespace tiledb::sm; +using namespace tiledb::test; /* ********************************* */ /* STRUCT DEFINITION */ @@ -121,9 +122,9 @@ void CMetadataFx::create_default_array_1d() { {"a", "b", "c"}, {TILEDB_INT32, TILEDB_CHAR, TILEDB_FLOAT32}, {1, TILEDB_VAR_NUM, 2}, - {::Compressor(TILEDB_FILTER_NONE, -1), - ::Compressor(TILEDB_FILTER_ZSTD, -1), - ::Compressor(TILEDB_FILTER_LZ4, -1)}, + {tiledb::test::Compressor(TILEDB_FILTER_NONE, -1), + tiledb::test::Compressor(TILEDB_FILTER_ZSTD, -1), + tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1)}, TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR, 2); @@ -146,9 +147,9 @@ void CMetadataFx::create_default_array_1d_with_key() { {"a", "b", "c"}, {TILEDB_INT32, TILEDB_CHAR, TILEDB_FLOAT32}, {1, TILEDB_VAR_NUM, 2}, - {::Compressor(TILEDB_FILTER_NONE, -1), - ::Compressor(TILEDB_FILTER_ZSTD, -1), - ::Compressor(TILEDB_FILTER_LZ4, -1)}, + {tiledb::test::Compressor(TILEDB_FILTER_NONE, -1), + tiledb::test::Compressor(TILEDB_FILTER_ZSTD, -1), + tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1)}, TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR, 2); diff --git a/test/src/unit-capi-query_2.cc b/test/src/unit-capi-query_2.cc index 35b54466d727..15a54482b21d 100644 --- a/test/src/unit-capi-query_2.cc +++ b/test/src/unit-capi-query_2.cc @@ -38,6 +38,8 @@ #include #include +using namespace tiledb::test; + const uint64_t DIM_DOMAIN[4] = {1, 10, 1, 10}; /** Tests for C API subarray. */ diff --git a/test/src/unit-capi-serialized_queries.cc b/test/src/unit-capi-serialized_queries.cc index 4b00c29b9d7c..9033f4b88c9d 100644 --- a/test/src/unit-capi-serialized_queries.cc +++ b/test/src/unit-capi-serialized_queries.cc @@ -154,6 +154,49 @@ struct SerializationFx { deserialize_query(ctx, serialized, &query, true); } + void write_sparse_array_split_coords() { + std::vector d1 = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + + std::vector d2 = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + + std::vector a1; + std::vector a2; + std::vector a3_data; + std::vector a3_offsets; + + const unsigned ncells = 10; + for (unsigned i = 0; i < ncells; i++) { + a1.push_back(i); + a2.push_back(i); + a2.push_back(2 * i); + + std::string a3 = "a"; + for (unsigned j = 0; j < i; j++) + a3.push_back('a'); + a3_offsets.push_back(a3_data.size()); + a3_data.insert(a3_data.end(), a3.begin(), a3.end()); + } + + Array array(ctx, array_uri, TILEDB_WRITE); + Query query(ctx, array); + query.set_layout(TILEDB_UNORDERED); + query.set_buffer("d1", d1); + query.set_buffer("d2", d2); + query.set_buffer("a1", a1); + query.set_buffer("a2", a2); + query.set_buffer("a3", a3_offsets, a3_data); + + // Serialize into a copy and submit. + std::vector serialized; + serialize_query(ctx, query, &serialized, true); + Array array2(ctx, array_uri, TILEDB_WRITE); + Query query2(ctx, array2); + deserialize_query(ctx, serialized, &query2, false); + query2.submit(); + serialize_query(ctx, query2, &serialized, false); + deserialize_query(ctx, serialized, &query, true); + } + /** * Helper function that serializes a query from the "client" or "server" * perspective. The flow being mimicked here is (for read queries): @@ -523,3 +566,52 @@ TEST_CASE_METHOD( std::free(b); } } + +TEST_CASE_METHOD( + SerializationFx, + "Query serialization, split coords, sparse", + "[query][sparse][serialization][split-coords]") { + create_array(TILEDB_SPARSE); + write_sparse_array_split_coords(); + + SECTION("- Read all") { + Array array(ctx, array_uri, TILEDB_READ); + Query query(ctx, array); + std::vector coords(1000); + std::vector a1(1000); + std::vector a2(1000); + std::vector a3_data(1000 * 100); + std::vector a3_offsets(1000); + std::vector subarray = {1, 10, 1, 10}; + + query.set_subarray(subarray); + query.set_coordinates(coords); + query.set_buffer("a1", a1); + query.set_buffer("a2", a2); + query.set_buffer("a3", a3_offsets, a3_data); + + // Serialize into a copy and submit. + std::vector serialized; + serialize_query(ctx, query, &serialized, true); + Array array2(ctx, array_uri, TILEDB_READ); + Query query2(ctx, array2); + deserialize_query(ctx, serialized, &query2, false); + auto to_free = allocate_query_buffers(ctx, array2, &query2); + query2.submit(); + serialize_query(ctx, query2, &serialized, false); + + // Deserialize into original query + deserialize_query(ctx, serialized, &query, true); + REQUIRE(query.query_status() == Query::Status::COMPLETE); + + auto result_el = query.result_buffer_elements(); + REQUIRE(result_el[TILEDB_COORDS].second == 20); + REQUIRE(result_el["a1"].second == 10); + REQUIRE(result_el["a2"].second == 20); + REQUIRE(result_el["a3"].first == 10); + REQUIRE(result_el["a3"].second == 55); + + for (void* b : to_free) + std::free(b); + } +} diff --git a/test/src/unit-capi-sparse_array.cc b/test/src/unit-capi-sparse_array.cc index c2dc6428935f..4fc3614e4177 100644 --- a/test/src/unit-capi-sparse_array.cc +++ b/test/src/unit-capi-sparse_array.cc @@ -50,6 +50,8 @@ #include #include +using namespace tiledb::test; + const uint64_t DIM_DOMAIN[4] = {1, 4, 1, 4}; struct SparseArrayFx { @@ -5737,6 +5739,7 @@ TEST_CASE_METHOD( remove_array(array_name); } + TEST_CASE_METHOD( SparseArrayFx, "C API: Test sparse array, global order with 0-sized buffers", @@ -5793,3 +5796,484 @@ TEST_CASE_METHOD( tiledb_array_free(&array); tiledb_ctx_free(&ctx); } + +TEST_CASE_METHOD( + SparseArrayFx, + "C API: Test sparse array, split coordinate buffers", + "[capi][sparse][split-coords]") { + std::string array_name = + FILE_URI_PREFIX + FILE_TEMP_DIR + "sparse_split_coords"; + create_sparse_array(array_name); + + // ---- WRITE ---- + + // Prepare cell buffers + int buffer_a1[] = {0, 1, 2, 3, 4, 5, 6, 7}; + uint64_t buffer_a2[] = {0, 1, 3, 6, 10, 11, 13, 16}; + char buffer_var_a2[] = "abbcccddddeffggghhhh"; + float buffer_a3[] = {0.1f, + 0.2f, + 1.1f, + 1.2f, + 2.1f, + 2.2f, + 3.1f, + 3.2f, + 4.1f, + 4.2f, + 5.1f, + 5.2f, + 6.1f, + 6.2f, + 7.1f, + 7.2f}; + uint64_t buffer_a1_size = sizeof(buffer_a1); + uint64_t buffer_a2_size = sizeof(buffer_a2); + // No need to store the last '\0' character + uint64_t buffer_var_a2_size = sizeof(buffer_var_a2) - 1; + uint64_t buffer_a3_size = sizeof(buffer_a3); + uint64_t buffer_d1[] = {1, 1, 1, 2, 3, 3, 3, 4}; + uint64_t buffer_d1_size = sizeof(buffer_d1); + uint64_t buffer_d2[] = {1, 2, 4, 3, 1, 3, 4, 2}; + uint64_t buffer_d2_size = sizeof(buffer_d2); + + // Open array + tiledb_array_t* array; + int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); + CHECK(rc == TILEDB_OK); + rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); + CHECK(rc == TILEDB_OK); + + // Create query + tiledb_query_t* query; + rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_layout(ctx_, query, TILEDB_UNORDERED); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_buffer(ctx_, query, "a1", buffer_a1, &buffer_a1_size); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_buffer_var( + ctx_, + query, + "a2", + (uint64_t*)buffer_a2, + &buffer_a2_size, + buffer_var_a2, + &buffer_var_a2_size); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_buffer(ctx_, query, "a3", buffer_a3, &buffer_a3_size); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_buffer(ctx_, query, "d1", buffer_d1, &buffer_d1_size); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_buffer(ctx_, query, "d2", buffer_d2, &buffer_d2_size); + CHECK(rc == TILEDB_OK); + + // Submit query + rc = tiledb_query_submit(ctx_, query); + CHECK(rc == TILEDB_OK); + + // Finalize query + rc = tiledb_query_finalize(ctx_, query); + CHECK(rc == TILEDB_OK); + + // Close array + rc = tiledb_array_close(ctx_, array); + CHECK(rc == TILEDB_OK); + + // Clean up + tiledb_array_free(&array); + tiledb_query_free(&query); + + // ---- READ ---- + + // Create array + rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); + CHECK(rc == TILEDB_OK); + + // Open the array + rc = tiledb_array_open(ctx_, array, TILEDB_READ); + CHECK(rc == TILEDB_OK); + + // Create buffers + int b_a1[30]; + uint64_t b_a1_size = sizeof(b_a1); + uint64_t b_a2_off[30]; + uint64_t b_a2_off_size = sizeof(b_a2_off); + char b_a2_val[30]; + uint64_t b_a2_val_size = sizeof(b_a2_val); + float b_a3[30]; + uint64_t b_a3_size = sizeof(b_a3); + uint64_t b_coords[30]; + uint64_t b_coords_size = sizeof(b_coords); + + // Create query + rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_buffer(ctx_, query, "a1", b_a1, &b_a1_size); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_buffer_var( + ctx_, query, "a2", b_a2_off, &b_a2_off_size, b_a2_val, &b_a2_val_size); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_buffer(ctx_, query, "a3", b_a3, &b_a3_size); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_buffer( + ctx_, query, TILEDB_COORDS, b_coords, &b_coords_size); + REQUIRE(rc == TILEDB_OK); + + // Set a subarray + uint64_t subarray[] = {1, 4, 1, 4}; + rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray(ctx_, query, subarray); + REQUIRE(rc == TILEDB_OK); + + // Submit query + rc = tiledb_query_submit(ctx_, query); + REQUIRE(rc == TILEDB_OK); + + tiledb_query_status_t status; + rc = tiledb_query_get_status(ctx_, query, &status); + REQUIRE(rc == TILEDB_OK); + REQUIRE(status == TILEDB_COMPLETED); + + // Check buffer sizes + CHECK(b_a1_size == buffer_a1_size); + CHECK(b_a2_off_size == buffer_a2_size); + CHECK(b_a2_val_size == buffer_var_a2_size); + CHECK(b_a3_size == buffer_a3_size); + CHECK(b_coords_size == buffer_d1_size + buffer_d2_size); + + // Check buffer data + int c_b_a1[] = {0, 1, 2, 3, 4, 5, 6, 7}; + uint64_t c_b_a2_off[] = {0, 1, 3, 6, 10, 11, 13, 16}; + char c_b_a2_val[] = "abbcccddddeffggghhhh"; + float c_b_a3[] = {0.1f, + 0.2f, + 1.1f, + 1.2f, + 2.1f, + 2.2f, + 3.1f, + 3.2f, + 4.1f, + 4.2f, + 5.1f, + 5.2f, + 6.1f, + 6.2f, + 7.1f, + 7.2f}; + uint64_t c_b_coords[] = {1, 1, 1, 2, 1, 4, 2, 3, 3, 1, 3, 3, 3, 4, 4, 2}; + CHECK(!memcmp(c_b_a1, b_a1, b_a1_size)); + CHECK(!memcmp(c_b_a2_off, b_a2_off, b_a2_off_size)); + CHECK(!memcmp(c_b_a2_val, b_a2_val, b_a2_val_size)); + CHECK(!memcmp(c_b_a3, b_a3, b_a3_size)); + CHECK(!memcmp(c_b_coords, b_coords, b_coords_size)); + + // Close array + rc = tiledb_array_close(ctx_, array); + CHECK(rc == TILEDB_OK); + + // Clean up + tiledb_array_free(&array); + tiledb_query_free(&query); + + remove_array(array_name); +} + +TEST_CASE_METHOD( + SparseArrayFx, + "C API: Test sparse array, split coordinate buffers, global write", + "[capi][sparse][split-coords][global]") { + std::string array_name = + FILE_URI_PREFIX + FILE_TEMP_DIR + "sparse_split_coords_global"; + create_sparse_array(array_name); + + // ---- WRITE ---- + + // Prepare cell buffers + int buffer_a1[] = {0, 1, 2, 3, 4, 5, 6, 7}; + uint64_t buffer_a2[] = {0, 1, 3, 6, 10, 11, 13, 16}; + char buffer_var_a2[] = "abbcccddddeffggghhhh"; + float buffer_a3[] = {0.1f, + 0.2f, + 1.1f, + 1.2f, + 2.1f, + 2.2f, + 3.1f, + 3.2f, + 4.1f, + 4.2f, + 5.1f, + 5.2f, + 6.1f, + 6.2f, + 7.1f, + 7.2f}; + uint64_t buffer_a1_size = sizeof(buffer_a1); + uint64_t buffer_a2_size = sizeof(buffer_a2); + // No need to store the last '\0' character + uint64_t buffer_var_a2_size = sizeof(buffer_var_a2) - 1; + uint64_t buffer_a3_size = sizeof(buffer_a3); + uint64_t buffer_d1[] = {1, 1, 1, 2, 3, 4, 3, 3}; + uint64_t buffer_d1_size = sizeof(buffer_d1); + uint64_t buffer_d2[] = {1, 2, 4, 3, 1, 2, 3, 4}; + uint64_t buffer_d2_size = sizeof(buffer_d2); + + // Open array + tiledb_array_t* array; + int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); + CHECK(rc == TILEDB_OK); + rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); + CHECK(rc == TILEDB_OK); + + // Create query + tiledb_query_t* query; + rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_buffer(ctx_, query, "a1", buffer_a1, &buffer_a1_size); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_buffer_var( + ctx_, + query, + "a2", + (uint64_t*)buffer_a2, + &buffer_a2_size, + buffer_var_a2, + &buffer_var_a2_size); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_buffer(ctx_, query, "a3", buffer_a3, &buffer_a3_size); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_buffer(ctx_, query, "d1", buffer_d1, &buffer_d1_size); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_buffer(ctx_, query, "d2", buffer_d2, &buffer_d2_size); + CHECK(rc == TILEDB_OK); + + // Submit query + rc = tiledb_query_submit(ctx_, query); + CHECK(rc == TILEDB_OK); + + // Finalize query + rc = tiledb_query_finalize(ctx_, query); + CHECK(rc == TILEDB_OK); + + // Close array + rc = tiledb_array_close(ctx_, array); + CHECK(rc == TILEDB_OK); + + // Clean up + tiledb_array_free(&array); + tiledb_query_free(&query); + + // ---- READ ---- + + // Create array + rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); + CHECK(rc == TILEDB_OK); + + // Open the array + rc = tiledb_array_open(ctx_, array, TILEDB_READ); + CHECK(rc == TILEDB_OK); + + // Create buffers + int b_a1[30]; + uint64_t b_a1_size = sizeof(b_a1); + uint64_t b_a2_off[30]; + uint64_t b_a2_off_size = sizeof(b_a2_off); + char b_a2_val[30]; + uint64_t b_a2_val_size = sizeof(b_a2_val); + float b_a3[30]; + uint64_t b_a3_size = sizeof(b_a3); + uint64_t b_coords[30]; + uint64_t b_coords_size = sizeof(b_coords); + + // Create query + rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_buffer(ctx_, query, "a1", b_a1, &b_a1_size); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_buffer_var( + ctx_, query, "a2", b_a2_off, &b_a2_off_size, b_a2_val, &b_a2_val_size); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_buffer(ctx_, query, "a3", b_a3, &b_a3_size); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_buffer( + ctx_, query, TILEDB_COORDS, b_coords, &b_coords_size); + REQUIRE(rc == TILEDB_OK); + + // Set a subarray + uint64_t subarray[] = {1, 4, 1, 4}; + rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray(ctx_, query, subarray); + REQUIRE(rc == TILEDB_OK); + + // Submit query + rc = tiledb_query_submit(ctx_, query); + REQUIRE(rc == TILEDB_OK); + + tiledb_query_status_t status; + rc = tiledb_query_get_status(ctx_, query, &status); + REQUIRE(rc == TILEDB_OK); + REQUIRE(status == TILEDB_COMPLETED); + + // Check buffer sizes + CHECK(b_a1_size == buffer_a1_size); + CHECK(b_a2_off_size == buffer_a2_size); + CHECK(b_a2_val_size == buffer_var_a2_size); + CHECK(b_a3_size == buffer_a3_size); + CHECK(b_coords_size == buffer_d1_size + buffer_d2_size); + + // Check buffer data + int c_b_a1[] = {0, 1, 2, 3, 4, 5, 6, 7}; + uint64_t c_b_a2_off[] = {0, 1, 3, 6, 10, 11, 13, 16}; + char c_b_a2_val[] = "abbcccddddeffggghhhh"; + float c_b_a3[] = {0.1f, + 0.2f, + 1.1f, + 1.2f, + 2.1f, + 2.2f, + 3.1f, + 3.2f, + 4.1f, + 4.2f, + 5.1f, + 5.2f, + 6.1f, + 6.2f, + 7.1f, + 7.2f}; + uint64_t c_b_coords[] = {1, 1, 1, 2, 1, 4, 2, 3, 3, 1, 4, 2, 3, 3, 3, 4}; + CHECK(!memcmp(c_b_a1, b_a1, b_a1_size)); + CHECK(!memcmp(c_b_a2_off, b_a2_off, b_a2_off_size)); + CHECK(!memcmp(c_b_a2_val, b_a2_val, b_a2_val_size)); + CHECK(!memcmp(c_b_a3, b_a3, b_a3_size)); + CHECK(!memcmp(c_b_coords, b_coords, b_coords_size)); + + // Close array + rc = tiledb_array_close(ctx_, array); + CHECK(rc == TILEDB_OK); + + // Clean up + tiledb_array_free(&array); + tiledb_query_free(&query); + + remove_array(array_name); +} + +TEST_CASE_METHOD( + SparseArrayFx, + "C API: Test sparse array, split coordinate buffers, errors", + "[capi][sparse][split-coords][errors]") { + std::string array_name = + FILE_URI_PREFIX + FILE_TEMP_DIR + "sparse_split_coords_errors"; + create_sparse_array(array_name); + + // Prepare cell buffers + uint64_t buffer_d1[] = {1, 1, 1, 2, 3, 3, 3, 4}; + uint64_t buffer_d1_size = sizeof(buffer_d1); + uint64_t buffer_d2[] = {1, 2, 4, 3, 1, 3}; + uint64_t buffer_d2_size = sizeof(buffer_d2); + + // Open array + tiledb_array_t* array; + int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); + CHECK(rc == TILEDB_OK); + rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); + CHECK(rc == TILEDB_OK); + + // Create query + tiledb_query_t* query; + rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_layout(ctx_, query, TILEDB_UNORDERED); + CHECK(rc == TILEDB_OK); + + // Set buffers with different coord number + rc = tiledb_query_set_buffer(ctx_, query, "d1", buffer_d1, &buffer_d1_size); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_buffer(ctx_, query, "d2", buffer_d2, &buffer_d2_size); + CHECK(rc == TILEDB_ERR); + + // Set zipped coordinates buffer after separate coordinate buffers + // have been set + uint64_t buffer_coords[] = {1, 1, 2, 3}; + uint64_t buffer_coords_size = sizeof(buffer_coords); + rc = tiledb_query_set_buffer( + ctx_, query, TILEDB_COORDS, buffer_coords, &buffer_coords_size); + CHECK(rc == TILEDB_ERR); + + // Set coordinates buffer for dimension that does not exist + rc = tiledb_query_set_buffer(ctx_, query, "foo", buffer_d1, &buffer_d1_size); + CHECK(rc == TILEDB_ERR); + + // Set rest of the attribute buffers + int buffer_a1[] = {0, 1, 2, 3, 4, 5, 6, 7}; + uint64_t buffer_a2[] = {0, 1, 3, 6, 10, 11, 13, 16}; + char buffer_var_a2[] = "abbcccddddeffggghhhh"; + float buffer_a3[] = {0.1f, + 0.2f, + 1.1f, + 1.2f, + 2.1f, + 2.2f, + 3.1f, + 3.2f, + 4.1f, + 4.2f, + 5.1f, + 5.2f, + 6.1f, + 6.2f, + 7.1f, + 7.2f}; + uint64_t buffer_a1_size = sizeof(buffer_a1); + uint64_t buffer_a2_size = sizeof(buffer_a2); + // No need to store the last '\0' character + uint64_t buffer_var_a2_size = sizeof(buffer_var_a2) - 1; + uint64_t buffer_a3_size = sizeof(buffer_a3); + rc = tiledb_query_set_buffer(ctx_, query, "a1", buffer_a1, &buffer_a1_size); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_buffer(ctx_, query, "a3", buffer_a3, &buffer_a3_size); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_buffer_var( + ctx_, + query, + "a2", + (uint64_t*)buffer_a2, + &buffer_a2_size, + buffer_var_a2, + &buffer_var_a2_size); + CHECK(rc == TILEDB_OK); + + // Submit query + rc = tiledb_query_submit(ctx_, query); + CHECK(rc == TILEDB_ERR); + + tiledb_query_free(&query); + + // Set zipped coordinates first and the separate coordinate buffers + rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_layout(ctx_, query, TILEDB_UNORDERED); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_buffer( + ctx_, query, TILEDB_COORDS, buffer_coords, &buffer_coords_size); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_buffer(ctx_, query, "d1", buffer_d1, &buffer_d1_size); + CHECK(rc == TILEDB_ERR); + + // Close array + rc = tiledb_array_close(ctx_, array); + CHECK(rc == TILEDB_OK); + + // Clean up + tiledb_array_free(&array); + tiledb_query_free(&query); + + remove_array(array_name); +} diff --git a/test/src/unit-capi-string.cc b/test/src/unit-capi-string.cc index 617f7827d4bf..7899282616da 100644 --- a/test/src/unit-capi-string.cc +++ b/test/src/unit-capi-string.cc @@ -38,6 +38,8 @@ #include #include +using namespace tiledb::test; + const char UTF8_STRINGS[] = u8"aabbccdd"; const char UTF8_STRINGS_VAR[] = u8"aαbββcγγγdδδδδ"; uint64_t UTF8_NULL_SIZE = sizeof(u8""); diff --git a/test/src/unit-cppapi-metadata.cc b/test/src/unit-cppapi-metadata.cc index e3c2eba93bb0..d6b79e12d573 100644 --- a/test/src/unit-cppapi-metadata.cc +++ b/test/src/unit-cppapi-metadata.cc @@ -47,6 +47,7 @@ #include using namespace tiledb; +using namespace tiledb::test; /* ********************************* */ /* STRUCT DEFINITION */ @@ -118,9 +119,9 @@ void CPPMetadataFx::create_default_array_1d() { {"a", "b", "c"}, {TILEDB_INT32, TILEDB_CHAR, TILEDB_FLOAT32}, {1, TILEDB_VAR_NUM, 2}, - {::Compressor(TILEDB_FILTER_NONE, -1), - ::Compressor(TILEDB_FILTER_ZSTD, -1), - ::Compressor(TILEDB_FILTER_LZ4, -1)}, + {tiledb::test::Compressor(TILEDB_FILTER_NONE, -1), + tiledb::test::Compressor(TILEDB_FILTER_ZSTD, -1), + tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1)}, TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR, 2); @@ -143,9 +144,9 @@ void CPPMetadataFx::create_default_array_1d_with_key() { {"a", "b", "c"}, {TILEDB_INT32, TILEDB_CHAR, TILEDB_FLOAT32}, {1, TILEDB_VAR_NUM, 2}, - {::Compressor(TILEDB_FILTER_NONE, -1), - ::Compressor(TILEDB_FILTER_ZSTD, -1), - ::Compressor(TILEDB_FILTER_LZ4, -1)}, + {tiledb::test::Compressor(TILEDB_FILTER_NONE, -1), + tiledb::test::Compressor(TILEDB_FILTER_ZSTD, -1), + tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1)}, TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR, 2); diff --git a/tiledb/sm/array_schema/array_schema.cc b/tiledb/sm/array_schema/array_schema.cc index e36332bf2f93..7ea755610037 100644 --- a/tiledb/sm/array_schema/array_schema.cc +++ b/tiledb/sm/array_schema/array_schema.cc @@ -104,6 +104,11 @@ ArraySchema::ArraySchema(const ArraySchema* array_schema) { } for (const auto& attr : attributes_) attribute_map_[attr->name()] = attr; + auto dim_num = array_schema->dim_num(); + for (unsigned d = 0; d < dim_num; ++d) { + auto dim = dimension(d); + dim_map_[dim->name()] = dim; + } } ArraySchema::~ArraySchema() { @@ -291,6 +296,18 @@ const Dimension* ArraySchema::dimension(unsigned int i) const { return domain_->dimension(i); } +const Dimension* ArraySchema::dimension(std::string name) const { + bool anonymous = name.empty(); + auto dim_num = this->dim_num(); + for (unsigned d = 0; d < dim_num; ++d) { + auto dim = this->dimension(d); + if ((dim->name() == name) || (anonymous && dim->is_anonymous())) { + return dim; + } + } + return nullptr; +} + unsigned int ArraySchema::dim_num() const { return domain_->dim_num(); } @@ -409,11 +426,24 @@ Datatype ArraySchema::type(const std::string& attribute) const { return it->second->type(); } -bool ArraySchema::var_size(const std::string& attribute) const { - auto it = attribute_map_.find(attribute); - if (it == attribute_map_.end()) +bool ArraySchema::var_size(const std::string& name) const { + // Special case for zipped coordinates + if (name == constants::coords) return false; - return it->second->var_size(); + + // Attribute + auto attr_it = attribute_map_.find(name); + if (attr_it != attribute_map_.end()) + return attr_it->second->var_size(); + + // Dimension + auto dim_it = dim_map_.find(name); + if (dim_it != dim_map_.end()) + return dim_it->second->var_size(); + + // Name is not an attribute or dimension + assert(false); + return false; } Status ArraySchema::add_attribute(const Attribute* attr, bool check_special) { @@ -523,13 +553,17 @@ Status ArraySchema::init() { attribute_map_.clear(); for (const auto& attr : attributes_) attribute_map_[attr->name()] = attr; + dim_map_.clear(); + auto dim_num = domain_->dim_num(); + for (unsigned d = 0; d < dim_num; ++d) { + auto dim = dimension(d); + dim_map_[dim->name()] = dim; + } // Set cell sizes for (auto& attr : attributes_) cell_sizes_[attr->name()] = compute_cell_size(attr->name()); cell_sizes_[constants::coords] = compute_cell_size(constants::coords); - - auto dim_num = domain_->dim_num(); coords_size_ = dim_num * datatype_size(coords_type()); // Success diff --git a/tiledb/sm/array_schema/array_schema.h b/tiledb/sm/array_schema/array_schema.h index ff80bd25302d..3f31455d0319 100644 --- a/tiledb/sm/array_schema/array_schema.h +++ b/tiledb/sm/array_schema/array_schema.h @@ -90,10 +90,16 @@ class ArraySchema { /** Returns the array uri. */ const URI& array_uri() const; - /** Returns a constant pointer to the selected attribute (NULL if error). */ + /** + * Returns a constant pointer to the selected attribute (nullptr if it + * does not exist). + */ const Attribute* attribute(unsigned int id) const; - /** Returns a constant pointer to the selected attribute (NULL if error). */ + /** + * Returns a constant pointer to the selected attribute (nullptr if it + * does not exist). + */ const Attribute* attribute(std::string name) const; /** @@ -188,6 +194,12 @@ class ArraySchema { /** Returns the i-th dimension. */ const Dimension* dimension(unsigned int i) const; + /** + * Returns a constant pointer to the selected dimension (nullptr if it + * does not exist). + */ + const Dimension* dimension(std::string name) const; + /** Returns the number of dimensions. */ unsigned int dim_num() const; @@ -221,8 +233,11 @@ class ArraySchema { /** Returns the type of the input attribute (could be coordinates). */ Datatype type(const std::string& attribute) const; - /** Returns *true* if the indicated attribute has variable-sized values. */ - bool var_size(const std::string& attribute) const; + /** + * Returns *true* if the input attribute/dimension has variable-sized + * values. + */ + bool var_size(const std::string& name) const; /** * Adds an attribute, copying the input. @@ -323,6 +338,9 @@ class ArraySchema { /** The size (in bytes) of the coordinates. */ uint64_t coords_size_; + /** It maps each dimension name to the corresponding dimension object. */ + std::unordered_map dim_map_; + /** The array domain. */ Domain* domain_; diff --git a/tiledb/sm/array_schema/dimension.cc b/tiledb/sm/array_schema/dimension.cc index 1476c5e290f6..d6ac2cabcf83 100644 --- a/tiledb/sm/array_schema/dimension.cc +++ b/tiledb/sm/array_schema/dimension.cc @@ -426,6 +426,11 @@ Datatype Dimension::type() const { return type_; } +bool Dimension::var_size() const { + // TODO: to fix when adding var-sized support to dimensions + return false; +} + /* ********************************* */ /* PRIVATE METHODS */ /* ********************************* */ diff --git a/tiledb/sm/array_schema/dimension.h b/tiledb/sm/array_schema/dimension.h index 23ff33d2df30..03a2a3d7b1f5 100644 --- a/tiledb/sm/array_schema/dimension.h +++ b/tiledb/sm/array_schema/dimension.h @@ -165,6 +165,9 @@ class Dimension { /** Returns the dimension type. */ Datatype type() const; + /** Returns true if the dimension is var-sized. */ + bool var_size() const; + private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ diff --git a/tiledb/sm/query/query.cc b/tiledb/sm/query/query.cc index a3f39da0a452..924d49b72542 100644 --- a/tiledb/sm/query/query.cc +++ b/tiledb/sm/query/query.cc @@ -186,17 +186,16 @@ const ArraySchema* Query::array_schema() const { return reader_.array_schema(); } -std::vector Query::attributes() const { +std::vector Query::buffer_names() const { if (type_ == QueryType::WRITE) - return writer_.attributes(); - return reader_.attributes(); + return writer_.buffer_names(); + return reader_.attributes(); // TODO: this will change in a subsequent PR } -AttributeBuffer Query::attribute_buffer( - const std::string& attribute_name) const { +QueryBuffer Query::buffer(const std::string& name) const { if (type_ == QueryType::WRITE) - return writer_.buffer(attribute_name); - return reader_.buffer(attribute_name); + return writer_.buffer(name); + return reader_.buffer(name); } Status Query::finalize() { @@ -219,39 +218,41 @@ Status Query::finalize() { return Status::Ok(); } +// TODO: fix normalized for coords Status Query::get_buffer( - const char* attribute, void** buffer, uint64_t** buffer_size) const { + const char* name, void** buffer, uint64_t** buffer_size) const { // Normalize attribute std::string normalized; - RETURN_NOT_OK(ArraySchema::attribute_name_normalized(attribute, &normalized)); + RETURN_NOT_OK(ArraySchema::attribute_name_normalized(name, &normalized)); // Check attribute auto array_schema = this->array_schema(); if (normalized != constants::coords) { - if (array_schema->attribute(normalized) == nullptr) + if (array_schema->attribute(normalized) == nullptr && + array_schema->dimension(normalized) == nullptr) return LOG_STATUS(Status::QueryError( - std::string("Cannot get buffer; Invalid attribute name '") + + std::string("Cannot get buffer; Invalid attribute/dimension name '") + normalized + "'")); } if (array_schema->var_size(normalized)) return LOG_STATUS(Status::QueryError( - std::string("Cannot get buffer; Attribute '") + normalized + - "' is var-sized")); + std::string("Cannot get buffer; '") + normalized + "' is var-sized")); if (type_ == QueryType::WRITE) return writer_.get_buffer(normalized, buffer, buffer_size); return reader_.get_buffer(normalized, buffer, buffer_size); } +// TODO: fix normalized for coords Status Query::get_buffer( - const char* attribute, + const char* name, uint64_t** buffer_off, uint64_t** buffer_off_size, void** buffer_val, uint64_t** buffer_val_size) const { // Normalize attribute std::string normalized; - RETURN_NOT_OK(ArraySchema::attribute_name_normalized(attribute, &normalized)); + RETURN_NOT_OK(ArraySchema::attribute_name_normalized(name, &normalized)); // Check attribute auto array_schema = this->array_schema(); @@ -259,14 +260,14 @@ Status Query::get_buffer( return LOG_STATUS( Status::QueryError("Cannot get buffer; Coordinates are not var-sized")); } - if (array_schema->attribute(normalized) == nullptr) + if (array_schema->attribute(normalized) == nullptr && + array_schema->dimension(normalized) == nullptr) return LOG_STATUS(Status::QueryError( - std::string("Cannot get buffer; Invalid attribute name '") + + std::string("Cannot get buffer; Invalid attribute/dimension name '") + normalized + "'")); if (!array_schema->var_size(normalized)) return LOG_STATUS(Status::QueryError( - std::string("Cannot get buffer; Attribute '") + normalized + - "' is fixed-sized")); + std::string("Cannot get buffer; '") + normalized + "' is fixed-sized")); if (type_ == QueryType::WRITE) return writer_.get_buffer( diff --git a/tiledb/sm/query/query.h b/tiledb/sm/query/query.h index 116ce8ddbb1d..641f00ab0997 100644 --- a/tiledb/sm/query/query.h +++ b/tiledb/sm/query/query.h @@ -175,14 +175,14 @@ class Query { /** Returns the array schema. */ const ArraySchema* array_schema() const; + /** Returns the names of the buffers set by the user for the query. */ + std::vector buffer_names() const; + /** - * Return vector of attributes - * @return attributes for query + * Gets the query buffer for the input attribute/dimension. + * An empty string means the special default attribute. */ - std::vector attributes() const; - - /** Gets the AttributeBuffer for an attribute. */ - AttributeBuffer attribute_buffer(const std::string& attribute_name) const; + QueryBuffer buffer(const std::string& name) const; /** * Marks a query that has not yet been started as failed. This should not be @@ -212,21 +212,23 @@ class Query { Status finalize(); /** - * Retrieves the buffer of a fixed-sized attribute. + * Retrieves the buffer of a fixed-sized attribute/dimension. * - * @param attribute The buffer attribute. An empty string means - * the special default attribute. + * @param name The buffer attribute/dimension name. An empty string means + * the special default attribute/dimension. * @param buffer The buffer to be retrieved. * @param buffer_size A pointer to the buffer size to be retrieved. * @return Status */ Status get_buffer( - const char* attribute, void** buffer, uint64_t** buffer_size) const; + const char* name, void** buffer, uint64_t** buffer_size) const; /** - * Retrieves the offsets and values buffers of a var-sized attribute. + * Retrieves the offsets and values buffers of a var-sized + * attribute/dimension. * - * @param attribute The buffer attribute. + * @param name The attribute/dimension name. An empty string means + * the special default attribute/dimension. * @param buffer_off The offsets buffer to be retrieved. * @param buffer_off_size A pointer to the offsets buffer size to be * retrieved. @@ -235,7 +237,7 @@ class Query { * @return Status */ Status get_buffer( - const char* attribute, + const char* name, uint64_t** buffer_off, uint64_t** buffer_off_size, void** buffer_val, diff --git a/tiledb/sm/query/reader.cc b/tiledb/sm/query/reader.cc index add47723fc82..054a62809bc5 100644 --- a/tiledb/sm/query/reader.cc +++ b/tiledb/sm/query/reader.cc @@ -143,17 +143,20 @@ std::vector Reader::attributes() const { return attributes_; } -AttributeBuffer Reader::buffer(const std::string& attribute) const { - auto attrbuf = attr_buffers_.find(attribute); - if (attrbuf == attr_buffers_.end()) - return AttributeBuffer{}; - return attrbuf->second; +QueryBuffer Reader::buffer(const std::string& name) const { + // TODO: fetch separate coordinate buffers as well. To be addressed in + // TODO: subsequent PR + auto buf = attr_buffers_.find(name); + if (buf == attr_buffers_.end()) + return QueryBuffer{}; + return buf->second; } bool Reader::incomplete() const { return read_state_.overflowed_ || !read_state_.done(); } +// TODO: handle both attributes and dimensions Status Reader::get_buffer( const std::string& attribute, void** buffer, uint64_t** buffer_size) const { auto it = attr_buffers_.find(attribute); @@ -168,6 +171,7 @@ Status Reader::get_buffer( return Status::Ok(); } +// TODO: handle both attributes and dimensions Status Reader::get_buffer( const std::string& attribute, uint64_t** buffer_off, @@ -399,8 +403,7 @@ Status Reader::set_buffer( attributes_.emplace_back(attribute); // Set attribute buffer - attr_buffers_[attribute] = - AttributeBuffer(buffer, nullptr, buffer_size, nullptr); + attr_buffers_[attribute] = QueryBuffer(buffer, nullptr, buffer_size, nullptr); return Status::Ok(); } @@ -451,7 +454,7 @@ Status Reader::set_buffer( // Set attribute buffer attr_buffers_[attribute] = - AttributeBuffer(buffer_off, buffer_val, buffer_off_size, buffer_val_size); + QueryBuffer(buffer_off, buffer_val, buffer_off_size, buffer_val_size); return Status::Ok(); } diff --git a/tiledb/sm/query/reader.h b/tiledb/sm/query/reader.h index 2aecde03d8d8..8ca8c1bf6087 100644 --- a/tiledb/sm/query/reader.h +++ b/tiledb/sm/query/reader.h @@ -175,12 +175,8 @@ class Reader { */ std::vector attributes() const; - /** - * Fetch AttributeBuffer for attribute - * @param attribute to fetch - * @return AttributeBuffer for attribute - */ - AttributeBuffer buffer(const std::string& attribute) const; + /** Fetch QueryBuffer for the input attribute/dimension. */ + QueryBuffer buffer(const std::string& name) const; /** * Returns `true` if the query was incomplete, i.e., if all subarray @@ -494,7 +490,7 @@ class Reader { std::vector attributes_; /** Maps attribute names to their buffers. */ - std::unordered_map attr_buffers_; + std::unordered_map attr_buffers_; /** The fragment metadata. */ std::vector fragment_metadata_; diff --git a/tiledb/sm/query/types.h b/tiledb/sm/query/types.h index 6b5fdcec66b7..7fa349f9f53d 100644 --- a/tiledb/sm/query/types.h +++ b/tiledb/sm/query/types.h @@ -39,16 +39,16 @@ namespace sm { /* TYPE DEFINITIONS */ /* ********************************* */ -/** Contains the buffer(s) and buffer size(s) for some attribute. */ -struct AttributeBuffer { +/** Contains the buffer(s) and buffer size(s) for some attribute / dimension. */ +struct QueryBuffer { /** - * The attribute buffer. In case the attribute is var-sized, this is - * the offsets buffer. + * The attribute/coordinate buffer. In case the attribute/dimension is + * var-sized, this is the offsets buffer. */ void* buffer_; /** - * For a var-sized attribute, this is the data buffer. It is `nullptr` - * for fixed-sized attributes. + * For a var-sized attribute/dimension, this is the data buffer. It is + * `nullptr` for fixed-sized attributes/dimensions. */ void* buffer_var_; /** @@ -73,7 +73,7 @@ struct AttributeBuffer { uint64_t original_buffer_var_size_; /** Constructor. */ - AttributeBuffer() { + QueryBuffer() { buffer_ = nullptr; buffer_var_ = nullptr; buffer_size_ = nullptr; @@ -83,69 +83,7 @@ struct AttributeBuffer { } /** Constructor. */ - AttributeBuffer( - void* buffer, - void* buffer_var, - uint64_t* buffer_size, - uint64_t* buffer_var_size) - : buffer_(buffer) - , buffer_var_(buffer_var) - , buffer_size_(buffer_size) - , buffer_var_size_(buffer_var_size) { - original_buffer_size_ = *buffer_size; - original_buffer_var_size_ = - (buffer_var_size_ != nullptr) ? *buffer_var_size : 0; - } -}; - -/** - * Contains the buffer(s) and buffer size(s) for the coordinates of - * a dimension. - */ -struct CoordBuffer { - /** - * The coordinate buffer. In case the dimension is var-sized, this is - * the offsets buffer. - */ - void* buffer_; - /** - * For a var-sized dimension, this is the data buffer. It is `nullptr` - * for fixed-sized dimensions. - */ - void* buffer_var_; - /** - * The size (in bytes) of `buffer_`. Note that this size may be altered by - * a read query to reflect the useful data written in the buffer. - */ - uint64_t* buffer_size_; - /** - * The size (in bytes) of `buffer_var_`. Note that this size may be altered - * by a read query to reflect the useful data written in the buffer. - */ - uint64_t* buffer_var_size_; - /** - * This is the original size (in bytes) of `buffer_` (before - * potentially altered by the query). - */ - uint64_t original_buffer_size_; - /** - * This is the original size (in bytes) of `buffer_var_` (before - * potentially altered by the query). - */ - uint64_t original_buffer_var_size_; - - /** Constructor. */ - CoordBuffer() { - buffer_ = nullptr; - buffer_var_ = nullptr; - buffer_size_ = nullptr; - buffer_var_size_ = nullptr; - original_buffer_size_ = 0; - original_buffer_var_size_ = 0; - } - - /** Constructor. */ - CoordBuffer( + QueryBuffer( void* buffer, void* buffer_var, uint64_t* buffer_size, diff --git a/tiledb/sm/query/writer.cc b/tiledb/sm/query/writer.cc index 42d569411814..2d8a987ecf16 100644 --- a/tiledb/sm/query/writer.cc +++ b/tiledb/sm/query/writer.cc @@ -74,28 +74,45 @@ Writer::~Writer() { /* API */ /* ****************************** */ -std::vector Writer::attributes() const { +const ArraySchema* Writer::array_schema() const { + return array_schema_; +} + +std::vector Writer::buffer_names() const { std::vector ret; + + // Attributes for (const auto& it : attr_buffers_) ret.push_back(it.first); - if (coords_buffer_ != nullptr) + + // Coordinates + if (coords_buffer_ != nullptr) { ret.push_back(constants::coords); - return ret; -} + } else { + for (const auto& it : coord_buffers_) + ret.push_back(it.first); + } -const ArraySchema* Writer::array_schema() const { - return array_schema_; + return ret; } -AttributeBuffer Writer::buffer(const std::string& name) const { +QueryBuffer Writer::buffer(const std::string& name) const { + // Special zipped coordinates if (name == constants::coords) - return AttributeBuffer( - coords_buffer_, nullptr, coords_buffer_size_, nullptr); + return QueryBuffer(coords_buffer_, nullptr, coords_buffer_size_, nullptr); - auto attrbuf = attr_buffers_.find(name); - if (attrbuf == attr_buffers_.end()) - return AttributeBuffer{}; - return attrbuf->second; + // Attribute + auto attr_buf = attr_buffers_.find(name); + if (attr_buf != attr_buffers_.end()) + return attr_buf->second; + + // Dimension + auto coord_buf = coord_buffers_.find(name); + if (coord_buf != coord_buffers_.end()) + return coord_buf->second; + + // Named buffer does not exist + return QueryBuffer{}; } Status Writer::finalize() { @@ -106,21 +123,33 @@ Status Writer::finalize() { Status Writer::get_buffer( const std::string& name, void** buffer, uint64_t** buffer_size) const { + // Special zipped coordinates if (name == constants::coords) { *buffer = coords_buffer_; *buffer_size = coords_buffer_size_; return Status::Ok(); } - auto it = attr_buffers_.find(name); - if (it == attr_buffers_.end()) { - *buffer = nullptr; - *buffer_size = nullptr; - } else { - *buffer = it->second.buffer_; - *buffer_size = it->second.buffer_size_; + // Attribute + auto attr_it = attr_buffers_.find(name); + if (attr_it != attr_buffers_.end()) { + *buffer = attr_it->second.buffer_; + *buffer_size = attr_it->second.buffer_size_; + return Status::Ok(); + } + + // Dimension + auto coord_it = coord_buffers_.find(name); + if (coord_it != coord_buffers_.end()) { + *buffer = coord_it->second.buffer_; + *buffer_size = coord_it->second.buffer_size_; + return Status::Ok(); } + // Named buffer does not exist + *buffer = nullptr; + *buffer_size = nullptr; + return Status::Ok(); } @@ -130,19 +159,32 @@ Status Writer::get_buffer( uint64_t** buffer_off_size, void** buffer_val, uint64_t** buffer_val_size) const { - auto it = attr_buffers_.find(name); - if (it == attr_buffers_.end()) { - *buffer_off = nullptr; - *buffer_off_size = nullptr; - *buffer_val = nullptr; - *buffer_val_size = nullptr; - } else { - *buffer_off = (uint64_t*)it->second.buffer_; - *buffer_off_size = it->second.buffer_size_; - *buffer_val = it->second.buffer_var_; - *buffer_val_size = it->second.buffer_var_size_; + // Attribute + auto attr_it = attr_buffers_.find(name); + if (attr_it != attr_buffers_.end()) { + *buffer_off = (uint64_t*)attr_it->second.buffer_; + *buffer_off_size = attr_it->second.buffer_size_; + *buffer_val = attr_it->second.buffer_var_; + *buffer_val_size = attr_it->second.buffer_var_size_; + return Status::Ok(); + } + + // Dimension + auto coord_it = coord_buffers_.find(name); + if (coord_it != coord_buffers_.end()) { + *buffer_off = (uint64_t*)coord_it->second.buffer_; + *buffer_off_size = coord_it->second.buffer_size_; + *buffer_val = coord_it->second.buffer_var_; + *buffer_val_size = coord_it->second.buffer_var_size_; + return Status::Ok(); } + // Named buffer does not exist + *buffer_off = nullptr; + *buffer_off_size = nullptr; + *buffer_val = nullptr; + *buffer_val_size = nullptr; + return Status::Ok(); } @@ -186,7 +228,7 @@ Status Writer::init() { RETURN_NOT_OK(set_subarray(nullptr)); RETURN_NOT_OK(check_subarray()); RETURN_NOT_OK(check_buffer_sizes()); - RETURN_NOT_OK(check_attributes()); + RETURN_NOT_OK(check_buffer_names()); optimize_layout_for_1D(); @@ -205,6 +247,11 @@ Status Writer::init() { dedup_coords_ = !strcmp(dedup_coords, "true"); initialized_ = true; + auto dim_num = array_schema_->dim_num(); + coord_sizes_.resize(dim_num); + for (unsigned d = 0; d < dim_num; ++d) + coord_sizes_[d] = array_schema_->dimension(d)->coord_size(); + return Status::Ok(); } @@ -232,37 +279,18 @@ Status Writer::set_buffer( return LOG_STATUS( Status::WriterError("Cannot set buffer; Array schema not set")); - // Check that attribute exists - if (name != constants::coords && array_schema_->attribute(name) == nullptr) - return LOG_STATUS( - Status::WriterError("Cannot set buffer; Invalid attribute")); - - // Check that attribute is fixed-sized - bool var_size = (name != constants::coords && array_schema_->var_size(name)); - if (var_size) - return LOG_STATUS(Status::WriterError( - std::string("Cannot set buffer; Input attribute '") + name + - "' is var-sized")); - - // Error if setting a new attribute after initialization - bool attr_exists = attr_buffers_.find(name) != attr_buffers_.end(); - if (initialized_ && name != constants::coords && !attr_exists) - return LOG_STATUS(Status::WriterError( - std::string("Cannot set buffer for new attribute '") + name + - "' after initialization")); - - // Error if setting non-existing coordinates after initialization - bool has_coords = coords_buffer_ != nullptr || !coord_buffers_.empty(); - if (initialized_ && name == constants::coords && has_coords) - return LOG_STATUS(Status::WriterError( - std::string("Cannot set coordinates after initialization"))); - + // Invoke the appropriate fuinction based on the buffer name if (name == constants::coords) { - coords_buffer_ = buffer; - coords_buffer_size_ = buffer_size; + return set_coords_buffer(buffer, buffer_size); + } else if (array_schema_->attribute(name) != nullptr) { + return set_attr_buffer(name, buffer, buffer_size); + } else if (array_schema_->dimension(name) != nullptr) { + return set_coord_buffer(name, buffer, buffer_size); } else { - attr_buffers_[name] = - AttributeBuffer(buffer, nullptr, buffer_size, nullptr); + return LOG_STATUS(Status::WriterError( + std::string("Cannot set buffer; Invalid buffer name '") + name + + "' (it should " + "be an attribute or dimension)")); } return Status::Ok(); @@ -306,7 +334,7 @@ Status Writer::set_buffer( // Set attribute buffer attr_buffers_[name] = - AttributeBuffer(buffer_off, buffer_val, buffer_off_size, buffer_val_size); + QueryBuffer(buffer_off, buffer_val, buffer_off_size, buffer_val_size); return Status::Ok(); } @@ -416,7 +444,7 @@ void Writer::add_written_fragment_info(const URI& uri) { written_fragment_info_.emplace_back(uri, timestamp_range); } -Status Writer::check_attributes() { +Status Writer::check_buffer_names() { bool has_coords = !coord_buffers_.empty() || coords_buffer_ != nullptr; // If the array is sparse, the coordinates must be provided @@ -435,6 +463,12 @@ Status Writer::check_attributes() { return LOG_STATUS( Status::WriterError("Writes expect all attributes to be set")); + // If coordinates were given, they must be given for all dimensions + if (!coord_buffers_.empty() && + coord_buffers_.size() != array_schema_->dim_num()) + return LOG_STATUS(Status::WriterError( + "Writes expect coordinate buffers to be set for all dimensions")); + return Status::Ok(); } @@ -2559,7 +2593,6 @@ Status Writer::split_coords_buffer() { clear_coord_buffers(); coord_buffers_alloced_ = true; - coord_sizes_.resize(dim_num); // New coord buffer allocations for (unsigned d = 0; d < dim_num; ++d) { @@ -2567,14 +2600,13 @@ Status Writer::split_coords_buffer() { const auto& dim_name = dim->name(); auto coord_buffer_size = coords_num_ * dim->coord_size(); auto it = coord_buffer_sizes_.emplace(dim_name, coord_buffer_size); - CoordBuffer buff; + QueryBuffer buff; buff.buffer_size_ = &(it.first->second); buff.buffer_ = std::malloc(coord_buffer_size); if (buff.buffer_ == nullptr) RETURN_NOT_OK(Status::WriterError( "Cannot split coordinate buffers; memory allocation failed")); coord_buffers_.emplace(dim_name, buff); - coord_sizes_[d] = array_schema_->dimension(d)->coord_size(); } // Split coordinates @@ -2898,5 +2930,83 @@ void Writer::clean_up(const URI& uri) { global_write_state_.reset(nullptr); } +Status Writer::set_coords_buffer(void* buffer, uint64_t* buffer_size) { + // Error if setting non-existing coordinates after initialization + bool has_coords = coords_buffer_ != nullptr || !coord_buffers_.empty(); + if (initialized_ && has_coords) + return LOG_STATUS(Status::WriterError( + std::string("Cannot set coordinates after initialization"))); + + if (!coord_buffers_.empty()) + return LOG_STATUS(Status::WriterError( + std::string("Cannot set zipped coordinates buffer after having set " + "separate coordinate buffers"))); + + // Set zipped coordinates buffer + coords_buffer_ = buffer; + coords_buffer_size_ = buffer_size; + + return Status::Ok(); +} + +Status Writer::set_coord_buffer( + const std::string& name, void* buffer, uint64_t* buffer_size) { + // Check that dimension is fixed-sized + auto dim = array_schema_->dimension(name); + if (dim->var_size()) + return LOG_STATUS(Status::WriterError( + std::string("Cannot set buffer; Input dimension '") + name + + "' is var-sized")); + + // Check if zipped coordinates buffer is set + if (coords_buffer_ != nullptr) + return LOG_STATUS(Status::WriterError( + std::string("Cannot set separate coordinates buffer after having " + "set the zipped coordinates buffer"))); + + // Check number of coordinates + uint64_t coords_num = *buffer_size / dim->coord_size(); + if (!coord_buffers_.empty() && coords_num != coords_num_) + return LOG_STATUS(Status::WriterError( + std::string("Cannot set buffer; Input buffer for dimension '") + name + + "' has a different number of coordinates than previously " + "set coordinate buffers")); + + // Error if setting a new dimension after initialization + bool dim_exists = coord_buffers_.find(name) != coord_buffers_.end(); + if (initialized_ && !dim_exists) + return LOG_STATUS(Status::WriterError( + std::string("Cannot set buffer for new dimension '") + name + + "' after initialization")); + + // Set coordinate buffer + coord_buffers_[name] = QueryBuffer(buffer, nullptr, buffer_size, nullptr); + coords_num_ = coords_num; + + return Status::Ok(); +} + +Status Writer::set_attr_buffer( + const std::string& name, void* buffer, uint64_t* buffer_size) { + // Check that attribute is fixed-sized + bool var_size = (array_schema_->var_size(name)); + if (var_size) + return LOG_STATUS(Status::WriterError( + std::string("Cannot set buffer; Input attribute '") + name + + "' is var-sized")); + + // Error if setting a new attribute after initialization + bool attr_exists = attr_buffers_.find(name) != attr_buffers_.end(); + if (initialized_ && !attr_exists) + return LOG_STATUS(Status::WriterError( + std::string("Cannot set buffer for new attribute '") + name + + "' after initialization")); + + // Set attribute buffer + attr_buffers_[name] = QueryBuffer(buffer, nullptr, buffer_size, nullptr); + + return Status::Ok(); +} + } // namespace sm } // namespace tiledb diff --git a/tiledb/sm/query/writer.h b/tiledb/sm/query/writer.h index 711e3ffd9567..445854d88a2d 100644 --- a/tiledb/sm/query/writer.h +++ b/tiledb/sm/query/writer.h @@ -129,14 +129,17 @@ class Writer { /* API */ /* ********************************* */ - /** Returns the attributes the user has set buffers for. */ - std::vector attributes() const; - /** Returns the array schema. */ const ArraySchema* array_schema() const; - /** Returns the query buffer with the given name. */ - AttributeBuffer buffer(const std::string& name) const; + /** Returns the names of the buffers set by the user for the write query. */ + std::vector buffer_names() const; + + /** + * Returns the query buffer for the given attribute/dimension name. + * The name can be TILEDB_COORDS. + */ + QueryBuffer buffer(const std::string& name) const; /** Finalizes the reader. */ Status finalize(); @@ -289,10 +292,10 @@ class Writer { const ArraySchema* array_schema_; /** Maps attribute names to their buffers. */ - std::unordered_map attr_buffers_; + std::unordered_map attr_buffers_; /** Maps dimension names to their coordinate buffers. */ - std::unordered_map coord_buffers_; + std::unordered_map coord_buffers_; /** The coordinates buffer potentially set by the user. */ void* coords_buffer_; @@ -383,8 +386,8 @@ class Writer { /** Adss a fragment to `written_fragment_info_`. */ void add_written_fragment_info(const URI& uri); - /** Checks if attributes has been appropriately set for the query. */ - Status check_attributes(); + /** Checks if the buffers names have been appropriately set for the query. */ + Status check_buffer_names(); /** Correctness checks for buffer sizes. */ Status check_buffer_sizes() const; @@ -1063,6 +1066,38 @@ class Writer { * in the global write state are empty. */ bool all_last_tiles_empty() const; + + /** + * Sets the (zipped) coordinates buffer (set with TILEDB_COORDS as the + * buffer name). + * + * @param buffer The buffer that has the input data to be written. + * @param buffer_size The size of `buffer` in bytes. + * @return Status + */ + Status set_coords_buffer(void* buffer, uint64_t* buffer_size); + + /** + * Sets the coordinate buffer on a single fixed-sized dimension. + * + * @param name The name of the dimension the buffer corresponds to. + * @param buffer The buffer that has the input data to be written. + * @param buffer_size The size of `buffer` in bytes. + * @return Status + */ + Status set_coord_buffer( + const std::string& name, void* buffer, uint64_t* buffer_size); + + /** + * Sets the attribute buffer on a single fixed-sized attribute. + * + * @param name The name of the dimension the buffer corresponds to. + * @param buffer The buffer that has the input data to be written. + * @param buffer_size The size of `buffer` in bytes. + * @return Status + */ + Status set_attr_buffer( + const std::string& name, void* buffer, uint64_t* buffer_size); }; } // namespace sm diff --git a/tiledb/sm/rest/rest_client.cc b/tiledb/sm/rest/rest_client.cc index 7ee4b663bfc8..7d95b32cc43e 100644 --- a/tiledb/sm/rest/rest_client.cc +++ b/tiledb/sm/rest/rest_client.cc @@ -625,55 +625,14 @@ Status RestClient::update_attribute_buffer_sizes( if (query->type() != QueryType::READ) return Status::Ok(); - const auto schema = query->array_schema(); - if (schema == nullptr) - return LOG_STATUS(Status::RestError( - "Error updating attribute buffer sizes; array schema is null")); - - const auto attrs = query->attributes(); - std::set attr_names; - attr_names.insert(constants::coords); - for (const auto& name : attrs) - attr_names.insert(name); - - for (const auto& attr_name : attr_names) { - const bool is_coords = attr_name == constants::coords; - const auto* attr = schema->attribute(attr_name); - if (!is_coords && attr == nullptr) - return LOG_STATUS(Status::RestError( - "Error updating attribute buffer sizes; no attribute object for '" + - attr_name + "'")); - - // Skip attributes that were not a part of the copy process. - auto copy_state_it = copy_state.find(attr_name); - if (copy_state_it == copy_state.end()) - continue; - auto attr_state = copy_state_it->second; - - const bool var_size = !is_coords && attr->var_size(); - if (var_size) { - uint64_t* offset_buffer = nullptr; - uint64_t* offset_buffer_size = nullptr; - void* buffer = nullptr; - uint64_t* buffer_size = nullptr; - RETURN_NOT_OK(query->get_buffer( - attr_name.c_str(), - &offset_buffer, - &offset_buffer_size, - &buffer, - &buffer_size)); - if (offset_buffer_size != nullptr) - *offset_buffer_size = attr_state.offset_size; - if (buffer_size != nullptr) - *buffer_size = attr_state.data_size; - } else { - void* buffer = nullptr; - uint64_t* buffer_size = nullptr; - RETURN_NOT_OK( - query->get_buffer(attr_name.c_str(), &buffer, &buffer_size)); - if (buffer_size != nullptr) - *buffer_size = attr_state.data_size; - } + for (const auto& cit : copy_state) { + const auto& name = cit.first; + auto state = cit.second; + auto query_buffer = query->buffer(name); + if (query_buffer.buffer_var_size_ != nullptr) + *query_buffer.buffer_var_size_ = state.offset_size; + if (query_buffer.buffer_size_ != nullptr) + *query_buffer.buffer_size_ = state.data_size; } return Status::Ok(); diff --git a/tiledb/sm/serialization/query.cc b/tiledb/sm/serialization/query.cc index c0d3fb0a37d0..85c51f972d61 100644 --- a/tiledb/sm/serialization/query.cc +++ b/tiledb/sm/serialization/query.cc @@ -420,39 +420,29 @@ Status query_to_capnp( } // Serialize attribute buffer metadata - const auto attr_names = query.attributes(); + const auto buffer_names = query.buffer_names(); auto attr_buffers_builder = - query_builder->initAttributeBufferHeaders(attr_names.size()); + query_builder->initAttributeBufferHeaders(buffer_names.size()); uint64_t total_fixed_len_bytes = 0; uint64_t total_var_len_bytes = 0; - for (uint64_t i = 0; i < attr_names.size(); i++) { + for (uint64_t i = 0; i < buffer_names.size(); i++) { auto attr_buffer_builder = attr_buffers_builder[i]; - const auto& attribute_name = attr_names[i]; - const auto& buff = query.attribute_buffer(attribute_name); - const bool is_coords = attribute_name == constants::coords; - const auto* attr = schema->attribute(attribute_name); - if (!is_coords && attr == nullptr) - return LOG_STATUS(Status::SerializationError( - "Cannot serialize; no attribute named '" + attribute_name + "'.")); - - const bool var_size = !is_coords && attr->var_size(); - attr_buffer_builder.setName(attribute_name); - if (var_size && - (buff.buffer_var_ != nullptr && buff.buffer_var_size_ != nullptr)) { - // Variable-sized attribute. - if (buff.buffer_ == nullptr || buff.buffer_size_ == nullptr) - return LOG_STATUS(Status::SerializationError( - "Cannot serialize; no offset buffer set for attribute '" + - attribute_name + "'.")); + const auto& name = buffer_names[i]; + const auto& buff = query.buffer(name); + attr_buffer_builder.setName(name); + if (buff.buffer_var_ != nullptr && buff.buffer_var_size_ != nullptr) { + // Variable-sized buffer total_var_len_bytes += *buff.buffer_var_size_; attr_buffer_builder.setVarLenBufferSizeInBytes(*buff.buffer_var_size_); total_fixed_len_bytes += *buff.buffer_size_; attr_buffer_builder.setFixedLenBufferSizeInBytes(*buff.buffer_size_); } else if (buff.buffer_ != nullptr && buff.buffer_size_ != nullptr) { - // Fixed-length attribute + // Fixed-length buffer total_fixed_len_bytes += *buff.buffer_size_; attr_buffer_builder.setFixedLenBufferSizeInBytes(*buff.buffer_size_); attr_buffer_builder.setVarLenBufferSizeInBytes(0); + } else { + assert(false); } } @@ -521,13 +511,7 @@ Status query_from_capnp( auto buffer_headers = query_reader.getAttributeBufferHeaders(); auto attribute_buffer_start = static_cast(buffer_start); for (auto buffer_header : buffer_headers) { - const std::string attribute_name = buffer_header.getName().cStr(); - const bool is_coords = attribute_name == constants::coords; - const auto* attr = schema->attribute(attribute_name); - if (!is_coords && attr == nullptr) - return LOG_STATUS(Status::SerializationError( - "Cannot deserialize; no attribute named '" + attribute_name + - "' in array schema.")); + const std::string name = buffer_header.getName().cStr(); // Get buffer sizes required const uint64_t fixedlen_size = buffer_header.getFixedLenBufferSizeInBytes(); @@ -537,42 +521,34 @@ Status query_from_capnp( // for memcpy into user buffers). QueryBufferCopyState* attr_copy_state = nullptr; if (copy_state != nullptr) - attr_copy_state = &(*copy_state)[attribute_name]; + attr_copy_state = &(*copy_state)[name]; // Get any buffers already set on this query object. uint64_t* existing_offset_buffer = nullptr; uint64_t* existing_offset_buffer_size = nullptr; void* existing_buffer = nullptr; uint64_t* existing_buffer_size = nullptr; - const bool var_size = !is_coords && attr->var_size(); + + auto var_size = schema->var_size(name); if (var_size) { RETURN_NOT_OK(query->get_buffer( - attribute_name.c_str(), + name.c_str(), &existing_offset_buffer, &existing_offset_buffer_size, &existing_buffer, &existing_buffer_size)); } else { RETURN_NOT_OK(query->get_buffer( - attribute_name.c_str(), &existing_buffer, &existing_buffer_size)); + name.c_str(), &existing_buffer, &existing_buffer_size)); } if (context == SerializationContext::CLIENT) { // For queries on the client side, we require that buffers have been // set by the user, and that they are large enough for all the serialized // data. - const bool null_buffer = - existing_buffer == nullptr || existing_buffer_size == nullptr; - const bool null_offset_buffer = existing_offset_buffer == nullptr || - existing_offset_buffer_size == nullptr; - if ((var_size && (null_buffer || null_offset_buffer)) || - (!var_size && null_buffer)) - return LOG_STATUS(Status::SerializationError( - "Error deserializing read query; buffer not set for attribute '" + - attribute_name + "'.")); - // Check sizes const uint64_t curr_data_size = attr_copy_state == nullptr ? 0 : attr_copy_state->data_size; + assert(existing_buffer_size != nullptr); const uint64_t data_size_left = *existing_buffer_size - curr_data_size; const uint64_t curr_offset_size = attr_copy_state == nullptr ? 0 : attr_copy_state->offset_size; @@ -580,13 +556,14 @@ Status query_from_capnp( existing_offset_buffer_size == nullptr ? 0 : *existing_offset_buffer_size - curr_offset_size; + if ((var_size && (offset_size_left < fixedlen_size || data_size_left < varlen_size)) || (!var_size && data_size_left < fixedlen_size)) { return LOG_STATUS(Status::SerializationError( - "Error deserializing read query; buffer too small for attribute " + "Error deserializing read query; buffer too small for buffer " "'" + - attribute_name + "'.")); + name + "'.")); } // For reads, copy the response data into user buffers. For writes, @@ -633,8 +610,7 @@ Status query_from_capnp( "buffer set on server-side.")); Query::SerializationState::AttrState* attr_state; - RETURN_NOT_OK( - query->get_attr_serialization_state(attribute_name, &attr_state)); + RETURN_NOT_OK(query->get_attr_serialization_state(name, &attr_state)); if (type == QueryType::READ) { // On reads, just set null pointers with accurate size so that the // server can introspect and allocate properly sized buffers separately. @@ -646,7 +622,7 @@ Status query_from_capnp( attr_state->var_len_data.swap(varlen_buff); if (var_size) { RETURN_NOT_OK(query->set_buffer( - attribute_name, + name, nullptr, &attr_state->fixed_len_size, nullptr, @@ -654,7 +630,7 @@ Status query_from_capnp( false)); } else { RETURN_NOT_OK(query->set_buffer( - attribute_name, nullptr, &attr_state->fixed_len_size, false)); + name, nullptr, &attr_state->fixed_len_size, false)); } } else { // On writes, just set buffer pointers wrapping the data in the message. @@ -669,7 +645,7 @@ Status query_from_capnp( attr_state->fixed_len_data.swap(offsets_buff); attr_state->var_len_data.swap(varlen_buff); RETURN_NOT_OK(query->set_buffer( - attribute_name, + name, offsets, &attr_state->fixed_len_size, varlen_data, @@ -683,8 +659,8 @@ Status query_from_capnp( attr_state->var_len_size = varlen_size; attr_state->fixed_len_data.swap(buff); attr_state->var_len_data.swap(varlen_buff); - RETURN_NOT_OK(query->set_buffer( - attribute_name, data, &attr_state->fixed_len_size)); + RETURN_NOT_OK( + query->set_buffer(name, data, &attr_state->fixed_len_size)); } } } @@ -761,60 +737,28 @@ Status query_serialize( header.write(message_chars.begin(), message_chars.size())); RETURN_NOT_OK(serialized_buffer->add_buffer(std::move(header))); - // Iterate over attributes and concatenate buffers to end of message. + // Concatenate buffers to end of message if (serialize_buffers) { auto attr_buffer_builders = query_builder.getAttributeBufferHeaders(); for (auto attr_buffer_builder : attr_buffer_builders) { - const std::string attribute_name = - attr_buffer_builder.getName().cStr(); - const bool is_coords = attribute_name == constants::coords; - const auto* attr = array_schema->attribute(attribute_name); - if (!is_coords && attr == nullptr) - return LOG_STATUS(Status::SerializationError( - "Cannot serialize; no attribute named '" + attribute_name + - "'.")); - - const bool var_size = !is_coords && attr->var_size(); - if (var_size) { - // Variable size attribute buffer - uint64_t* offset_buffer = nullptr; - uint64_t* offset_buffer_size = nullptr; - void* buffer = nullptr; - uint64_t* buffer_size = nullptr; - RETURN_NOT_OK(query->get_buffer( - attribute_name.c_str(), - &offset_buffer, - &offset_buffer_size, - &buffer, - &buffer_size)); - - if (offset_buffer != nullptr) { - if (offset_buffer_size == nullptr || buffer == nullptr || - buffer_size == nullptr) - return LOG_STATUS(Status::SerializationError( - "Cannot serialize; unexpected null buffers.")); - - Buffer offsets(offset_buffer, *offset_buffer_size); - Buffer data(buffer, *buffer_size); - RETURN_NOT_OK( - serialized_buffer->add_buffer(std::move(offsets))); - RETURN_NOT_OK(serialized_buffer->add_buffer(std::move(data))); - } + const std::string name = attr_buffer_builder.getName().cStr(); + + auto query_buffer = query->buffer(name); + + if (query_buffer.buffer_var_size_ != nullptr && + query_buffer.buffer_var_ != nullptr) { + Buffer offsets(query_buffer.buffer_, *query_buffer.buffer_size_); + RETURN_NOT_OK(serialized_buffer->add_buffer(std::move(offsets))); + Buffer data( + query_buffer.buffer_var_, *query_buffer.buffer_var_size_); + RETURN_NOT_OK(serialized_buffer->add_buffer(std::move(data))); + } else if ( + query_buffer.buffer_size_ != nullptr && + query_buffer.buffer_ != nullptr) { + Buffer data(query_buffer.buffer_, *query_buffer.buffer_size_); + RETURN_NOT_OK(serialized_buffer->add_buffer(std::move(data))); } else { - // Fixed size attribute buffer - void* buffer = nullptr; - uint64_t* buffer_size = nullptr; - RETURN_NOT_OK(query->get_buffer( - attribute_name.c_str(), &buffer, &buffer_size)); - - if (buffer != nullptr) { - if (buffer_size == nullptr) - return LOG_STATUS(Status::SerializationError( - "Cannot serialize; unexpected null buffer size.")); - - Buffer data(buffer, *buffer_size); - RETURN_NOT_OK(serialized_buffer->add_buffer(std::move(data))); - } + assert(false); } } }