Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Set separate coordinate buffers to write queries #1458

Merged
merged 1 commit into from
Dec 13, 2019
Merged

Set separate coordinate buffers to write queries #1458

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions HISTORY.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
## Improvements

* Added support for indicating zero-value metadata by returning `value_num` == 1 from the `_get_metadatata` and `Array::get_metadata` APIs [#1438](https://github.com/TileDB-Inc/TileDB/pull/1438) (this is a non-breaking change, as the documented return of `value == nullptr` to indicate missing keys does not change)`
* User can set coordinate buffers separately for write queries.

## Deprecations

Expand Down
36 changes: 21 additions & 15 deletions test/src/helpers.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,9 @@
#include "helpers.h"
#include "catch.hpp"

namespace tiledb {
namespace test {

template <class T>
void check_partitions(
tiledb::sm::SubarrayPartitioner& partitioner,
Expand Down Expand Up @@ -449,7 +452,7 @@ void write_array(
tiledb_ctx_t* ctx,
const std::string& array_name,
tiledb_layout_t layout,
const AttrBuffers& attr_buffers) {
const QueryBuffers& buffers) {
// Open array
tiledb_array_t* array;
int rc = tiledb_array_alloc(ctx, array_name.c_str(), &array);
Expand All @@ -465,7 +468,7 @@ void write_array(
CHECK(rc == TILEDB_OK);

// Set buffers
for (const auto& b : attr_buffers) {
for (const auto& b : buffers) {
if (b.second.var_ == nullptr) { // Fixed-sized
rc = tiledb_query_set_buffer(
ctx,
Expand Down Expand Up @@ -510,7 +513,7 @@ void read_array(
tiledb_array_t* array,
const SubarrayRanges<T>& ranges,
tiledb_layout_t layout,
const AttrBuffers& attr_buffers) {
const QueryBuffers& buffers) {
// Create query
tiledb_query_t* query;
int rc = tiledb_query_alloc(ctx, array, TILEDB_READ, &query);
Expand All @@ -529,7 +532,7 @@ void read_array(
}

// Set buffers
for (const auto& b : attr_buffers) {
for (const auto& b : buffers) {
if (b.second.var_ == nullptr) { // Fixed-sized
rc = tiledb_query_set_buffer(
ctx,
Expand Down Expand Up @@ -566,7 +569,6 @@ void read_array(
}

// Explicit template instantiations

template void check_subarray<int8_t>(
tiledb::sm::Subarray& subarray, const SubarrayRanges<int8_t>& ranges);
template void check_subarray<uint8_t>(
Expand Down Expand Up @@ -685,58 +687,62 @@ template void read_array<int8_t>(
tiledb_array_t* array,
const SubarrayRanges<int8_t>& ranges,
tiledb_layout_t layout,
const AttrBuffers& attr_buffers);
const QueryBuffers& buffers);
template void read_array<uint8_t>(
tiledb_ctx_t* ctx,
tiledb_array_t* array,
const SubarrayRanges<uint8_t>& ranges,
tiledb_layout_t layout,
const AttrBuffers& attr_buffers);
const QueryBuffers& buffers);
template void read_array<int16_t>(
tiledb_ctx_t* ctx,
tiledb_array_t* array,
const SubarrayRanges<int16_t>& ranges,
tiledb_layout_t layout,
const AttrBuffers& attr_buffers);
const QueryBuffers& buffers);
template void read_array<uint16_t>(
tiledb_ctx_t* ctx,
tiledb_array_t* array,
const SubarrayRanges<uint16_t>& ranges,
tiledb_layout_t layout,
const AttrBuffers& attr_buffers);
const QueryBuffers& buffers);
template void read_array<int32_t>(
tiledb_ctx_t* ctx,
tiledb_array_t* array,
const SubarrayRanges<int32_t>& ranges,
tiledb_layout_t layout,
const AttrBuffers& attr_buffers);
const QueryBuffers& buffers);
template void read_array<uint32_t>(
tiledb_ctx_t* ctx,
tiledb_array_t* array,
const SubarrayRanges<uint32_t>& ranges,
tiledb_layout_t layout,
const AttrBuffers& attr_buffers);
const QueryBuffers& buffers);
template void read_array<int64_t>(
tiledb_ctx_t* ctx,
tiledb_array_t* array,
const SubarrayRanges<int64_t>& ranges,
tiledb_layout_t layout,
const AttrBuffers& attr_buffers);
const QueryBuffers& buffers);
template void read_array<uint64_t>(
tiledb_ctx_t* ctx,
tiledb_array_t* array,
const SubarrayRanges<uint64_t>& ranges,
tiledb_layout_t layout,
const AttrBuffers& attr_buffers);
const QueryBuffers& buffers);
template void read_array<float>(
tiledb_ctx_t* ctx,
tiledb_array_t* array,
const SubarrayRanges<float>& ranges,
tiledb_layout_t layout,
const AttrBuffers& attr_buffers);
const QueryBuffers& buffers);
template void read_array<double>(
tiledb_ctx_t* ctx,
tiledb_array_t* array,
const SubarrayRanges<double>& ranges,
tiledb_layout_t layout,
const AttrBuffers& attr_buffers);
const QueryBuffers& buffers);

} // End of namespace test

} // End of namespace tiledb
35 changes: 23 additions & 12 deletions test/src/helpers.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,31 +42,38 @@
#include <string>
#include <thread>

namespace tiledb {

namespace test {

// For easy reference
typedef std::pair<tiledb_filter_type_t, int> Compressor;
template <class T>
using SubarrayRanges = std::vector<std::vector<T>>;

/** Helper struct for the buffers of an attribute (fixed- or var-sized). */
struct AttrBuffer {
/**
* Helper struct for the buffers of an attribute/dimension
* (fixed- or var-sized).
*/
struct QueryBuffer {
/**
* For fixed-sized attributes, it contains the fixed-sized values.
* For var-sized attributes, it contains the offsets.
* For fixed-sized attributes/dimensions, it contains the fixed-sized values.
* For var-sized attributes/dimensions, it contains the offsets.
* var buffer is nullptr.
*/
void* fixed_;
/** Size of fixed buffer. */
uint64_t fixed_size_;
/**
* For fixed-sized attributes, it is `nullptr`.
* For var-sized attributes, it contains the var-sized values.
* For fixed-sized attributes/dimensions, it is `nullptr`.
* For var-sized attributes/dimensions, it contains the var-sized values.
*/
void* var_;
/** Size of var buffer. */
uint64_t var_size_;
};
/** Map attribute_name -> AttrBuffer */
typedef std::map<std::string, AttrBuffer> AttrBuffers;
/** Map attribute/dimension name -> QueryBuffer */
typedef std::map<std::string, QueryBuffer> QueryBuffers;

/**
* Checks that the input partitioner produces the input partitions
Expand Down Expand Up @@ -297,13 +304,13 @@ int set_attribute_compression_filter(
* @param ctx The TileDB context.
* @param array_name The array name.
* @param layout The layout to write into.
* @param attr_buffers The attribute buffers to be written.
* @param buffers The attribute/dimension buffers to be written.
*/
void write_array(
tiledb_ctx_t* ctx,
const std::string& array_name,
tiledb_layout_t layout,
const AttrBuffers& attr_buffers);
const QueryBuffers& buffers);

/**
* Performs a single read to an array.
Expand All @@ -313,14 +320,18 @@ void write_array(
* @param array The input array.
* @param The subarray ranges.
* @param layout The query layout.
* @param attr_buffers The attribute buffers to be read.
* @param buffers The attribute/dimension buffers to be read.
*/
template <class T>
void read_array(
tiledb_ctx_t* ctx,
tiledb_array_t* array,
const SubarrayRanges<T>& ranges,
tiledb_layout_t layout,
const AttrBuffers& attr_buffers);
const QueryBuffers& buffers);

} // End of namespace test

} // End of namespace tiledb

#endif
21 changes: 11 additions & 10 deletions test/src/unit-CellSlabIter.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
#include <iostream>

using namespace tiledb::sm;
using namespace tiledb::test;

/* ********************************* */
/* STRUCT DEFINITION */
Expand Down Expand Up @@ -151,8 +152,8 @@ TEST_CASE_METHOD(
{"a", "b"},
{TILEDB_INT32, TILEDB_INT32},
{1, TILEDB_VAR_NUM},
{::Compressor(TILEDB_FILTER_LZ4, -1),
::Compressor(TILEDB_FILTER_LZ4, -1)},
{tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1),
tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1)},
TILEDB_ROW_MAJOR,
TILEDB_ROW_MAJOR,
2);
Expand Down Expand Up @@ -200,8 +201,8 @@ TEST_CASE_METHOD(
{"a", "b"},
{TILEDB_INT32, TILEDB_INT32},
{1, TILEDB_VAR_NUM},
{::Compressor(TILEDB_FILTER_LZ4, -1),
::Compressor(TILEDB_FILTER_LZ4, -1)},
{tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1),
tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1)},
TILEDB_ROW_MAJOR,
TILEDB_ROW_MAJOR,
2);
Expand Down Expand Up @@ -254,8 +255,8 @@ TEST_CASE_METHOD(
{"a", "b"},
{TILEDB_INT32, TILEDB_INT32},
{1, TILEDB_VAR_NUM},
{::Compressor(TILEDB_FILTER_LZ4, -1),
::Compressor(TILEDB_FILTER_LZ4, -1)},
{tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1),
tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1)},
TILEDB_ROW_MAJOR,
TILEDB_ROW_MAJOR,
2);
Expand Down Expand Up @@ -306,8 +307,8 @@ TEST_CASE_METHOD(
{"a", "b"},
{TILEDB_INT32, TILEDB_INT32},
{1, TILEDB_VAR_NUM},
{::Compressor(TILEDB_FILTER_LZ4, -1),
::Compressor(TILEDB_FILTER_LZ4, -1)},
{tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1),
tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1)},
TILEDB_ROW_MAJOR,
TILEDB_ROW_MAJOR,
2);
Expand Down Expand Up @@ -355,8 +356,8 @@ TEST_CASE_METHOD(
{"a", "b"},
{TILEDB_INT32, TILEDB_INT32},
{1, TILEDB_VAR_NUM},
{::Compressor(TILEDB_FILTER_LZ4, -1),
::Compressor(TILEDB_FILTER_LZ4, -1)},
{tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1),
tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1)},
TILEDB_ROW_MAJOR,
TILEDB_ROW_MAJOR,
2);
Expand Down
33 changes: 17 additions & 16 deletions test/src/unit-ReadCellSlabIter.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
#include <iostream>

using namespace tiledb::sm;
using namespace tiledb::test;

/* ********************************* */
/* STRUCT DEFINITION */
Expand Down Expand Up @@ -199,8 +200,8 @@ TEST_CASE_METHOD(
{"a", "b"},
{TILEDB_INT32, TILEDB_INT32},
{1, TILEDB_VAR_NUM},
{::Compressor(TILEDB_FILTER_LZ4, -1),
::Compressor(TILEDB_FILTER_LZ4, -1)},
{tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1),
tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1)},
TILEDB_ROW_MAJOR,
TILEDB_ROW_MAJOR,
2);
Expand Down Expand Up @@ -257,8 +258,8 @@ TEST_CASE_METHOD(
{"a", "b"},
{TILEDB_INT32, TILEDB_INT32},
{1, TILEDB_VAR_NUM},
{::Compressor(TILEDB_FILTER_LZ4, -1),
::Compressor(TILEDB_FILTER_LZ4, -1)},
{tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1),
tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1)},
TILEDB_ROW_MAJOR,
TILEDB_ROW_MAJOR,
2);
Expand Down Expand Up @@ -315,8 +316,8 @@ TEST_CASE_METHOD(
{"a", "b"},
{TILEDB_INT32, TILEDB_INT32},
{1, TILEDB_VAR_NUM},
{::Compressor(TILEDB_FILTER_LZ4, -1),
::Compressor(TILEDB_FILTER_LZ4, -1)},
{tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1),
tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1)},
TILEDB_ROW_MAJOR,
TILEDB_ROW_MAJOR,
2);
Expand Down Expand Up @@ -379,8 +380,8 @@ TEST_CASE_METHOD(
{"a", "b"},
{TILEDB_INT32, TILEDB_INT32},
{1, TILEDB_VAR_NUM},
{::Compressor(TILEDB_FILTER_LZ4, -1),
::Compressor(TILEDB_FILTER_LZ4, -1)},
{tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1),
tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1)},
TILEDB_ROW_MAJOR,
TILEDB_ROW_MAJOR,
2);
Expand Down Expand Up @@ -580,8 +581,8 @@ TEST_CASE_METHOD(
{"a", "b"},
{TILEDB_INT32, TILEDB_INT32},
{1, TILEDB_VAR_NUM},
{::Compressor(TILEDB_FILTER_LZ4, -1),
::Compressor(TILEDB_FILTER_LZ4, -1)},
{tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1),
tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1)},
tile_order,
cell_order,
2);
Expand Down Expand Up @@ -749,8 +750,8 @@ TEST_CASE_METHOD(
{"a", "b"},
{TILEDB_INT32, TILEDB_INT32},
{1, TILEDB_VAR_NUM},
{::Compressor(TILEDB_FILTER_LZ4, -1),
::Compressor(TILEDB_FILTER_LZ4, -1)},
{tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1),
tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1)},
tile_order,
cell_order,
2);
Expand Down Expand Up @@ -931,8 +932,8 @@ TEST_CASE_METHOD(
{"a", "b"},
{TILEDB_INT32, TILEDB_INT32},
{1, TILEDB_VAR_NUM},
{::Compressor(TILEDB_FILTER_LZ4, -1),
::Compressor(TILEDB_FILTER_LZ4, -1)},
{tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1),
tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1)},
tile_order,
cell_order,
2);
Expand Down Expand Up @@ -1158,8 +1159,8 @@ TEST_CASE_METHOD(
{"a", "b"},
{TILEDB_INT32, TILEDB_INT32},
{1, TILEDB_VAR_NUM},
{::Compressor(TILEDB_FILTER_LZ4, -1),
::Compressor(TILEDB_FILTER_LZ4, -1)},
{tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1),
tiledb::test::Compressor(TILEDB_FILTER_LZ4, -1)},
tile_order,
cell_order,
2);
Expand Down
1 change: 1 addition & 0 deletions test/src/unit-Reader.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
#include <iostream>

using namespace tiledb::sm;
using namespace tiledb::test;

/* ********************************* */
/* STRUCT DEFINITION */
Expand Down
Loading