Skip to content

Commit

Permalink
User can now set coordinate buffers separately for each dimension to …
Browse files Browse the repository at this point in the history 
…write queries.
  • Loading branch information
@stavrospapadopoulos
stavrospapadopoulos committed Dec 13, 2019
1 parent e217587 commit 01f29f5
Show file tree
Hide file tree
Showing 35 changed files with 1,258 additions and 555 deletions.
1 change: 1 addition & 0 deletions HISTORY.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
## Improvements

* Added support for indicating zero-value metadata by returning `value_num` == 1 from the `_get_metadatata` and `Array::get_metadata` APIs [#1438](https://github.com/TileDB-Inc/TileDB/pull/1438) (this is a non-breaking change, as the documented return of `value == nullptr` to indicate missing keys does not change)`
* User can set coordinate buffers separately for write queries.

## Deprecations

Expand Down
71 changes: 57 additions & 14 deletions test/src/helpers.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,9 @@
#include "helpers.h"
#include "catch.hpp"

namespace tiledb {
namespace test {

template <class T>
void check_partitions(
tiledb::sm::SubarrayPartitioner& partitioner,
Expand Down Expand Up @@ -449,7 +452,7 @@ void write_array(
tiledb_ctx_t* ctx,
const std::string& array_name,
tiledb_layout_t layout,
const AttrBuffers& attr_buffers) {
const QueryBuffers& buffers) {
// Open array
tiledb_array_t* array;
int rc = tiledb_array_alloc(ctx, array_name.c_str(), &array);
Expand All @@ -465,7 +468,7 @@ void write_array(
CHECK(rc == TILEDB_OK);

// Set buffers
for (const auto& b : attr_buffers) {
for (const auto& b : buffers) {
if (b.second.var_ == nullptr) { // Fixed-sized
rc = tiledb_query_set_buffer(
ctx,
Expand Down Expand Up @@ -510,7 +513,7 @@ void read_array(
tiledb_array_t* array,
const SubarrayRanges<T>& ranges,
tiledb_layout_t layout,
const AttrBuffers& attr_buffers) {
const QueryBuffers& buffers) {
// Create query
tiledb_query_t* query;
int rc = tiledb_query_alloc(ctx, array, TILEDB_READ, &query);
Expand All @@ -529,7 +532,7 @@ void read_array(
}

// Set buffers
for (const auto& b : attr_buffers) {
for (const auto& b : buffers) {
if (b.second.var_ == nullptr) { // Fixed-sized
rc = tiledb_query_set_buffer(
ctx,
Expand Down Expand Up @@ -569,22 +572,31 @@ void read_array(

template void check_subarray<int8_t>(
tiledb::sm::Subarray& subarray, const SubarrayRanges<int8_t>& ranges);

template void check_subarray<uint8_t>(
tiledb::sm::Subarray& subarray, const SubarrayRanges<uint8_t>& ranges);

template void check_subarray<int16_t>(
tiledb::sm::Subarray& subarray, const SubarrayRanges<int16_t>& ranges);

template void check_subarray<uint16_t>(
tiledb::sm::Subarray& subarray, const SubarrayRanges<uint16_t>& ranges);

template void check_subarray<int32_t>(
tiledb::sm::Subarray& subarray, const SubarrayRanges<int32_t>& ranges);

template void check_subarray<uint32_t>(
tiledb::sm::Subarray& subarray, const SubarrayRanges<uint32_t>& ranges);

template void check_subarray<int64_t>(
tiledb::sm::Subarray& subarray, const SubarrayRanges<int64_t>& ranges);

template void check_subarray<uint64_t>(
tiledb::sm::Subarray& subarray, const SubarrayRanges<uint64_t>& ranges);

template void check_subarray<float>(
tiledb::sm::Subarray& subarray, const SubarrayRanges<float>& ranges);

template void check_subarray<double>(
tiledb::sm::Subarray& subarray, const SubarrayRanges<double>& ranges);

Expand All @@ -593,46 +605,55 @@ template void create_subarray<int8_t>(
const SubarrayRanges<int8_t>& ranges,
tiledb::sm::Layout layout,
tiledb::sm::Subarray* subarray);

template void create_subarray<uint8_t>(
tiledb::sm::Array* array,
const SubarrayRanges<uint8_t>& ranges,
tiledb::sm::Layout layout,
tiledb::sm::Subarray* subarray);

template void create_subarray<int16_t>(
tiledb::sm::Array* array,
const SubarrayRanges<int16_t>& ranges,
tiledb::sm::Layout layout,
tiledb::sm::Subarray* subarray);

template void create_subarray<uint16_t>(
tiledb::sm::Array* array,
const SubarrayRanges<uint16_t>& ranges,
tiledb::sm::Layout layout,
tiledb::sm::Subarray* subarray);

template void create_subarray<int32_t>(
tiledb::sm::Array* array,
const SubarrayRanges<int32_t>& ranges,
tiledb::sm::Layout layout,
tiledb::sm::Subarray* subarray);

template void create_subarray<uint32_t>(
tiledb::sm::Array* array,
const SubarrayRanges<uint32_t>& ranges,
tiledb::sm::Layout layout,
tiledb::sm::Subarray* subarray);

template void create_subarray<int64_t>(
tiledb::sm::Array* array,
const SubarrayRanges<int64_t>& ranges,
tiledb::sm::Layout layout,
tiledb::sm::Subarray* subarray);

template void create_subarray<uint64_t>(
tiledb::sm::Array* array,
const SubarrayRanges<uint64_t>& ranges,
tiledb::sm::Layout layout,
tiledb::sm::Subarray* subarray);

template void create_subarray<float>(
tiledb::sm::Array* array,
const SubarrayRanges<float>& ranges,
tiledb::sm::Layout layout,
tiledb::sm::Subarray* subarray);

template void create_subarray<double>(
tiledb::sm::Array* array,
const SubarrayRanges<double>& ranges,
Expand All @@ -643,38 +664,47 @@ template void check_partitions<int8_t>(
tiledb::sm::SubarrayPartitioner& partitioner,
const std::vector<SubarrayRanges<int8_t>>& partitions,
bool last_unsplittable);

template void check_partitions<uint8_t>(
tiledb::sm::SubarrayPartitioner& partitioner,
const std::vector<SubarrayRanges<uint8_t>>& partitions,
bool last_unsplittable);

template void check_partitions<int16_t>(
tiledb::sm::SubarrayPartitioner& partitioner,
const std::vector<SubarrayRanges<int16_t>>& partitions,
bool last_unsplittable);

template void check_partitions<uint16_t>(
tiledb::sm::SubarrayPartitioner& partitioner,
const std::vector<SubarrayRanges<uint16_t>>& partitions,
bool last_unsplittable);

template void check_partitions<int32_t>(
tiledb::sm::SubarrayPartitioner& partitioner,
const std::vector<SubarrayRanges<int32_t>>& partitions,
bool last_unsplittable);

template void check_partitions<uint32_t>(
tiledb::sm::SubarrayPartitioner& partitioner,
const std::vector<SubarrayRanges<uint32_t>>& partitions,
bool last_unsplittable);

template void check_partitions<int64_t>(
tiledb::sm::SubarrayPartitioner& partitioner,
const std::vector<SubarrayRanges<int64_t>>& partitions,
bool last_unsplittable);

template void check_partitions<uint64_t>(
tiledb::sm::SubarrayPartitioner& partitioner,
const std::vector<SubarrayRanges<uint64_t>>& partitions,
bool last_unsplittable);

template void check_partitions<float>(
tiledb::sm::SubarrayPartitioner& partitioner,
const std::vector<SubarrayRanges<float>>& partitions,
bool last_unsplittable);

template void check_partitions<double>(
tiledb::sm::SubarrayPartitioner& partitioner,
const std::vector<SubarrayRanges<double>>& partitions,
Expand All @@ -685,58 +715,71 @@ template void read_array<int8_t>(
tiledb_array_t* array,
const SubarrayRanges<int8_t>& ranges,
tiledb_layout_t layout,
const AttrBuffers& attr_buffers);
const QueryBuffers& buffers);

template void read_array<uint8_t>(
tiledb_ctx_t* ctx,
tiledb_array_t* array,
const SubarrayRanges<uint8_t>& ranges,
tiledb_layout_t layout,
const AttrBuffers& attr_buffers);
const QueryBuffers& buffers);

template void read_array<int16_t>(
tiledb_ctx_t* ctx,
tiledb_array_t* array,
const SubarrayRanges<int16_t>& ranges,
tiledb_layout_t layout,
const AttrBuffers& attr_buffers);
const QueryBuffers& buffers);

template void read_array<uint16_t>(
tiledb_ctx_t* ctx,
tiledb_array_t* array,
const SubarrayRanges<uint16_t>& ranges,
tiledb_layout_t layout,
const AttrBuffers& attr_buffers);
const QueryBuffers& buffers);

template void read_array<int32_t>(
tiledb_ctx_t* ctx,
tiledb_array_t* array,
const SubarrayRanges<int32_t>& ranges,
tiledb_layout_t layout,
const AttrBuffers& attr_buffers);
const QueryBuffers& buffers);

template void read_array<uint32_t>(
tiledb_ctx_t* ctx,
tiledb_array_t* array,
const SubarrayRanges<uint32_t>& ranges,
tiledb_layout_t layout,
const AttrBuffers& attr_buffers);
const QueryBuffers& buffers);

template void read_array<int64_t>(
tiledb_ctx_t* ctx,
tiledb_array_t* array,
const SubarrayRanges<int64_t>& ranges,
tiledb_layout_t layout,
const AttrBuffers& attr_buffers);
const QueryBuffers& buffers);

template void read_array<uint64_t>(
tiledb_ctx_t* ctx,
tiledb_array_t* array,
const SubarrayRanges<uint64_t>& ranges,
tiledb_layout_t layout,
const AttrBuffers& attr_buffers);
const QueryBuffers& buffers);

template void read_array<float>(
tiledb_ctx_t* ctx,
tiledb_array_t* array,
const SubarrayRanges<float>& ranges,
tiledb_layout_t layout,
const AttrBuffers& attr_buffers);
const QueryBuffers& buffers);

template void read_array<double>(
tiledb_ctx_t* ctx,
tiledb_array_t* array,
const SubarrayRanges<double>& ranges,
tiledb_layout_t layout,
const AttrBuffers& attr_buffers);
const QueryBuffers& buffers);

} // End of namespace test

} // End of namespace tiledb
35 changes: 23 additions & 12 deletions test/src/helpers.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,31 +42,38 @@
#include <string>
#include <thread>

namespace tiledb {

namespace test {

// For easy reference
typedef std::pair<tiledb_filter_type_t, int> Compressor;
template <class T>
using SubarrayRanges = std::vector<std::vector<T>>;

/** Helper struct for the buffers of an attribute (fixed- or var-sized). */
struct AttrBuffer {
/**
* Helper struct for the buffers of an attribute/dimension
* (fixed- or var-sized).
*/
struct QueryBuffer {
/**
* For fixed-sized attributes, it contains the fixed-sized values.
* For var-sized attributes, it contains the offsets.
* For fixed-sized attributes/dimensions, it contains the fixed-sized values.
* For var-sized attributes/dimensions, it contains the offsets.
* var buffer is nullptr.
*/
void* fixed_;
/** Size of fixed buffer. */
uint64_t fixed_size_;
/**
* For fixed-sized attributes, it is `nullptr`.
* For var-sized attributes, it contains the var-sized values.
* For fixed-sized attributes/dimensions, it is `nullptr`.
* For var-sized attributes/dimensions, it contains the var-sized values.
*/
void* var_;
/** Size of var buffer. */
uint64_t var_size_;
};
/** Map attribute_name -> AttrBuffer */
typedef std::map<std::string, AttrBuffer> AttrBuffers;
/** Map attribute/dimension name -> QueryBuffer */
typedef std::map<std::string, QueryBuffer> QueryBuffers;

/**
* Checks that the input partitioner produces the input partitions
Expand Down Expand Up @@ -297,13 +304,13 @@ int set_attribute_compression_filter(
* @param ctx The TileDB context.
* @param array_name The array name.
* @param layout The layout to write into.
* @param attr_buffers The attribute buffers to be written.
* @param buffers The attribute/dimension buffers to be written.
*/
void write_array(
tiledb_ctx_t* ctx,
const std::string& array_name,
tiledb_layout_t layout,
const AttrBuffers& attr_buffers);
const QueryBuffers& buffers);

/**
* Performs a single read to an array.
Expand All @@ -313,14 +320,18 @@ void write_array(
* @param array The input array.
* @param The subarray ranges.
* @param layout The query layout.
* @param attr_buffers The attribute buffers to be read.
* @param buffers The attribute/dimension buffers to be read.
*/
template <class T>
void read_array(
tiledb_ctx_t* ctx,
tiledb_array_t* array,
const SubarrayRanges<T>& ranges,
tiledb_layout_t layout,
const AttrBuffers& attr_buffers);
const QueryBuffers& buffers);

} // End of namespace test

} // End of namespace tiledb

#endif
Loading

0 comments on commit 01f29f5

Please sign in to comment.