Skip to content

Commit

Permalink
Towards addressing #93. Split writer algorithms such that the dimensi…
Browse files Browse the repository at this point in the history 
…on coordinates are split in different buffers and handled separately.
  • Loading branch information
@stavrospapadopoulos
stavrospapadopoulos committed Dec 9, 2019
1 parent 452e08b commit ef02c45
Show file tree
Hide file tree
Showing 15 changed files with 2,034 additions and 750 deletions.
2 changes: 1 addition & 1 deletion test/src/unit-capi-dense_array.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4160,7 +4160,7 @@ TEST_CASE_METHOD(
TEST_CASE_METHOD(
DenseArrayFx,
"C API: Test dense array, check if coords exist in unordered writes",
"[capi], [dense], [dense-coords-exist-unordered]") {
"[capi][dense][coords-exist-unordered]") {
SECTION("- No serialization") {
serialize_query_ = false;
}
Expand Down
21 changes: 9 additions & 12 deletions test/src/unit-cppapi-array.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -388,7 +388,7 @@ TEST_CASE(
vfs.remove_dir(array_name_1d);
}

TEST_CASE("C++ API: Read subarray with expanded domain", "[cppapi], [dense]") {
TEST_CASE("C++ API: Read subarray with expanded domain", "[cppapi][dense]") {
const std::vector<tiledb_layout_t> tile_layouts = {TILEDB_ROW_MAJOR,
TILEDB_COL_MAJOR},
cell_layouts = {TILEDB_ROW_MAJOR,
Expand Down Expand Up @@ -455,8 +455,7 @@ TEST_CASE("C++ API: Read subarray with expanded domain", "[cppapi], [dense]") {
}
}

TEST_CASE(
"C++ API: Consolidation of empty arrays", "[cppapi], [consolidation]") {
TEST_CASE("C++ API: Consolidation of empty arrays", "[cppapi][consolidation]") {
Context ctx;
VFS vfs(ctx);
const std::string array_name = "cpp_unit_array";
Expand Down Expand Up @@ -530,7 +529,7 @@ TEST_CASE(
vfs.remove_dir(array_name);
}

TEST_CASE("C++ API: Encrypted array", "[cppapi], [encryption]") {
TEST_CASE("C++ API: Encrypted array", "[cppapi][encryption]") {
Context ctx;
VFS vfs(ctx);
const std::string array_name = "cpp_unit_array";
Expand Down Expand Up @@ -625,9 +624,7 @@ TEST_CASE("C++ API: Encrypted array", "[cppapi], [encryption]") {
vfs.remove_dir(array_name);
}

TEST_CASE(
"C++ API: Encrypted array, std::string key",
"[cppapi][encryption][cppapi-encryption]") {
TEST_CASE("C++ API: Encrypted array, std::string key", "[cppapi][encryption]") {
Context ctx;
VFS vfs(ctx);
const std::string array_name = "cpp_unit_array";
Expand Down Expand Up @@ -711,7 +708,7 @@ TEST_CASE(

TEST_CASE(
"C++ API: Open array with anonymous attribute",
"[cppapi], [cppapi-open-array-anon-attr]") {
"[cppapi][open-array-anon-attr]") {
Context ctx;
VFS vfs(ctx);
const std::string array_name = "cppapi_open_array_anon_attr";
Expand All @@ -736,7 +733,7 @@ TEST_CASE(
vfs.remove_dir(array_name);
}

TEST_CASE("C++ API: Open array at", "[cppapi], [cppapi-open-array-at]") {
TEST_CASE("C++ API: Open array at", "[cppapi][open-array-at]") {
Context ctx;
VFS vfs(ctx);
const std::string array_name = "cppapi_open_array_at";
Expand Down Expand Up @@ -840,8 +837,7 @@ TEST_CASE("C++ API: Open array at", "[cppapi], [cppapi-open-array-at]") {
}

TEST_CASE(
"C++ API: Open encrypted array at",
"[cppapi], [cppapi-open-encrypted-array-at]") {
"C++ API: Open encrypted array at", "[cppapi][open-encrypted-array-at]") {
const char key[] = "0123456789abcdeF0123456789abcdeF";
uint32_t key_len = (uint32_t)strlen(key);

Expand Down Expand Up @@ -943,7 +939,8 @@ TEST_CASE(
}

TEST_CASE(
"C++ API: Writing single cell with global order", "[cppapi], [sparse]") {
"C++ API: Writing single cell with global order",
"[cppapi][sparse][global]") {
const std::string array_name = "cpp_unit_array";
Context ctx;
VFS vfs(ctx);
Expand Down
142 changes: 142 additions & 0 deletions tiledb/sm/array_schema/dimension.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@

#include <cassert>
#include <iostream>
#include <sstream>

namespace tiledb {
namespace sm {
Expand All @@ -48,20 +49,23 @@ Dimension::Dimension() {
domain_ = nullptr;
tile_extent_ = nullptr;
type_ = Datatype::INT32;
set_oob_func();
}

Dimension::Dimension(const std::string& name, Datatype type)
: name_(name)
, type_(type) {
domain_ = nullptr;
tile_extent_ = nullptr;
set_oob_func();
}

Dimension::Dimension(const Dimension* dim) {
assert(dim != nullptr);

name_ = dim->name();
type_ = dim->type_;
oob_func_ = dim->oob_func_;
uint64_t type_size = datatype_size(type_);
domain_ = std::malloc(2 * type_size);
std::memcpy(domain_, dim->domain(), 2 * type_size);
Expand All @@ -85,6 +89,67 @@ Dimension::~Dimension() {
/* API */
/* ********************************* */

uint64_t Dimension::coord_size() const {
return datatype_size(type_);
}

std::string Dimension::coord_to_str(const void* coord) const {
std::stringstream ss;
assert(coord != nullptr);

switch (type_) {
case Datatype::INT32:
ss << *((int32_t*)coord);
break;
case Datatype::INT64:
ss << *((int64_t*)coord);
break;
case Datatype::INT8:
ss << *((int8_t*)coord);
break;
case Datatype::UINT8:
ss << *((uint8_t*)coord);
break;
case Datatype::INT16:
ss << *((int16_t*)coord);
break;
case Datatype::UINT16:
ss << *((uint16_t*)coord);
break;
case Datatype::UINT32:
ss << *((uint32_t*)coord);
break;
case Datatype::UINT64:
ss << *((uint64_t*)coord);
break;
case Datatype::FLOAT32:
ss << *((float*)coord);
break;
case Datatype::FLOAT64:
ss << *((double*)coord);
break;
case Datatype::DATETIME_YEAR:
case Datatype::DATETIME_MONTH:
case Datatype::DATETIME_WEEK:
case Datatype::DATETIME_DAY:
case Datatype::DATETIME_HR:
case Datatype::DATETIME_MIN:
case Datatype::DATETIME_SEC:
case Datatype::DATETIME_MS:
case Datatype::DATETIME_US:
case Datatype::DATETIME_NS:
case Datatype::DATETIME_PS:
case Datatype::DATETIME_FS:
case Datatype::DATETIME_AS:
ss << *((int64_t*)coord);
break;
default:
break;
}

return ss.str();
}

// ===== FORMAT =====
// dimension_name_size (uint32_t)
// dimension_name (string)
Expand Down Expand Up @@ -124,6 +189,8 @@ Status Dimension::deserialize(ConstBuffer* buff, Datatype type) {
RETURN_NOT_OK(buff->read(tile_extent_, datatype_size(type_)));
}

set_oob_func();

return Status::Ok();
}

Expand Down Expand Up @@ -153,6 +220,27 @@ bool Dimension::is_anonymous() const {
utils::parse::starts_with(name_, constants::default_dim_name);
}

template <class T>
bool Dimension::oob(
const Dimension* dim, const void* coord, std::string* err_msg) {
auto domain = (const T*)dim->domain();
auto coord_t = (const T*)coord;
if (*coord_t < domain[0] || *coord_t > domain[1]) {
std::stringstream ss;
ss << "Coordinate " << *coord_t << " is out of domain bounds [" << domain[0]
<< ", " << domain[1] << "] on dimension '" << dim->name() << "'";
*err_msg = ss.str();
return true;
}

return false;
}

bool Dimension::oob(const void* coord, std::string* err_msg) const {
assert(oob_func_ != nullptr);
return oob_func_(this, coord, err_msg);
}

// ===== FORMAT =====
// dimension_name_size (uint32_t)
// dimension_name (string)
Expand Down Expand Up @@ -478,6 +566,60 @@ Status Dimension::check_tile_extent() const {
return Status::Ok();
}

void Dimension::set_oob_func() {
// Set
switch (type_) {
case Datatype::INT32:
oob_func_ = oob<int32_t>;
break;
case Datatype::INT64:
oob_func_ = oob<int64_t>;
break;
case Datatype::INT8:
oob_func_ = oob<int8_t>;
break;
case Datatype::UINT8:
oob_func_ = oob<uint8_t>;
break;
case Datatype::INT16:
oob_func_ = oob<int16_t>;
break;
case Datatype::UINT16:
oob_func_ = oob<uint16_t>;
break;
case Datatype::UINT32:
oob_func_ = oob<uint32_t>;
break;
case Datatype::UINT64:
oob_func_ = oob<uint64_t>;
break;
case Datatype::FLOAT32:
oob_func_ = oob<float>;
break;
case Datatype::FLOAT64:
oob_func_ = oob<double>;
break;
case Datatype::DATETIME_YEAR:
case Datatype::DATETIME_MONTH:
case Datatype::DATETIME_WEEK:
case Datatype::DATETIME_DAY:
case Datatype::DATETIME_HR:
case Datatype::DATETIME_MIN:
case Datatype::DATETIME_SEC:
case Datatype::DATETIME_MS:
case Datatype::DATETIME_US:
case Datatype::DATETIME_NS:
case Datatype::DATETIME_PS:
case Datatype::DATETIME_FS:
case Datatype::DATETIME_AS:
oob_func_ = oob<int64_t>;
break;
default:
oob_func_ = nullptr;
break;
}
}

} // namespace sm

} // namespace tiledb
43 changes: 43 additions & 0 deletions tiledb/sm/array_schema/dimension.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,12 @@ class Dimension {
/* API */
/* ********************************* */

/** Returns the size (in bytes) of a coordinate in this dimension. */
uint64_t coord_size() const;

/** Returns the input coordinate in string format. */
std::string coord_to_str(const void* coord) const;

/**
* Populates the object members from the data in the input binary buffer.
*
Expand All @@ -97,6 +103,33 @@ class Dimension {
/** Returns true if this is an anonymous (unlabled) dimension **/
bool is_anonymous() const;

/**
* Returns true if the input coordinate is out-of-bounds with respect
* to the dimension domain.
*
* @param coord The coordinate to be checked. It will properly be
* type-cast to the dimension datatype.
* @param err_msg An error message to be retrieved in case the function
* returns true.
* @return True if the input coordinates is out-of-bounds.
*/
bool oob(const void* coord, std::string* err_msg) const;

/**
* Returns true if the input coordinate is out-of-bounds with respect
* to the dimension domain.
*
* @param dim The dimension to apply the oob check on.
* @param coord The coordinate to be checked. It will properly be
* type-cast to the dimension datatype.
* @param err_msg An error message to be retrieved in case the function
* returns true.
* @return True if the input coordinates is out-of-bounds.
*/
template <class T>
static bool oob(
const Dimension* dim, const void* coord, std::string* err_msg);

/**
* Serializes the object members into a binary buffer.
*
Expand Down Expand Up @@ -149,6 +182,13 @@ class Dimension {
/** The dimension type. */
Datatype type_;

/**
* Stores the appropriate templated oob() function based on the
* dimension datatype.
*/
std::function<bool(const Dimension* dim, const void*, std::string*)>
oob_func_;

/* ********************************* */
/* PRIVATE METHODS */
/* ********************************* */
Expand Down Expand Up @@ -229,6 +269,9 @@ class Dimension {
*/
template <class T>
Status check_tile_extent() const;

/** Sets the templated oob function. */
void set_oob_func();
};

} // namespace sm
Expand Down
Loading

0 comments on commit ef02c45

Please sign in to comment.