Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Towards splitting coordinates in separate buffers and files #1454

Merged
merged 1 commit into from
Dec 10, 2019
Merged

Towards splitting coordinates in separate buffers and files #1454

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion test/src/unit-capi-dense_array.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4160,7 +4160,7 @@ TEST_CASE_METHOD(
TEST_CASE_METHOD(
DenseArrayFx,
"C API: Test dense array, check if coords exist in unordered writes",
"[capi], [dense], [dense-coords-exist-unordered]") {
"[capi][dense][coords-exist-unordered]") {
SECTION("- No serialization") {
serialize_query_ = false;
}
Expand Down
21 changes: 9 additions & 12 deletions test/src/unit-cppapi-array.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -388,7 +388,7 @@ TEST_CASE(
vfs.remove_dir(array_name_1d);
}

TEST_CASE("C++ API: Read subarray with expanded domain", "[cppapi], [dense]") {
TEST_CASE("C++ API: Read subarray with expanded domain", "[cppapi][dense]") {
const std::vector<tiledb_layout_t> tile_layouts = {TILEDB_ROW_MAJOR,
TILEDB_COL_MAJOR},
cell_layouts = {TILEDB_ROW_MAJOR,
Expand Down Expand Up @@ -455,8 +455,7 @@ TEST_CASE("C++ API: Read subarray with expanded domain", "[cppapi], [dense]") {
}
}

TEST_CASE(
"C++ API: Consolidation of empty arrays", "[cppapi], [consolidation]") {
TEST_CASE("C++ API: Consolidation of empty arrays", "[cppapi][consolidation]") {
Context ctx;
VFS vfs(ctx);
const std::string array_name = "cpp_unit_array";
Expand Down Expand Up @@ -530,7 +529,7 @@ TEST_CASE(
vfs.remove_dir(array_name);
}

TEST_CASE("C++ API: Encrypted array", "[cppapi], [encryption]") {
TEST_CASE("C++ API: Encrypted array", "[cppapi][encryption]") {
Context ctx;
VFS vfs(ctx);
const std::string array_name = "cpp_unit_array";
Expand Down Expand Up @@ -625,9 +624,7 @@ TEST_CASE("C++ API: Encrypted array", "[cppapi], [encryption]") {
vfs.remove_dir(array_name);
}

TEST_CASE(
"C++ API: Encrypted array, std::string key",
"[cppapi][encryption][cppapi-encryption]") {
TEST_CASE("C++ API: Encrypted array, std::string key", "[cppapi][encryption]") {
Context ctx;
VFS vfs(ctx);
const std::string array_name = "cpp_unit_array";
Expand Down Expand Up @@ -711,7 +708,7 @@ TEST_CASE(

TEST_CASE(
"C++ API: Open array with anonymous attribute",
"[cppapi], [cppapi-open-array-anon-attr]") {
"[cppapi][open-array-anon-attr]") {
Context ctx;
VFS vfs(ctx);
const std::string array_name = "cppapi_open_array_anon_attr";
Expand All @@ -736,7 +733,7 @@ TEST_CASE(
vfs.remove_dir(array_name);
}

TEST_CASE("C++ API: Open array at", "[cppapi], [cppapi-open-array-at]") {
TEST_CASE("C++ API: Open array at", "[cppapi][open-array-at]") {
Context ctx;
VFS vfs(ctx);
const std::string array_name = "cppapi_open_array_at";
Expand Down Expand Up @@ -840,8 +837,7 @@ TEST_CASE("C++ API: Open array at", "[cppapi], [cppapi-open-array-at]") {
}

TEST_CASE(
"C++ API: Open encrypted array at",
"[cppapi], [cppapi-open-encrypted-array-at]") {
"C++ API: Open encrypted array at", "[cppapi][open-encrypted-array-at]") {
const char key[] = "0123456789abcdeF0123456789abcdeF";
uint32_t key_len = (uint32_t)strlen(key);

Expand Down Expand Up @@ -943,7 +939,8 @@ TEST_CASE(
}

TEST_CASE(
"C++ API: Writing single cell with global order", "[cppapi], [sparse]") {
"C++ API: Writing single cell with global order",
"[cppapi][sparse][global]") {
const std::string array_name = "cpp_unit_array";
Context ctx;
VFS vfs(ctx);
Expand Down
142 changes: 142 additions & 0 deletions tiledb/sm/array_schema/dimension.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@

#include <cassert>
#include <iostream>
#include <sstream>

namespace tiledb {
namespace sm {
Expand All @@ -48,20 +49,23 @@ Dimension::Dimension() {
domain_ = nullptr;
tile_extent_ = nullptr;
type_ = Datatype::INT32;
set_oob_func();
}

Dimension::Dimension(const std::string& name, Datatype type)
: name_(name)
, type_(type) {
domain_ = nullptr;
tile_extent_ = nullptr;
set_oob_func();
}

Dimension::Dimension(const Dimension* dim) {
assert(dim != nullptr);

name_ = dim->name();
type_ = dim->type_;
oob_func_ = dim->oob_func_;
uint64_t type_size = datatype_size(type_);
domain_ = std::malloc(2 * type_size);
std::memcpy(domain_, dim->domain(), 2 * type_size);
Expand All @@ -85,6 +89,67 @@ Dimension::~Dimension() {
/* API */
/* ********************************* */

uint64_t Dimension::coord_size() const {
return datatype_size(type_);
}

std::string Dimension::coord_to_str(const void* coord) const {
std::stringstream ss;
assert(coord != nullptr);

switch (type_) {
case Datatype::INT32:
ss << *((int32_t*)coord);
break;
case Datatype::INT64:
ss << *((int64_t*)coord);
break;
case Datatype::INT8:
ss << *((int8_t*)coord);
break;
case Datatype::UINT8:
ss << *((uint8_t*)coord);
break;
case Datatype::INT16:
ss << *((int16_t*)coord);
break;
case Datatype::UINT16:
ss << *((uint16_t*)coord);
break;
case Datatype::UINT32:
ss << *((uint32_t*)coord);
break;
case Datatype::UINT64:
ss << *((uint64_t*)coord);
break;
case Datatype::FLOAT32:
ss << *((float*)coord);
break;
case Datatype::FLOAT64:
ss << *((double*)coord);
break;
case Datatype::DATETIME_YEAR:
case Datatype::DATETIME_MONTH:
case Datatype::DATETIME_WEEK:
case Datatype::DATETIME_DAY:
case Datatype::DATETIME_HR:
case Datatype::DATETIME_MIN:
case Datatype::DATETIME_SEC:
case Datatype::DATETIME_MS:
case Datatype::DATETIME_US:
case Datatype::DATETIME_NS:
case Datatype::DATETIME_PS:
case Datatype::DATETIME_FS:
case Datatype::DATETIME_AS:
ss << *((int64_t*)coord);
break;
default:
break;
}

return ss.str();
}

// ===== FORMAT =====
// dimension_name_size (uint32_t)
// dimension_name (string)
Expand Down Expand Up @@ -124,6 +189,8 @@ Status Dimension::deserialize(ConstBuffer* buff, Datatype type) {
RETURN_NOT_OK(buff->read(tile_extent_, datatype_size(type_)));
}

set_oob_func();

return Status::Ok();
}

Expand Down Expand Up @@ -153,6 +220,27 @@ bool Dimension::is_anonymous() const {
utils::parse::starts_with(name_, constants::default_dim_name);
}

template <class T>
bool Dimension::oob(
const Dimension* dim, const void* coord, std::string* err_msg) {
auto domain = (const T*)dim->domain();
auto coord_t = (const T*)coord;
if (*coord_t < domain[0] || *coord_t > domain[1]) {
std::stringstream ss;
ss << "Coordinate " << *coord_t << " is out of domain bounds [" << domain[0]
<< ", " << domain[1] << "] on dimension '" << dim->name() << "'";
*err_msg = ss.str();
return true;
}

return false;
}

bool Dimension::oob(const void* coord, std::string* err_msg) const {
assert(oob_func_ != nullptr);
return oob_func_(this, coord, err_msg);
}

// ===== FORMAT =====
// dimension_name_size (uint32_t)
// dimension_name (string)
Expand Down Expand Up @@ -478,6 +566,60 @@ Status Dimension::check_tile_extent() const {
return Status::Ok();
}

void Dimension::set_oob_func() {
// Set
switch (type_) {
case Datatype::INT32:
oob_func_ = oob<int32_t>;
break;
case Datatype::INT64:
oob_func_ = oob<int64_t>;
break;
case Datatype::INT8:
oob_func_ = oob<int8_t>;
break;
case Datatype::UINT8:
oob_func_ = oob<uint8_t>;
break;
case Datatype::INT16:
oob_func_ = oob<int16_t>;
break;
case Datatype::UINT16:
oob_func_ = oob<uint16_t>;
break;
case Datatype::UINT32:
oob_func_ = oob<uint32_t>;
break;
case Datatype::UINT64:
oob_func_ = oob<uint64_t>;
break;
case Datatype::FLOAT32:
oob_func_ = oob<float>;
break;
case Datatype::FLOAT64:
oob_func_ = oob<double>;
break;
case Datatype::DATETIME_YEAR:
case Datatype::DATETIME_MONTH:
case Datatype::DATETIME_WEEK:
case Datatype::DATETIME_DAY:
case Datatype::DATETIME_HR:
case Datatype::DATETIME_MIN:
case Datatype::DATETIME_SEC:
case Datatype::DATETIME_MS:
case Datatype::DATETIME_US:
case Datatype::DATETIME_NS:
case Datatype::DATETIME_PS:
case Datatype::DATETIME_FS:
case Datatype::DATETIME_AS:
oob_func_ = oob<int64_t>;
break;
default:
oob_func_ = nullptr;
break;
}
}

} // namespace sm

} // namespace tiledb
43 changes: 43 additions & 0 deletions tiledb/sm/array_schema/dimension.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,12 @@ class Dimension {
/* API */
/* ********************************* */

/** Returns the size (in bytes) of a coordinate in this dimension. */
uint64_t coord_size() const;

/** Returns the input coordinate in string format. */
std::string coord_to_str(const void* coord) const;

/**
* Populates the object members from the data in the input binary buffer.
*
Expand All @@ -97,6 +103,33 @@ class Dimension {
/** Returns true if this is an anonymous (unlabled) dimension **/
bool is_anonymous() const;

/**
* Returns true if the input coordinate is out-of-bounds with respect
* to the dimension domain.
*
* @param coord The coordinate to be checked. It will properly be
* type-cast to the dimension datatype.
* @param err_msg An error message to be retrieved in case the function
* returns true.
* @return True if the input coordinates is out-of-bounds.
*/
bool oob(const void* coord, std::string* err_msg) const;

/**
* Returns true if the input coordinate is out-of-bounds with respect
* to the dimension domain.
*
* @param dim The dimension to apply the oob check on.
* @param coord The coordinate to be checked. It will properly be
* type-cast to the dimension datatype.
* @param err_msg An error message to be retrieved in case the function
* returns true.
* @return True if the input coordinates is out-of-bounds.
*/
template <class T>
static bool oob(
const Dimension* dim, const void* coord, std::string* err_msg);

/**
* Serializes the object members into a binary buffer.
*
Expand Down Expand Up @@ -149,6 +182,13 @@ class Dimension {
/** The dimension type. */
Datatype type_;

/**
* Stores the appropriate templated oob() function based on the
* dimension datatype.
*/
std::function<bool(const Dimension* dim, const void*, std::string*)>
oob_func_;

/* ********************************* */
/* PRIVATE METHODS */
/* ********************************* */
Expand Down Expand Up @@ -229,6 +269,9 @@ class Dimension {
*/
template <class T>
Status check_tile_extent() const;

/** Sets the templated oob function. */
void set_oob_func();
};

} // namespace sm
Expand Down
Loading