Skip to content

Commit

Permalink
iterating on unit-test cases
Browse files Browse the repository at this point in the history
  • Loading branch information
@johnkerl
johnkerl committed Aug 19, 2024
1 parent 1a9660f commit daecd1b
Show file tree
Hide file tree
Showing 4 changed files with 91 additions and 13 deletions.
22 changes: 18 additions & 4 deletions libtiledbsoma/src/utils/arrow_adapter.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -654,6 +654,8 @@ ArraySchema ArrowAdapter::tiledb_schema_from_arrow_schema(

std::map<std::string, Dimension> dims;

bool use_current_domain = true;

for (int64_t sch_idx = 0; sch_idx < arrow_schema->n_children; ++sch_idx) {
auto child = arrow_schema->children[sch_idx];
auto type = ArrowAdapter::to_tiledb_format(child->format);
Expand All @@ -664,7 +666,9 @@ ArraySchema ArrowAdapter::tiledb_schema_from_arrow_schema(
bool isattr = true;

for (int64_t i = 0; i < index_column_schema->n_children; ++i) {
auto col_name = index_column_schema->children[i]->name;
auto achild = index_column_array->children[i];
auto schild = index_column_schema->children[i];
auto col_name = schild->name;
if (strcmp(child->name, col_name) == 0) {
if (ArrowAdapter::_isvar(child->format)) {
type = TILEDB_STRING_ASCII;
Expand All @@ -673,7 +677,18 @@ ArraySchema ArrowAdapter::tiledb_schema_from_arrow_schema(
FilterList filter_list = ArrowAdapter::_create_dim_filter_list(
child->name, platform_config, soma_type, ctx);

const void* buff = index_column_array->children[i]->buffers[1];
if (achild->length == 3) {
use_current_domain = false;
} else if (achild->length == 5) {
// This is fine
} else {
throw TileDBSOMAError(fmt::format(
"ArrowAdapter: unexpected length {} for name {}",
achild->length,
col_name));
}

const void* buff = achild->buffers[1];
auto dim = ArrowAdapter::_create_dim(
type, child->name, buff, ctx);
dim.set_filter_list(filter_list);
Expand Down Expand Up @@ -737,8 +752,7 @@ ArraySchema ArrowAdapter::tiledb_schema_from_arrow_schema(

// Note: this must be done after we've got the core domain, since the
// NDRectangle constructor requires access to the core domain.
bool have_current_domain_info = index_column_array->length == 5;
if (have_current_domain_info) {
if (use_current_domain) {
CurrentDomain current_domain(*ctx);
NDRectangle ndrect(*ctx, domain);

Expand Down
32 changes: 28 additions & 4 deletions libtiledbsoma/test/common.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,12 +69,14 @@ std::pair<std::unique_ptr<ArrowSchema>, ArrowTable> create_arrow_schema(
arrow_schema->dictionary = nullptr;
arrow_schema->release = &ArrowAdapter::release_schema;
arrow_schema->children = new ArrowSchema*[arrow_schema->n_children];

ArrowSchema* dim = arrow_schema->children[0] = new ArrowSchema;
dim->format = "l";
dim->name = "d0";
dim->n_children = 0;
dim->dictionary = nullptr;
dim->release = &ArrowAdapter::release_schema;

ArrowSchema* attr = arrow_schema->children[1] = new ArrowSchema;
attr->format = "l";
attr->name = "a0";
Expand All @@ -90,6 +92,7 @@ std::pair<std::unique_ptr<ArrowSchema>, ArrowTable> create_arrow_schema(
col_info_schema->dictionary = nullptr;
col_info_schema->release = &ArrowAdapter::release_schema;
col_info_schema->children = new ArrowSchema*[col_info_schema->n_children];

dim = col_info_schema->children[0] = new ArrowSchema;
dim->format = "l";
dim->name = "d0";
Expand All @@ -107,7 +110,9 @@ std::pair<std::unique_ptr<ArrowSchema>, ArrowTable> create_arrow_schema(
col_info_array->n_children = 2;
col_info_array->release = &ArrowAdapter::release_array;
col_info_array->children = new ArrowArray*[1];

int n = use_current_domain ? 5 : 3;

auto d0_info = col_info_array->children[0] = new ArrowArray;
d0_info->length = n;
d0_info->null_count = 0;
Expand All @@ -118,8 +123,16 @@ std::pair<std::unique_ptr<ArrowSchema>, ArrowTable> create_arrow_schema(
d0_info->buffers[0] = nullptr;
d0_info->buffers[1] = malloc(sizeof(int64_t) * n);
d0_info->n_children = 0;
int64_t dom[] = {0, dim_max, 1, 0, CORE_DOMAIN_MAX};
std::memcpy((void*)d0_info->buffers[1], &dom, sizeof(int64_t) * n);

if (use_current_domain) {
// domain big; current_domain small
int64_t dom[] = {0, CORE_DOMAIN_MAX, 1, 0, dim_max};
std::memcpy((void*)d0_info->buffers[1], &dom, sizeof(int64_t) * n);
} else {
// domain small; current_domain feature not being used
int64_t dom[] = {0, dim_max, 1};
std::memcpy((void*)d0_info->buffers[1], &dom, sizeof(int64_t) * n);
}

return std::pair(
std::move(arrow_schema),
Expand All @@ -134,6 +147,7 @@ ArrowTable create_column_index_info(int64_t dim_max, bool use_current_domain) {
col_info_schema->dictionary = nullptr;
col_info_schema->release = &ArrowAdapter::release_schema;
col_info_schema->children = new ArrowSchema*[col_info_schema->n_children];

ArrowSchema* dim = col_info_schema->children[0] = new ArrowSchema;
dim->format = "l";
dim->name = "soma_dim_0";
Expand All @@ -151,7 +165,9 @@ ArrowTable create_column_index_info(int64_t dim_max, bool use_current_domain) {
col_info_array->n_children = 2;
col_info_array->release = &ArrowAdapter::release_array;
col_info_array->children = new ArrowArray*[1];

int n = use_current_domain ? 5 : 3;

auto d0_info = col_info_array->children[0] = new ArrowArray;
d0_info->length = n;
d0_info->null_count = 0;
Expand All @@ -162,8 +178,16 @@ ArrowTable create_column_index_info(int64_t dim_max, bool use_current_domain) {
d0_info->buffers[0] = nullptr;
d0_info->buffers[1] = malloc(sizeof(int64_t) * n);
d0_info->n_children = 0;
int64_t dom[] = {0, dim_max, 1, 0, CORE_DOMAIN_MAX};
std::memcpy((void*)d0_info->buffers[1], &dom, sizeof(int64_t) * n);

if (use_current_domain) {
// domain big; current_domain small
int64_t dom[] = {0, CORE_DOMAIN_MAX, 1, 0, dim_max};
std::memcpy((void*)d0_info->buffers[1], &dom, sizeof(int64_t) * n);
} else {
// domain small; current_domain feature not being used
int64_t dom[] = {0, dim_max, 1};
std::memcpy((void*)d0_info->buffers[1], &dom, sizeof(int64_t) * n);
}

return ArrowTable(std::move(col_info_array), std::move(col_info_schema));
}
Expand Down
47 changes: 42 additions & 5 deletions libtiledbsoma/test/unit_soma_dense_ndarray.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,21 @@ TEST_CASE("SOMADenseNDArray: basic") {

auto index_columns = helper::create_column_index_info(
dim_max, use_current_domain);
if (use_current_domain) {
// Setting a current domain on a TileDB dense array is not (yet)
// supported
REQUIRE_THROWS(SOMADenseNDArray::create(
uri,
"l",
ArrowTable(
std::move(index_columns.first),
std::move(index_columns.second)),
ctx,
PlatformConfig(),
TimestampRange(0, 2)));
continue;
}

SOMADenseNDArray::create(
uri,
"l",
Expand All @@ -70,7 +85,14 @@ TEST_CASE("SOMADenseNDArray: basic") {
REQUIRE(schema->array_type() == TILEDB_DENSE);
REQUIRE(schema->domain().has_dimension("soma_dim_0"));
REQUIRE(soma_dense->ndim() == 1);
REQUIRE(soma_dense->shape() == std::vector<int64_t>{dim_max + 1});

if (use_current_domain) {
REQUIRE(soma_dense->shape() == std::vector<int64_t>{dim_max + 1});
} else {
REQUIRE(
soma_dense->maxshape() == std::vector<int64_t>{dim_max + 1});
}

soma_dense->close();

std::vector<int64_t> d0{1, 10};
Expand All @@ -97,13 +119,28 @@ TEST_CASE("SOMADenseNDArray: platform_config") {
bool use_current_domains[] = {false, true};
for (bool use_current_domain : use_current_domains) {
auto ctx = std::make_shared<SOMAContext>();
std::string uri = "mem://unit-test-dataframe-platform-config";
std::string uri = "mem://unit-test-dense-ndarray-platform-config";

PlatformConfig platform_config;
platform_config.dense_nd_array_dim_zstd_level = 6;

auto index_columns = helper::create_column_index_info(
dim_max, use_current_domain);
if (use_current_domain) {
// Setting a current domain on a TileDB dense array is not (yet)
// supported
REQUIRE_THROWS(SOMADenseNDArray::create(
uri,
"l",
ArrowTable(
std::move(index_columns.first),
std::move(index_columns.second)),
ctx,
platform_config));

continue;
}

SOMADenseNDArray::create(
uri,
"l",
Expand All @@ -113,16 +150,16 @@ TEST_CASE("SOMADenseNDArray: platform_config") {
ctx,
platform_config);

auto soma_dataframe = SOMADenseNDArray::open(uri, OpenMode::read, ctx);
auto dim_filter = soma_dataframe->tiledb_schema()
auto soma_dense = SOMADenseNDArray::open(uri, OpenMode::read, ctx);
auto dim_filter = soma_dense->tiledb_schema()
->domain()
.dimension("soma_dim_0")
.filter_list()
.filter(0);
REQUIRE(dim_filter.filter_type() == TILEDB_FILTER_ZSTD);
REQUIRE(dim_filter.get_option<int32_t>(TILEDB_COMPRESSION_LEVEL) == 6);

soma_dataframe->close();
soma_dense->close();
}
}

Expand Down
3 changes: 3 additions & 0 deletions libtiledbsoma/test/unit_soma_sparse_ndarray.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,9 @@ TEST_CASE("SOMASparseNDArray: basic") {

if (use_current_domain) {
REQUIRE(soma_sparse->shape() == std::vector<int64_t>{dim_max + 1});
} else {
REQUIRE(
soma_sparse->maxshape() == std::vector<int64_t>{dim_max + 1});
}

soma_sparse->close();
Expand Down

0 comments on commit daecd1b

Please sign in to comment.