[c++] Arrow utils with current-domain option [WIP]

single-cell-data · Aug 17, 2024 · ad3b5d1 · ad3b5d1
1 parent bdba3c3
commit ad3b5d1
Show file tree

Hide file tree

Showing 7 changed files with 78 additions and 32 deletions.
diff --git a/apis/r/tests/testthat/helper-test-data.R b/apis/r/tests/testthat/helper-test-data.R
@@ -75,6 +75,6 @@ create_arrow_table <- function(nrows = 10L, factors = FALSE) {
       soma_joinid = bit64::seq.integer64(from = 0L, to = nrows - 1L),
       bar = seq(nrows) + 0.1,
       baz = as.character(seq.int(nrows) + 1000L)
-      # schema = create_arrow_schema()
+      # schema = create_arrow_schema(false)
     )
 }
diff --git a/libtiledbsoma/test/common.cc b/libtiledbsoma/test/common.cc
@@ -51,7 +51,8 @@ ArraySchema create_schema(Context& ctx, bool allow_duplicates) {
     return schema;
 }
 
-std::pair<std::unique_ptr<ArrowSchema>, ArrowTable> create_arrow_schema() {
+std::pair<std::unique_ptr<ArrowSchema>, ArrowTable> create_arrow_schema(
+    bool use_current_domain) {
     // Create ArrowSchema for SOMAArray
     auto arrow_schema = std::make_unique<ArrowSchema>();
     arrow_schema->format = "+s";
@@ -97,25 +98,26 @@ std::pair<std::unique_ptr<ArrowSchema>, ArrowTable> create_arrow_schema() {
     col_info_array->n_children = 2;
     col_info_array->release = &ArrowAdapter::release_array;
     col_info_array->children = new ArrowArray*[1];
+    int n = use_current_domain ? 5 : 3;
     auto d0_info = col_info_array->children[0] = new ArrowArray;
-    d0_info->length = 3;
+    d0_info->length = n;
     d0_info->null_count = 0;
     d0_info->offset = 0;
     d0_info->n_buffers = 2;
     d0_info->release = &ArrowAdapter::release_array;
     d0_info->buffers = new const void*[2];
     d0_info->buffers[0] = nullptr;
-    d0_info->buffers[1] = malloc(sizeof(int64_t) * 3);
+    d0_info->buffers[1] = malloc(sizeof(int64_t) * n);
     d0_info->n_children = 0;
     int64_t dom[] = {0, 1000, 1};
-    std::memcpy((void*)d0_info->buffers[1], &dom, sizeof(int64_t) * 3);
+    std::memcpy((void*)d0_info->buffers[1], &dom, sizeof(int64_t) * n);
 
     return std::pair(
         std::move(arrow_schema),
         ArrowTable(std::move(col_info_array), std::move(col_info_schema)));
 }
 
-ArrowTable create_column_index_info() {
+ArrowTable create_column_index_info(bool use_current_domain) {
     // Create ArrowSchema for IndexColumnInfo
     auto col_info_schema = std::make_unique<ArrowSchema>();
     col_info_schema->format = "+s";
@@ -140,19 +142,20 @@ ArrowTable create_column_index_info() {
     col_info_array->n_children = 2;
     col_info_array->release = &ArrowAdapter::release_array;
     col_info_array->children = new ArrowArray*[1];
+    int n = use_current_domain ? 5 : 3;
     auto d0_info = col_info_array->children[0] = new ArrowArray;
-    d0_info->length = 3;
+    d0_info->length = n;
     d0_info->null_count = 0;
     d0_info->offset = 0;
     d0_info->n_buffers = 2;
     d0_info->release = &ArrowAdapter::release_array;
     d0_info->buffers = new const void*[2];
     d0_info->buffers[0] = nullptr;
-    d0_info->buffers[1] = malloc(sizeof(int64_t) * 3);
+    d0_info->buffers[1] = malloc(sizeof(int64_t) * n);
     d0_info->n_children = 0;
-    int64_t dom[] = {0, 1000, 1};
-    std::memcpy((void*)d0_info->buffers[1], &dom, sizeof(int64_t) * 3);
+    int64_t dom[] = {0, 1000, 1, 0, 2147483646};
+    std::memcpy((void*)d0_info->buffers[1], &dom, sizeof(int64_t) * n);
 
     return ArrowTable(std::move(col_info_array), std::move(col_info_schema));
 }
-}  // namespace helper
+}  // namespace helper
diff --git a/libtiledbsoma/test/common.h b/libtiledbsoma/test/common.h
@@ -61,7 +61,8 @@ static const std::string src_path = TILEDBSOMA_SOURCE_ROOT;
 
 namespace helper {
 ArraySchema create_schema(Context& ctx, bool allow_duplicates = false);
-std::pair<std::unique_ptr<ArrowSchema>, ArrowTable> create_arrow_schema();
-ArrowTable create_column_index_info();
+std::pair<std::unique_ptr<ArrowSchema>, ArrowTable> create_arrow_schema(
+    bool use_current_domain);
+ArrowTable create_column_index_info(bool use_current_domain);
 }  // namespace helper
-#endif
+#endif
diff --git a/libtiledbsoma/test/unit_soma_collection.cc b/libtiledbsoma/test/unit_soma_collection.cc
@@ -53,7 +53,11 @@ TEST_CASE("SOMACollection: add SOMASparseNDArray") {
     std::string sub_uri = "mem://unit-test-add-sparse-ndarray/sub";
 
     SOMACollection::create(base_uri, ctx, ts);
-    auto index_columns = helper::create_column_index_info();
+
+    // TODO: parameterize T/F and assert
+    bool use_current_domain = false;
+
+    auto index_columns = helper::create_column_index_info(use_current_domain);
     auto schema = helper::create_schema(*ctx->tiledb_ctx(), true);
 
     std::map<std::string, SOMAGroupEntry> expected_map{
@@ -94,7 +98,8 @@ TEST_CASE("SOMACollection: add SOMADenseNDArray") {
     std::string sub_uri = "mem://unit-test-add-dense-ndarray/sub";
 
     SOMACollection::create(base_uri, ctx, ts);
-    auto index_columns = helper::create_column_index_info();
+    // TODO: parameterize T/F and assert
+    auto index_columns = helper::create_column_index_info(false);
     auto schema = helper::create_schema(*ctx->tiledb_ctx(), true);
 
     std::map<std::string, SOMAGroupEntry> expected_map{
@@ -134,7 +139,8 @@ TEST_CASE("SOMACollection: add SOMADataFrame") {
     std::string sub_uri = "mem://unit-test-add-dataframe/sub";
 
     SOMACollection::create(base_uri, ctx, ts);
-    auto [schema, index_columns] = helper::create_arrow_schema();
+    // TODO: parameterize T/F and assert
+    auto [schema, index_columns] = helper::create_arrow_schema(false);
 
     std::map<std::string, SOMAGroupEntry> expected_map{
         {"dataframe", SOMAGroupEntry(sub_uri, "SOMAArray")}};
@@ -173,6 +179,7 @@ TEST_CASE("SOMACollection: add SOMACollection") {
     std::string sub_uri = "mem://unit-test-add-collection/sub";
 
     SOMACollection::create(base_uri, ctx);
+    // TODO: parameterize T/F and assert
     auto schema = helper::create_schema(*ctx->tiledb_ctx(), false);
 
     std::map<std::string, SOMAGroupEntry> expected_map{
@@ -198,7 +205,8 @@ TEST_CASE("SOMACollection: add SOMAExperiment") {
     std::string sub_uri = "mem://unit-test-add-experiment/sub";
 
     SOMACollection::create(base_uri, ctx);
-    auto [schema, index_columns] = helper::create_arrow_schema();
+    // TODO: parameterize T/F and assert
+    auto [schema, index_columns] = helper::create_arrow_schema(false);
 
     std::map<std::string, SOMAGroupEntry> expected_map{
         {"experiment", SOMAGroupEntry(sub_uri, "SOMAGroup")}};
@@ -230,7 +238,8 @@ TEST_CASE("SOMACollection: add SOMAMeasurement") {
     std::string sub_uri = "mem://unit-test-add-measurement/sub";
 
     SOMACollection::create(base_uri, ctx);
-    auto [schema, index_columns] = helper::create_arrow_schema();
+    // TODO: parameterize T/F and assert
+    auto [schema, index_columns] = helper::create_arrow_schema(false);
 
     std::map<std::string, SOMAGroupEntry> expected_map{
         {"measurement", SOMAGroupEntry(sub_uri, "SOMAGroup")}};
@@ -313,7 +322,8 @@ TEST_CASE("SOMAExperiment: metadata") {
     auto ctx = std::make_shared<SOMAContext>();
 
     std::string uri = "mem://unit-test-experiment";
-    auto [schema, index_columns] = helper::create_arrow_schema();
+    // TODO: parameterize T/F and assert
+    auto [schema, index_columns] = helper::create_arrow_schema(false);
     SOMAExperiment::create(
         uri,
         std::move(schema),
@@ -380,7 +390,8 @@ TEST_CASE("SOMAExperiment: metadata") {
 TEST_CASE("SOMAMeasurement: metadata") {
     auto ctx = std::make_shared<SOMAContext>();
     std::string uri = "mem://unit-test-measurement";
-    auto [schema, index_columns] = helper::create_arrow_schema();
+    // TODO: parameterize T/F and assert
+    auto [schema, index_columns] = helper::create_arrow_schema(false);
     SOMAMeasurement::create(
         uri,
         std::move(schema),

diff --git a/libtiledbsoma/test/unit_soma_dataframe.cc b/libtiledbsoma/test/unit_soma_dataframe.cc
@@ -38,7 +38,11 @@ TEST_CASE("SOMADataFrame: basic") {
 
     REQUIRE(!SOMADataFrame::exists(uri, ctx));
 
-    auto [schema, index_columns] = helper::create_arrow_schema();
+    // TODO: parameterize T/F and assert
+    bool use_current_domain = false;
+
+    auto [schema, index_columns] = helper::create_arrow_schema(
+        use_current_domain);
     SOMADataFrame::create(
         uri,
         std::move(schema),
@@ -148,7 +152,11 @@ TEST_CASE("SOMADataFrame: platform_config") {
                                     R"(]}})";
         }
 
-        auto [schema, index_columns] = helper::create_arrow_schema();
+        // TODO: parameterize T/F and assert
+        bool use_current_domain = false;
+
+        auto [schema, index_columns] = helper::create_arrow_schema(
+            use_current_domain);
         SOMADataFrame::create(
             uri,
             std::move(schema),
@@ -184,7 +192,12 @@ TEST_CASE("SOMADataFrame: platform_config") {
 TEST_CASE("SOMADataFrame: metadata") {
     auto ctx = std::make_shared<SOMAContext>();
     std::string uri = "mem://unit-test-collection";
-    auto [schema, index_columns] = helper::create_arrow_schema();
+
+    // TODO: parameterize T/F and assert
+    bool use_current_domain = false;
+
+    auto [schema, index_columns] = helper::create_arrow_schema(
+        use_current_domain);
     SOMADataFrame::create(
         uri,
         std::move(schema),

diff --git a/libtiledbsoma/test/unit_soma_dense_ndarray.cc b/libtiledbsoma/test/unit_soma_dense_ndarray.cc
@@ -38,7 +38,10 @@ TEST_CASE("SOMADenseNDArray: basic") {
 
     REQUIRE(!SOMADenseNDArray::exists(uri, ctx));
 
-    auto index_columns = helper::create_column_index_info();
+    // TODO: parameterize T/F and assert
+    bool use_current_domain = false;
+
+    auto index_columns = helper::create_column_index_info(use_current_domain);
     SOMADenseNDArray::create(
         uri,
         "l",
@@ -91,7 +94,10 @@ TEST_CASE("SOMADenseNDArray: platform_config") {
     PlatformConfig platform_config;
     platform_config.dense_nd_array_dim_zstd_level = 6;
 
-    auto index_columns = helper::create_column_index_info();
+    // TODO: parameterize T/F and assert
+    bool use_current_domain = false;
+
+    auto index_columns = helper::create_column_index_info(use_current_domain);
     SOMADenseNDArray::create(
         uri,
         "l",
@@ -117,8 +123,11 @@ TEST_CASE("SOMADenseNDArray: metadata") {
 
     std::string uri = "mem://unit-test-dense-ndarray";
 
-    auto index_columns = helper::create_column_index_info();
-    SOMASparseNDArray::create(
+    // TODO: parameterize T/F and assert
+    bool use_current_domain = false;
+
+    auto index_columns = helper::create_column_index_info(use_current_domain);
+    SOMASparseNDArray::create(  // XXX TODO: this looks like an old typo ...
         uri,
         "l",
         ArrowTable(

diff --git a/libtiledbsoma/test/unit_soma_sparse_ndarray.cc b/libtiledbsoma/test/unit_soma_sparse_ndarray.cc
@@ -38,7 +38,10 @@ TEST_CASE("SOMASparseNDArray: basic") {
 
     REQUIRE(!SOMASparseNDArray::exists(uri, ctx));
 
-    auto index_columns = helper::create_column_index_info();
+    // TODO: parameterize T/F and assert
+    bool use_current_domain = false;
+
+    auto index_columns = helper::create_column_index_info(use_current_domain);
     SOMASparseNDArray::create(
         uri,
         "l",
@@ -95,7 +98,10 @@ TEST_CASE("SOMASparseNDArray: platform_config") {
     PlatformConfig platform_config;
     platform_config.sparse_nd_array_dim_zstd_level = 6;
 
-    auto index_columns = helper::create_column_index_info();
+    // TODO: parameterize T/F and assert
+    bool use_current_domain = false;
+
+    auto index_columns = helper::create_column_index_info(use_current_domain);
     SOMASparseNDArray::create(
         uri,
         "l",
@@ -121,7 +127,10 @@ TEST_CASE("SOMASparseNDArray: metadata") {
 
     std::string uri = "mem://unit-test-sparse-ndarray";
 
-    auto index_columns = helper::create_column_index_info();
+    // TODO: parameterize T/F and assert
+    bool use_current_domain = false;
+
+    auto index_columns = helper::create_column_index_info(use_current_domain);
     SOMASparseNDArray::create(
         uri,
         "l",
@@ -182,4 +191,4 @@ TEST_CASE("SOMASparseNDArray: metadata") {
     soma_sparse->open(OpenMode::read, TimestampRange(0, 2));
     REQUIRE(!soma_sparse->has_metadata("md"));
     REQUIRE(soma_sparse->metadata_num() == 2);
-}
+}