Internal context-handling updates (#444)

* Internal context-handling updates

* more

* code-review feedback

.github/workflows/ci.yml

@@ -47,6 +47,12 @@ jobs:
         CC: ${{ matrix.cc }}
         CXX: ${{ matrix.cxx }}
 
+    - name: Show package versions
+      run: python scripts/show-versions.py
+
+    - name: Show XCode version
+      run: clang --version
+
     - name: Run pytests
       run: python -m pytest apis/python/tests
 

apis/python/src/tiledbsoma/__init__.py

@@ -39,6 +39,7 @@
     get_SOMA_version,
     get_storage_engine,
 )
+from .query_condition import QueryCondition  # type: ignore
 from .soma_collection import SOMACollection
 from .soma_dataframe import SOMADataFrame
 from .soma_dense_nd_array import SOMADenseNdArray
@@ -70,4 +71,5 @@
     "SOMAMeasurement",
     "SOMAMetadataMapping",
     "SOMASparseNdArray",
+    "QueryCondition",
 ]

apis/python/src/tiledbsoma/query_condition.py

@@ -4,7 +4,7 @@
 
 import ast
 from dataclasses import dataclass, field
-from typing import Any, Callable, List, Tuple, Type, Union
+from typing import Any, Callable, List, Tuple, Union
 
 import numpy as np
 import tiledb
@@ -99,19 +99,18 @@ class QueryCondition:
     >>>     # Select cells where the attribute values for `foo` are less than 5
     >>>     # and `bar` equal to string "asdf".
     >>>     # Note precedence is equivalent to:
-    >>>     # tiledb.QueryCondition("foo > 5 or ('asdf' == attr('b a r') and baz <= val(1.0))")
-    >>>     qc = tiledb.QueryCondition("foo > 5 or 'asdf' == attr('b a r') and baz <= val(1.0)")
+    >>>     # tiledbsoma.QueryCondition("foo > 5 or ('asdf' == attr('b a r') and baz <= val(1.0))")
+    >>>     qc = tiledbsoma.QueryCondition("foo > 5 or 'asdf' == attr('b a r') and baz <= val(1.0)")
     >>>     A.query(attr_cond=qc)
     >>>
     >>>     # Select cells where the attribute values for `foo` are equal to
     >>>     # 1, 2, or 3.
     >>>     # Note this is equivalent to:
-    >>>     # tiledb.QueryCondition("foo == 1 or foo == 2 or foo == 3")
-    >>>     A.query(attr_cond=tiledb.QueryCondition("foo in [1, 2, 3]"))
+    >>>     # tiledbsoma.QueryCondition("foo == 1 or foo == 2 or foo == 3")
+    >>>     A.query(attr_cond=tiledbsoma.QueryCondition("foo in [1, 2, 3]"))
     """
 
     expression: str
-    ctx: tiledb.Ctx = field(default_factory=tiledb.default_ctx, repr=False)
     tree: ast.Expression = field(init=False, repr=False)
     c_obj: PyQueryCondition = field(init=False, repr=False)
 
@@ -131,7 +130,7 @@ def __post_init__(self):
             )
 
     def init_query_condition(self, schema: tiledb.ArraySchema, query_attrs: List[str]):
-        qctree = QueryConditionTree(self.ctx, schema, query_attrs)
+        qctree = QueryConditionTree(schema, query_attrs)
         self.c_obj = qctree.visit(self.tree.body)
 
         if not isinstance(self.c_obj, PyQueryCondition):
@@ -143,7 +142,6 @@ def init_query_condition(self, schema: tiledb.ArraySchema, query_attrs: List[str
 
 @dataclass
 class QueryConditionTree(ast.NodeVisitor):
-    ctx: tiledb.Ctx
     schema: tiledb.ArraySchema
     query_attrs: List[str]
 
@@ -183,7 +181,7 @@ def visit_In(self, node):
     def visit_List(self, node):
         return list(node.elts)
 
-    def visit_Compare(self, node: Type[ast.Compare]) -> PyQueryCondition:
+    def visit_Compare(self, node: ast.Compare) -> PyQueryCondition:
         operator = self.visit(node.ops[0])
 
         if operator in (
@@ -241,7 +239,7 @@ def aux_visit_Compare(
         dtype = "string" if dt.kind in "SUa" else dt.name
         val = self.cast_val_to_dtype(val, dtype)
 
-        pyqc = PyQueryCondition(self.ctx)
+        pyqc = PyQueryCondition()
         self.init_pyqc(pyqc, dtype)(att, val, op)
 
         return pyqc
@@ -360,7 +358,7 @@ def get_val_from_node(self, node: QueryConditionNodeElem) -> Any:
         return val
 
     def cast_val_to_dtype(
-        self, val: Union[str, int, float, bytes], dtype: str
+        self, val: Union[str, int, float, bytes, np.int32], dtype: str
     ) -> Union[str, int, float, bytes]:
         if dtype != "string":
             try:

libtiledbsoma/CMakeLists.txt

@@ -152,7 +152,7 @@ if (MSVC)
   #   C4996: deprecation warning about e.g. sscanf.
   add_compile_options(/W4 /wd4101 /wd4146 /wd4244 /wd4251 /wd4456 /wd4457 /wd4702 /wd4800 /wd4996)
   # Warnings as errors:
-  if (ENABLE_TILEDBSOMA_WERROR)
+  if (TILEDBSOMA_ENABLE_WERROR)
     add_compile_options(/WX)
   endif()
   # Disable GDI (which we don't need, and causes some macro
@@ -168,7 +168,7 @@ if (MSVC)
                       )
 else()
   add_compile_options(-Wall -Wextra)
-  if (ENABLE_TILEDBSOMA_WERROR)
+  if (TILEDBSOMA_ENABLE_WERROR)
     add_compile_options(-Werror)
   endif()
   # Build-specific flags

libtiledbsoma/src/pyapi/libtiledbsoma.cc

@@ -172,7 +172,6 @@ PYBIND11_MODULE(libtiledbsoma, m) {
         // Binding overloaded methods to templated member functions requires
         // more effort, see:
         // https://pybind11.readthedocs.io/en/stable/classes.html#overloaded-methods
-
         .def(
             "set_dim_points",
             static_cast<void (SOMAReader::*)(

libtiledbsoma/src/pyapi/query_condition.cc

@@ -68,8 +68,11 @@ class PyQueryCondition {
   PyQueryCondition() = delete;
 
   PyQueryCondition(py::object ctx) {
+    (void)ctx;
     try {
-      set_ctx(ctx);
+      // create one global context for all query conditions
+      static Context context = Context();
+      ctx_ = context;
       qc_ = shared_ptr<QueryCondition>(new QueryCondition(ctx_));
     } catch (TileDBError &e) {
       TPY_ERROR_LOC(e.what());

libtiledbsoma/src/soma_reader.cc

@@ -148,6 +148,12 @@ uint64_t SOMAReader::nnz() {
     //===============================================================
     // If only one fragment, return total_cell_num
     auto fragment_count = fragment_info.fragment_num();
+
+    if (fragment_count == 0) {
+        // Array schema has been created but no data have been written
+        return 0;
+    }
+
     if (fragment_count == 1) {
         return fragment_info.total_cell_num();
     }

scripts/bld

@@ -77,6 +77,8 @@ else
   # non-trivial cmake debugging.
   if [ ! -z "$LIBTILEDBSOMA_DEBUG_BUILD" ]; then
     EXTRA_OPTS="-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON"
+    # TILEDB_WERROR=OFF is necessary to build core with XCode 14; doesn't hurt for XCode 13.
+    # EXTRA_OPTS="-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -DTILEDB_WERROR=OFF _DTILEDBSOMA_ENABLE_WERROR=OFF"
   fi
   # Also (pro-tip), set nproc=1 to get a more deterministic ordering of output lines.
   if [ ! -z "$LIBTILEDBSOMA_NO_PARALLEL_BUILD" ]; then

scripts/show-versions.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+
+import sys
+
+import tiledb
+import tiledbsoma
+import tiledbsoma.io
+
+t = tiledbsoma
+
+import anndata as ad
+import numpy as np
+import pandas as pd
+import pyarrow as pa
+import scanpy as sc
+import scipy as sp
+import pybind11
+
+print("tiledbsoma.__version__   ", tiledbsoma.__version__)
+print("tiledb.__version__       ", tiledb.__version__)
+print(
+    "core version             ",
+    ".".join(str(ijk) for ijk in list(tiledb.libtiledb.version())),
+)
+print("anndata.__version__  (ad)", ad.__version__)
+print("numpy.__version__    (np)", np.__version__)
+print("pandas.__version__   (pd)", pd.__version__)
+print("pyarrow.__version__  (pa)", pa.__version__)
+print("scanpy.__version__   (sc)", sc.__version__)
+print("scipy.__version__    (sp)", sp.__version__)
+print("pybind11.__version__     ", pybind11.__version__)
+print(
+    "python__version__        ",
+    ".".join(
+        [
+            str(e)
+            for e in [
+                sys.version_info.major,
+                sys.version_info.minor,
+                sys.version_info.micro,
+            ]
+        ]
+    ),
+)