Sync main to release-1.7 as much as possible

.github/workflows/pkgdown.yaml

@@ -1,5 +1,6 @@
 # Workflow derived from https://github.com/r-lib/actions/tree/master/examples
 # Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help
+
 on:
   #push:
   #  # To publish docs from your branch: list the branch name here instead of main.
@@ -9,6 +10,8 @@ on:
   #  branches: [main]
   release:
     types: [published]
+  schedule:
+    - cron: "42 9 * * *"
   workflow_dispatch:
 
 name: pkgdown
@@ -31,8 +34,12 @@ jobs:
           use-public-rspm: true
           working-directory: "apis/r"
 
+      # Run this daily to surface errors as tracked at
+      # https://github.com/single-cell-data/TileDB-SOMA/issues/2052
       - name: Install dependencies
         run: ./apis/r/tools/install-pkgdown-dependencies.sh
 
+      # Run this on releases, or on workflow dispatch
       - name: Deploy package
+        if: github.event_name != 'schedule'
         run: ./apis/r/tools/deploy-pkgdown.sh

apis/python/setup.py

@@ -40,14 +40,11 @@
 
 import version  # noqa E402
 
-# tiledb_dir and libtiledbsoma_dir may be specified by either environment variable
+# tiledb_dir and tiledbsoma_dir may be specified by either environment variable
 # or command-line argument. If both are provided, the latter wins.
 
 tiledb_dir: Optional[pathlib.Path] = None
-libtiledbsoma_dir: Optional[pathlib.Path] = None
-
-tiledb_dir = pathlib.Path(os.environ.get("TILEDB_PATH", this_dir))
-libtiledbsoma_dir = os.environ.get("TILEDBSOMA_PATH", None)
+tiledbsoma_dir: Optional[pathlib.Path] = None
 
 args = sys.argv[:]
 for arg in args:
@@ -56,16 +53,37 @@
         tiledb_dir = pathlib.Path(last)
         sys.argv.remove(arg)
     if (start, eq) == ("--libtiledbsoma", "="):
-        libtiledbsoma_dir = pathlib.Path(last)
+        tiledbsoma_dir = pathlib.Path(last)
         sys.argv.remove(arg)
 
-if libtiledbsoma_dir is None:
+tiledb_dir = os.environ.get("TILEDB_PATH", None)
+tiledb_given = tiledb_dir is not None
+tiledbsoma_dir = os.environ.get("TILEDBSOMA_PATH", None)
+
+if tiledbsoma_dir is not None and tiledb_dir is None:
+    raise ValueError(
+        "If TILEDBSOMA_PATH is set, then TILEDB_PATH must be "
+        "also be set. TILEDB_PATH must be set to the location of "
+        "the TileDB shared object library linked to the "
+        "TileDB-SOMA shared object library"
+    )
+
+if tiledbsoma_dir is None:
     scripts_dir = this_dir / "dist_links" / "scripts"
     scripts_dir = scripts_dir.resolve()
 
-    libtiledbsoma_dir = scripts_dir.parent / "dist"
+    tiledbsoma_dir = scripts_dir.parent / "dist"
 else:
-    libtiledbsoma_dir = pathlib.Path(libtiledbsoma_dir)
+    tiledbsoma_dir = pathlib.Path(tiledbsoma_dir)
+
+if tiledb_dir is None:
+    # tiledb_dir = pathlib.Path(this_dir)
+    scripts_dir = this_dir / "dist_links" / "scripts"
+    scripts_dir = scripts_dir.resolve()
+
+    tiledb_dir = scripts_dir.parent / "dist"
+else:
+    tiledb_dir = pathlib.Path(tiledb_dir)
 
 
 def get_libtiledbsoma_library_name():
@@ -106,12 +124,12 @@ def libtiledbsoma_exists():
     :return: The path to the TileDB-SOMA library, or None.
     """
     # Check if TileDB-SOMA is installed in user given path
-    dist_dirs = [libtiledbsoma_dir / "lib"]
+    dist_dirs = [tiledbsoma_dir / "lib"]
     if sys.platform.startswith("linux"):
-        dist_dirs.append(libtiledbsoma_dir / "lib64")
-        dist_dirs.append(libtiledbsoma_dir / "lib" / "x86_64-linux-gnu")
+        dist_dirs.append(tiledbsoma_dir / "lib64")
+        dist_dirs.append(tiledbsoma_dir / "lib" / "x86_64-linux-gnu")
     elif os.name == "nt":
-        dist_dirs.append(libtiledbsoma_dir / "bin")
+        dist_dirs.append(tiledbsoma_dir / "bin")
 
     for lib_dir in dist_dirs:
         full_lib_path = lib_dir / get_libtiledbsoma_library_name()
@@ -148,25 +166,42 @@ def find_or_build_package_data(setuptools_cmd):
         #
         # See `.github/workflows/python-ci-single.yml` for configuration.
         if os.name == "nt":
-            subprocess.run(["pwsh.exe", "./bld.ps1"], cwd=scripts_dir, check=True)
+            bld_command = ["pwsh.exe", "./bld.ps1"]
+            if tiledb_dir is not None:
+                bld_command.append(f"TileDBLocation={tiledb_dir}")
         else:
-            subprocess.run(["./bld"], cwd=scripts_dir, check=True)
+            bld_command = ["./bld"]
+            if tiledb_dir is not None:
+                bld_command.append(f"--tiledb={tiledb_dir}")
+
+        subprocess.run(bld_command, cwd=scripts_dir, check=True)
+
         lib_dir = libtiledbsoma_exists()
         assert lib_dir, "error when building libtiledbsoma from source"
 
-    # Copy native libs into the package dir so they can be found by package_data
-    package_data = []
-    src_dir = this_dir / "src" / "tiledbsoma"
-    for f in lib_dir.glob("*tiledbsoma*"):
-        if f.suffix != ".a":  # skip static library
-            print(f"  copying file {f} to {src_dir}")
-            shutil.copy(f, src_dir)
-            package_data.append(f.name)
-    assert package_data, f"libtiledbsoma artifacts absent from {lib_dir}"
-
-    # Install shared libraries inside the Python module via package_data.
-    print(f"  adding to package_data: {package_data}")
-    setuptools_cmd.distribution.package_data["tiledbsoma"] = package_data
+        # If we are building from source, then we are likely building wheels.
+        # Copy both the tiledbsoma and tiledb shared objects into the
+        # package dir so they can be found by package_data
+        package_data = []
+        src_dir = this_dir / "src" / "tiledbsoma"
+        for f in lib_dir.glob("*tiledbsoma.*"):
+            if f.suffix != ".a":  # skip static library
+                print(f"  copying file {f} to {src_dir}")
+                shutil.copy(f, src_dir)
+                package_data.append(f.name)
+        assert package_data, f"tiledbsoma artifacts absent from {lib_dir}"
+
+        if not tiledb_given:
+            for f in lib_dir.glob("*tiledb.*"):
+                if f.suffix != ".a":  # skip static library
+                    print(f"  copying file {f} to {src_dir}")
+                    shutil.copy(f, src_dir)
+                    package_data.append(f.name)
+                assert package_data, f"tiledb artifacts absent from {lib_dir}"
+
+        # Install shared libraries inside the Python module via package_data.
+        print(f"  adding to package_data: {package_data}")
+        setuptools_cmd.distribution.package_data["tiledbsoma"] = package_data
 
 
 class build_ext(setuptools.command.build_ext.build_ext):
@@ -185,34 +220,33 @@ def run(self):
     "dist_links/libtiledbsoma/include",
     "dist_links/libtiledbsoma/external/include",
     "../../build/externals/install/include",
-    str(libtiledbsoma_dir / "include"),
-    str(libtiledbsoma_dir.parent / "build/externals/install/include"),
+    str(tiledbsoma_dir / "include"),
+    str(tiledbsoma_dir.parent / "build/externals/install/include"),
+    str(tiledbsoma_dir / "include"),
     str(tiledb_dir / "include"),
 ]
 
 LIB_DIRS = [
-    str(libtiledbsoma_dir / "lib"),
+    str(tiledbsoma_dir / "lib"),
     str(tiledb_dir / "lib"),
 ]
 
 CXX_FLAGS = []
 
 if os.name != "nt":
-    CXX_FLAGS.append(f'-Wl,-rpath,{str(libtiledbsoma_dir / "lib")}')
+    CXX_FLAGS.append(f'-Wl,-rpath,{str(tiledbsoma_dir / "lib")}')
     CXX_FLAGS.append(f'-Wl,-rpath,{str(tiledb_dir / "lib")}')
 
 if sys.platform == "darwin":
-    CXX_FLAGS.append("-mmacosx-version-min=10.14")
+    CXX_FLAGS.append("-mmacosx-version-min=11.0")
 
 if os.name == "posix" and sys.platform != "darwin":
-    LIB_DIRS.append(str(libtiledbsoma_dir / "lib" / "x86_64-linux-gnu"))
-    LIB_DIRS.append(str(libtiledbsoma_dir / "lib64"))
+    LIB_DIRS.append(str(tiledbsoma_dir / "lib" / "x86_64-linux-gnu"))
+    LIB_DIRS.append(str(tiledbsoma_dir / "lib64"))
     LIB_DIRS.append(str(tiledb_dir / "lib" / "x86_64-linux-gnu"))
     LIB_DIRS.append(str(tiledb_dir / "lib64"))
-    CXX_FLAGS.append(
-        f'-Wl,-rpath,{str(libtiledbsoma_dir / "lib" / "x86_64-linux-gnu")}'
-    )
-    CXX_FLAGS.append(f'-Wl,-rpath,{str(libtiledbsoma_dir / "lib64")}')
+    CXX_FLAGS.append(f'-Wl,-rpath,{str(tiledbsoma_dir / "lib" / "x86_64-linux-gnu")}')
+    CXX_FLAGS.append(f'-Wl,-rpath,{str(tiledbsoma_dir / "lib64")}')
     CXX_FLAGS.append(f'-Wl,-rpath,{str(tiledb_dir / "lib" / "x86_64-linux-gnu")}')
     CXX_FLAGS.append(f'-Wl,-rpath,{str(tiledb_dir / "lib64")}')
 
@@ -263,6 +297,8 @@ def run(self):
                 "src/tiledbsoma/soma_array.cc",
                 "src/tiledbsoma/soma_object.cc",
                 "src/tiledbsoma/soma_dataframe.cc",
+                "src/tiledbsoma/soma_dense_ndarray.cc",
+                "src/tiledbsoma/soma_sparse_ndarray.cc",
                 "src/tiledbsoma/pytiledbsoma.cc",
             ],
             include_dirs=INC_DIRS,

apis/python/src/tiledbsoma/__init__.py

@@ -93,6 +93,48 @@
 # crucial that we include a separator (e.g. "Classes and functions") to make an entry in the
 # readthedocs table of contents.
 
+import ctypes
+import os
+import sys
+
+
+# Load native libraries. On wheel builds, we may have a shared library
+# already linked. In this case, we can import directly
+try:
+    from . import pytiledbsoma as clib
+
+    del clib
+except ImportError:
+    if os.name == "nt":
+        libtiledb_name = "tiledb.dll"
+    elif sys.platform == "darwin":
+        libtiledb_name = "libtiledb.dylib"
+    else:
+        libtiledb_name = "libtiledb.so"
+
+    try:
+        # Try loading the bundled native library.
+        lib_dir = os.path.dirname(os.path.abspath(__file__))
+        ctypes.CDLL(os.path.join(lib_dir, libtiledb_name), mode=ctypes.RTLD_GLOBAL)
+    except OSError:
+        # Otherwise try loading by name only.
+        ctypes.CDLL(libtiledb_name, mode=ctypes.RTLD_GLOBAL)
+
+    if os.name == "nt":
+        libtiledbsoma_name = "tiledbsoma.dll"
+    elif sys.platform == "darwin":
+        libtiledbsoma_name = "libtiledbsoma.dylib"
+    else:
+        libtiledbsoma_name = "libtiledbsoma.so"
+
+    try:
+        # Try loading the bundled native library.
+        lib_dir = os.path.dirname(os.path.abspath(__file__))
+        ctypes.CDLL(os.path.join(lib_dir, libtiledbsoma_name))
+    except OSError:
+        # Otherwise try loading by name only.
+        ctypes.CDLL(libtiledbsoma_name)
+
 from somacore import AxisColumnNames, AxisQuery, ExperimentAxisQuery
 from somacore.options import ResultOrder
 

apis/python/src/tiledbsoma/_tiledb_array.py

@@ -3,47 +3,22 @@
 #
 # Licensed under the MIT License.
 
-import ctypes
-import os
-import sys
 from typing import Any, Dict, List, Optional, Sequence, Tuple
 
 import pyarrow as pa
 import tiledb
 from somacore.options import ResultOrder, ResultOrderStr
 
 from . import _tdb_handles, _util
+
+# This package's pybind11 code
+from . import pytiledbsoma as clib  # noqa: E402
 from ._arrow_types import tiledb_schema_to_arrow
 from ._tiledb_object import TileDBObject
 from ._types import OpenTimestamp, is_nonstringy_sequence
 from .options._soma_tiledb_context import SOMATileDBContext
 
 
-def _load_libs() -> None:
-    """Loads the required TileDB-SOMA native library."""
-    if os.name == "nt":
-        lib_name = "tiledbsoma.dll"
-    elif sys.platform == "darwin":
-        lib_name = "libtiledbsoma.dylib"
-    else:
-        lib_name = "libtiledbsoma.so"
-
-    try:
-        # Try loading the bundled native library.
-        lib_dir = os.path.dirname(os.path.abspath(__file__))
-        ctypes.CDLL(os.path.join(lib_dir, lib_name))
-    except OSError:
-        # Otherwise try loading by name only.
-        ctypes.CDLL(lib_name)
-
-
-# Load native libraries
-_load_libs()
-
-# This package's pybind11 code
-from . import pytiledbsoma as clib  # noqa: E402
-
-
 class TileDBArray(TileDBObject[_tdb_handles.ArrayWrapper]):
     """Wraps arrays from TileDB-Py by retaining a URI, options, etc.
     Also serves as an abstraction layer to hide TileDB-specific details

apis/python/src/tiledbsoma/options/_tiledb_create_options.py

@@ -32,7 +32,7 @@
 
 The key ``_type`` is used as the name of the filter. Other entries in the
 dictionary are passed as named arguments. For example,
-``{"_name": "SomeFilter", "aggression": 5, "layers": 7}`` will call
+``{"_type": "SomeFilter", "aggression": 5, "layers": 7}`` will call
 ``SomeFilter(aggression=5, layers=7)``."""
 _FilterSpec = Union[str, _DictFilterSpec]
 """A declarative format for specifying filters:

apis/python/src/tiledbsoma/pytiledbsoma.cc

@@ -16,9 +16,11 @@ using namespace tiledbsoma;
 template <typename... Args>
 using overload_cast_ = pybind11::detail::overload_cast_impl<Args...>;
 
-void load_soma_array(py::module &);
 void load_soma_object(py::module &);
+void load_soma_array(py::module &);
 void load_soma_dataframe(py::module &);
+void load_soma_dense_ndarray(py::module &);
+void load_soma_sparse_ndarray(py::module &);
 void load_query_condition(py::module &);
 void load_reindexer(py::module &);
 
@@ -89,9 +91,11 @@ PYBIND11_MODULE(pytiledbsoma, m) {
         },
         "Print TileDB internal statistics. Lifecycle: experimental.");
 
-    load_soma_array(m);
     load_soma_object(m);
+    load_soma_array(m);
     load_soma_dataframe(m);
+    load_soma_dense_ndarray(m);
+    load_soma_sparse_ndarray(m);
     load_query_condition(m);
     load_reindexer(m);
 }