Merge to main after #2448 with core 2.22 et al.

.github/workflows/python-so-copying.yml

@@ -68,7 +68,7 @@ jobs:
         run: |
           mkdir -p external
           # Please do not edit manually -- let scripts/update-tiledb-version.py update this
-          wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.21.1/tiledb-linux-x86_64-2.21.1-acd5c50.tar.gz
+          wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.22.0/tiledb-linux-x86_64-2.22.0-52e981e.tar.gz
           tar -C external -xzf tiledb-linux-x86_64-*.tar.gz
           ls external/lib/
           echo "LD_LIBRARY_PATH=$(pwd)/external/lib" >> $GITHUB_ENV
@@ -139,7 +139,7 @@ jobs:
           ./venv-soma/bin/python -c "import tiledbsoma; print(tiledbsoma.pytiledbsoma.version())"
 
   macos:
-    runs-on: macos-12
+    runs-on: macos-13
     name: "macos TILEDB_EXISTS: ${{ matrix.TILEDB_EXISTS }} TILEDBSOMA_EXISTS: ${{ matrix.TILEDBSOMA_EXISTS }}"
     strategy:
       fail-fast: false
@@ -153,12 +153,14 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0 # for setuptools-scm
+      - name: Check if System Integrity Protection (SIP) is enabled
+        run: csrutil status
       - name: Install pre-built libtiledb
         if: ${{ matrix.TILEDB_EXISTS == 'yes' }}
         run: |
           mkdir -p external
           # Please do not edit manually -- let scripts/update-tiledb-version.py update this
-          wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.21.1/tiledb-macos-x86_64-2.21.1-acd5c50.tar.gz
+          wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.22.0/tiledb-macos-x86_64-2.22.0-52e981e.tar.gz
           tar -C external -xzf tiledb-macos-x86_64-*.tar.gz
           ls external/lib/
           echo "DYLD_LIBRARY_PATH=$(pwd)/external/lib" >> $GITHUB_ENV
@@ -249,10 +251,10 @@ jobs:
           if [ `uname -s` == "Darwin" ];
           then
             # Please do not edit manually -- let scripts/update-tiledb-version.py update this
-            wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.21.1/tiledb-macos-x86_64-2.21.1-acd5c50.tar.gz
+            wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.22.0/tiledb-macos-x86_64-2.22.0-52e981e.tar.gz
           else
             # Please do not edit manually -- let scripts/update-tiledb-version.py update this
-            wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.21.1/tiledb-linux-x86_64-2.21.1-acd5c50.tar.gz
+            wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.22.0/tiledb-linux-x86_64-2.22.0-52e981e.tar.gz
           fi
           tar -C external -xzf tiledb-*.tar.gz
           ls external/lib/

apis/python/setup.py

@@ -330,7 +330,7 @@ def run(self):
         "scanpy>=1.9.2",
         "scipy",
         "somacore==1.0.10",
-        "tiledb~=0.27.0",
+        "tiledb~=0.28.0",
         "typing-extensions",  # Note "-" even though `import typing_extensions`
     ],
     extras_require={

apis/python/src/tiledbsoma/__init__.py

@@ -147,7 +147,7 @@
 from ._constants import SOMA_JOINID
 from ._dataframe import DataFrame
 from ._dense_nd_array import DenseNDArray
-from ._exception import DoesNotExistError, SOMAError
+from ._exception import AlreadyExistsError, DoesNotExistError, SOMAError
 from ._experiment import Experiment
 from ._factory import open
 from ._general_utilities import (
@@ -156,6 +156,7 @@
     get_SOMA_version,
     get_storage_engine,
     show_package_versions,
+    verify_core_versions,
 )
 from ._indexer import IntIndexer, tiledbsoma_build_index
 from ._measurement import Measurement
@@ -168,9 +169,15 @@
     tiledbsoma_stats_reset,
 )
 
+# Ensure TileDB-Py and libtiledbsoma have matching core versions; if they don't, undefined behavior
+# / segfaults may ensue. Once libtiledbsoma is the only "path to core" (cf. #1632), this can be
+# removed.
+verify_core_versions()
+
 __version__ = get_implementation_version()
 
 __all__ = [
+    "AlreadyExistsError",
     "AxisColumnNames",
     "AxisQuery",
     "Collection",

apis/python/src/tiledbsoma/_collection.py

@@ -37,7 +37,12 @@
 from ._common_nd_array import NDArray
 from ._dataframe import DataFrame
 from ._dense_nd_array import DenseNDArray
-from ._exception import SOMAError, is_does_not_exist_error
+from ._exception import (
+    AlreadyExistsError,
+    SOMAError,
+    is_already_exists_error,
+    is_does_not_exist_error,
+)
 from ._funcs import typeguard_ignore
 from ._sparse_nd_array import SparseNDArray
 from ._tiledb_object import AnyTileDBObject, TileDBObject
@@ -112,20 +117,27 @@ def create(
                 the context.
 
         Raises:
+            tiledbsoma.AlreadyExistsError:
+                If the underlying object already exists at the given URI.
             TileDBError:
                 If unable to create the underlying object.
 
         Lifecycle:
             Experimental.
         """
         context = _validate_soma_tiledb_context(context)
-        tiledb.group_create(uri=uri, ctx=context.tiledb_ctx)
-        handle = cls._wrapper_type.open(uri, "w", context, tiledb_timestamp)
-        cls._set_create_metadata(handle)
-        return cls(
-            handle,
-            _dont_call_this_use_create_or_open_instead="tiledbsoma-internal-code",
-        )
+        try:
+            tiledb.group_create(uri=uri, ctx=context.tiledb_ctx)
+            handle = cls._wrapper_type.open(uri, "w", context, tiledb_timestamp)
+            cls._set_create_metadata(handle)
+            return cls(
+                handle,
+                _dont_call_this_use_create_or_open_instead="tiledbsoma-internal-code",
+            )
+        except tiledb.TileDBError as tdbe:
+            if is_already_exists_error(tdbe):
+                raise AlreadyExistsError(f"{uri!r} already exists")
+            raise
 
     @classmethod
     def open(

apis/python/src/tiledbsoma/_common_nd_array.py

@@ -16,6 +16,7 @@
 import tiledb
 
 from . import _arrow_types, _util
+from ._exception import AlreadyExistsError, is_already_exists_error
 from ._tiledb_array import TileDBArray
 from ._types import OpenTimestamp
 from .options._soma_tiledb_context import (
@@ -77,6 +78,8 @@ def create(
                 If the ``type`` is unsupported.
             ValueError:
                 If the ``shape`` is unsupported.
+            tiledbsoma.AlreadyExistsError:
+                If the underlying object already exists at the given URI.
             TileDBError:
                 If unable to create the underlying object.
 
@@ -91,11 +94,16 @@ def create(
             context,
             is_sparse=cls.is_sparse,
         )
-        handle = cls._create_internal(uri, schema, context, tiledb_timestamp)
-        return cls(
-            handle,
-            _dont_call_this_use_create_or_open_instead="tiledbsoma-internal-code",
-        )
+        try:
+            handle = cls._create_internal(uri, schema, context, tiledb_timestamp)
+            return cls(
+                handle,
+                _dont_call_this_use_create_or_open_instead="tiledbsoma-internal-code",
+            )
+        except tiledb.TileDBError as tdbe:
+            if is_already_exists_error(tdbe):
+                raise AlreadyExistsError(f"{uri!r} already exists")
+            raise
 
     @property
     def shape(self) -> Tuple[int, ...]:

apis/python/src/tiledbsoma/_dataframe.py

@@ -18,6 +18,7 @@
 from . import _arrow_types, _util
 from . import pytiledbsoma as clib
 from ._constants import SOMA_JOINID
+from ._exception import AlreadyExistsError, is_already_exists_error
 from ._query_condition import QueryCondition
 from ._read_iters import TableReadIter
 from ._tdb_handles import DataFrameWrapper
@@ -186,6 +187,8 @@ def create(
                 an undefined column name.
             ValueError:
                 If the ``schema`` specifies illegal column names.
+            tiledbsoma.AlreadyExistsError:
+                If the underlying object already exists at the given URI.
             TileDBError:
                 If unable to create the underlying object.
 
@@ -277,12 +280,16 @@ def create(
             platform_config=plt_cfg,
             timestamp=(0, timestamp_ms),
         )
-
-        handle = cls._wrapper_type.open(uri, "w", context, tiledb_timestamp)
-        return cls(
-            handle,
-            _dont_call_this_use_create_or_open_instead="tiledbsoma-internal-code",
-        )
+        try:
+            handle = cls._wrapper_type.open(uri, "w", context, tiledb_timestamp)
+            return cls(
+                handle,
+                _dont_call_this_use_create_or_open_instead="tiledbsoma-internal-code",
+            )
+        except tiledb.TileDBError as tdbe:
+            if is_already_exists_error(tdbe):
+                raise AlreadyExistsError(f"{uri!r} already exists")
+            raise
 
     def keys(self) -> Tuple[str, ...]:
         """Returns the names of the columns when read back as a dataframe.

apis/python/src/tiledbsoma/_exception.py

@@ -57,6 +57,36 @@ def is_does_not_exist_error(e: Union[RuntimeError, tiledb.TileDBError]) -> bool:
     return False
 
 
+class AlreadyExistsError(SOMAError):
+    """Raised when attempting to create an already existing SOMA object.
+
+    Lifecycle: experimental
+    """
+
+    pass
+
+
+def is_already_exists_error(e: tiledb.TileDBError) -> bool:
+    """Given a TileDBError, return true if it indicates the object already exists
+
+    Lifecycle: experimental
+
+    Example:
+        try:
+            tiledb.Array.create(uri, schema, ctx=ctx)
+                ...
+        except tiledb.TileDBError as e:
+            if is_already_exists_error(e):
+                ...
+            raise e
+    """
+    stre = str(e)
+    # Local-disk, S3, and TileDB Cloud exceptions all have the substring
+    # "already exists". Here we lower-case the exception message just
+    # in case someone ever uppercases it on the other end.
+    return "already exists" in stre.lower()
+
+
 def is_duplicate_group_key_error(e: tiledb.TileDBError) -> bool:
     """Given a TileDBError, return try if it indicates a duplicate member
     add request in a tiledb.Group.

apis/python/src/tiledbsoma/_general_utilities.py

@@ -5,13 +5,14 @@
 
 """General utility functions.
 """
-
+import os
 import platform
 import sys
+from re import fullmatch
 
 import tiledb
 
-from .pytiledbsoma import version as libtiledbsoma_version
+from .pytiledbsoma import version as libtiledbsoma_core_version_str
 
 
 def get_SOMA_version() -> str:
@@ -59,19 +60,75 @@ def get_storage_engine() -> str:
     return "tiledb"
 
 
+def get_tiledb_py_core_version() -> str:
+    """Returns the version of libtiledb ("core") used by the `tiledb` Python library.
+
+    Lifecycle: maturing
+    """
+    return ".".join(str(ijk) for ijk in list(tiledb.libtiledb.version()))
+
+
+def get_libtiledbsoma_core_version() -> str:
+    """Returns the version of libtiledb ("core") used by libtiledbsoma.
+
+    Lifecycle: maturing
+    """
+    v = libtiledbsoma_core_version_str()
+    m = fullmatch(r"libtiledb=(\d+\.\d+\.\d+)", v)
+    if m is None:
+        raise ValueError(f"Unexpected libtiledbsoma_core_version: {v}")
+    return m.group(1)
+
+
+# Set this env var to "err" to print an error to stderr when TileDB-Py's and libtiledbsoma's core
+# versions mismatch (by default, an AssertionError is raised).
+TILEDB_CORE_MISMATCHED_VERSIONS_ERROR_LEVEL_VAR = (
+    "TILEDB_CORE_MISMATCHED_VERSIONS_ERROR_LEVEL"
+)
+
+
+def verify_core_versions() -> None:
+    """Verify that the versions of libtiledb used by the `tiledb` Python library and
+    libtiledbsoma are the same.
+
+    See discussion on https://github.com/single-cell-data/TileDB-SOMA/issues/1837; this
+    will be unnecessary when libtiledbsoma is the only "path to core" (cf.
+    https://github.com/single-cell-data/TileDB-SOMA/issues/1632).
+
+    Lifecycle: maturing
+    """
+    tiledb_py_core_version = get_tiledb_py_core_version()
+    libtiledbsoma_core_version = get_libtiledbsoma_core_version()
+    if tiledb_py_core_version != libtiledbsoma_core_version:
+        msg = "libtiledb versions used by tiledb and libtiledbsoma differ: %s != %s" % (
+            tiledb_py_core_version,
+            libtiledbsoma_core_version,
+        )
+        if os.environ.get(TILEDB_CORE_MISMATCHED_VERSIONS_ERROR_LEVEL_VAR) == "err":
+            print(msg, file=sys.stderr)
+            print(
+                f"Continuing, since ${TILEDB_CORE_MISMATCHED_VERSIONS_ERROR_LEVEL_VAR} is set, but it is highly recommended you fix the core version mismatch, as undefined behavior and segfaults can result.",
+                file=sys.stderr,
+            )
+        else:
+            raise AssertionError(
+                f"libtiledb versions used by tiledb and libtiledbsoma differ: {tiledb_py_core_version} != {libtiledbsoma_core_version}"
+            )
+
+
 def show_package_versions() -> None:
     """Nominal use is for bug reports, so issue filers and issue fixers can be on
     the same page.
 
     Lifecycle: maturing
     """
-    print("tiledbsoma.__version__       ", get_implementation_version())
-    print("TileDB-Py tiledb.version()   ", tiledb.version())
-    print(
-        "TileDB core version          ",
-        ".".join(str(ijk) for ijk in list(tiledb.libtiledb.version())),
-    )
-    print("libtiledbsoma version()      ", libtiledbsoma_version())
-    print("python version               ", ".".join(str(v) for v in sys.version_info))
     u = platform.uname()
-    print("OS version                   ", u.system, u.release)
+    # fmt: off
+    print("tiledbsoma.__version__             ", get_implementation_version())
+    print("TileDB-Py version                  ", ".".join(str(v) for v in tiledb.version()))
+    print("TileDB core version (tiledb)       ", get_tiledb_py_core_version())
+    print("TileDB core version (libtiledbsoma)", get_libtiledbsoma_core_version())
+    print("python version                     ", ".".join(str(v) for v in sys.version_info))
+    print("OS version                         ", u.system, u.release)
+    # fmt: on
+    verify_core_versions()