Merge remote-tracking branch 'origin/dev' into db/sc-50707/aarch64

.github/workflows/build-wheels.yml

@@ -30,11 +30,11 @@ jobs:
       matrix:
         buildplat:
           - [ubuntu-22.04, manylinux_x86_64]
-          - [ubuntu-22.04, manylinux_aarch64]
+          - [linux-arm64-ubuntu24, manylinux_aarch64]
           - [macos-13, macosx_x86_64]
           - [macos-14, macosx_arm64]
           - [windows-2022, win_amd64]
-        python: ["cp39", "cp310", "cp311", "cp312"]
+        python: ["cp39", "cp310", "cp311", "cp312", "cp313"]
 
     steps:
       - uses: actions/checkout@v4
@@ -53,7 +53,7 @@ jobs:
           brew install automake pkg-config ninja llvm
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.18.1
+        uses: pypa/cibuildwheel@v2.21.3
         env:
           CIBW_BUILD_VERBOSITY: 3
           CIBW_ENVIRONMENT_PASS_LINUX: SETUPTOOLS_SCM_PRETEND_VERSION_FOR_TILEDB S3_BUCKET TILEDB_TOKEN TILEDB_NAMESPACE
@@ -106,8 +106,8 @@ jobs:
           - macos-14
           - windows-2022
           - ubuntu-22.04
-          # Add linux-aarch64 when available
-        python: ["3.9", "3.10", "3.11", "3.12"]
+          - linux-arm64-ubuntu24
+        python: ["3.9", "3.10", "3.11", "3.12", "3.13"]
     runs-on: ${{ matrix.os }}
     steps:
       - name: Set up Python ${{ matrix.python }}

.github/workflows/ci-tiledb-from-source.yml

@@ -0,0 +1,94 @@
+name: TileDB Python CI Using TileDB Core Source Build
+
+on:
+  workflow_dispatch:
+    inputs:
+      libtiledb_ref:
+        default: dev
+        type: string
+      libtiledb_version:
+        type: string
+
+jobs:
+
+  build_libtiledb:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout TileDB Core ${{ inputs.libtiledb_ref || 'dev' }}
+        uses: actions/checkout@v4
+        with:
+          repository: TileDB-Inc/TileDB
+          ref: ${{ inputs.libtiledb_ref || 'dev' }}
+
+      - name: Configure TileDB
+        run: |
+          cmake -S . -B build \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DBUILD_SHARED_LIBS=ON \
+            -DCMAKE_INSTALL_PREFIX=./dist \
+            -DTILEDB_INSTALL_LIBDIR=lib \
+            -DTILEDB_S3=ON \
+            -DTILEDB_AZURE=ON \
+            -DTILEDB_GCS=ON \
+            -DTILEDB_HDFS=ON \
+            -DTILEDB_SERIALIZATION=ON \
+            -DTILEDB_WEBP=ON \
+            -DTILEDB_TESTS=OFF \
+            -DVCPKG_TARGET_TRIPLET=x64-linux-release
+
+      - name: Build TileDB
+        env:
+          TILEDB_PACKAGE_VERSION: ${{ inputs.libtiledb_version || '0.1' }}
+        run: cmake --build build --config Release --target package
+
+      - name: Upload TileDB Core Artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: libtiledb
+          path: |
+            build/tiledb-*.tar.gz*
+            build/tiledb-*.zip*
+
+  build_tiledb_py:
+    needs:
+      - build_libtiledb
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Checkout TileDB-Py
+        uses: actions/checkout@v4
+
+      - name: Download TileDB Core Artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: libtiledb
+          path: ${{ github.workspace }}/libtiledb
+
+      - name: Unpack Release Archive
+        run: tar xvf ${{ github.workspace }}/libtiledb/*.tar.gz --directory ${{ github.workspace }}/libtiledb
+
+      - name: Build TileDB-Py Wheel
+        env:
+          TILEDB_PATH: ${{ github.workspace }}/libtiledb
+        run: |
+          python -m pip wheel -w dist --verbose .
+          WHEEL=$(ls dist/tiledb-*.whl)
+          python -m pip install ${WHEEL}[test]
+
+      - name: Upload TileDB Core Artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: tiledb-py
+          path: |
+            dist/tiledb-*.whl
+
+      - name: Run tests
+        run: |
+          PROJECT_CWD=$PWD
+          rm tiledb/__init__.py
+          cd /tmp
+          pytest -vv --showlocals $PROJECT_CWD

.github/workflows/ci.yml

@@ -1,11 +1,6 @@
 name: TileDB Python CI
 
-on: 
-  push:
-    branches: [dev]
-  pull_request:
-    branches: [dev]
-  workflow_dispatch:
+on: [push, pull_request, workflow_dispatch]
 
 concurrency:
   group: ${{ github.head_ref || github.run_id }}
@@ -30,7 +25,7 @@ jobs:
           # libfaketime tests fail on macos arm. Disable tests for now.
           # - macos-14
           - windows-latest
-        python-version: [ "3.9", "3.10", "3.11", "3.12"]
+        python-version: [ "3.9", "3.10", "3.11", "3.12", "3.13" ]
       fail-fast: false
     env:
       MACOSX_DEPLOYMENT_TARGET: "11"

.github/workflows/daily-test-build-numpy.yml

@@ -22,9 +22,11 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-13, macos-14, windows-latest]
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
         include:
           # https://github.com/scipy/oldest-supported-numpy/blob/main/setup.cfg
+          - python-version: "3.13"
+            numpy-version: "2.1.0"
           - python-version: "3.12"
             numpy-version: "1.26.4"
           - python-version: "3.12"

CMakeLists.txt

@@ -46,8 +46,8 @@ if (NOT TileDB_FOUND)
         message(STATUS "Downloading TileDB default version ...")
         # Download latest release
         fetch_prebuilt_tiledb(
-                VERSION 2.26.1
-                RELLIST_HASH SHA256=256216aa989015397f4efbbd319ebeccfead568baa73611aa0c1c0fcea35f8d5
+                VERSION 2.26.2
+                RELLIST_HASH SHA256=86c19d7c5246cb18e370a4272cead63ea84bd651789842e618de4d57d4510522
         )
     endif()
     find_package(TileDB REQUIRED)

HISTORY.md

@@ -1,3 +1,60 @@
+# Release 0.32.5
+
+* TileDB-Py 0.32.5 includes TileDB Embedded [2.26.2](https://github.com/TileDB-Inc/TileDB/releases/tag/2.26.2)
+
+## Improvements
+
+* Fix GroupMetadata backwards compatibility by @kounelisagis in https://github.com/TileDB-Inc/TileDB-Py/pull/2102
+* Fix typo in API doc for create_bucket by @nickvigilante in https://github.com/TileDB-Inc/TileDB-Py/pull/2101
+* Update enumerations exception message by @shaunrd0 in https://github.com/TileDB-Inc/TileDB-Py/pull/2096
+* Handle removal of 'StorageManager' from stats dumps in 2.27 - Part 2 by @kounelisagis https://github.com/TileDB-Inc/TileDB-Py/pull/2098
+
+## Build system changes
+
+* Revert "Do not run CI in PR twice (#2082)" by @kounelisagis in https://github.com/TileDB-Inc/TileDB-Py/pull/2100
+
+# Release 0.32.4
+
+TileDB-Py 0.32.4 was inadvertently released against TileDB [2.26.1](https://github.com/TileDB-Inc/TileDB/releases/tag/2.26.1). This will be corrected in 0.32.5, but the version delta does not justify yanking 0.32.4
+
+## Improvements
+
+* Add `TILEDB_DATETIME_DAY` type support for Arrow by @kounelisagis in https://github.com/TileDB-Inc/TileDB-Py/pull/2002
+* Extend the `GroupMetadata` functionality to support NumPy arrays by @kounelisagis in https://github.com/TileDB-Inc/TileDB-Py/pull/2085
+* Add extra argument to Group::add_member by @kounelisagis in https://github.com/TileDB-Inc/TileDB-Py/pull/2093
+* Handle removal of 'StorageManager' from stats dumps in 2.27 by @kounelisagis in https://github.com/TileDB-Inc/TileDB-Py/pull/2088
+* Make default value for `vfs.s3.region` in test conditional by @kounelisagis in https://github.com/TileDB-Inc/TileDB-Py/pull/2086
+* Make error message for non-existing `Enumeration` in test conditional by @kounelisagis in https://github.com/TileDB-Inc/TileDB-Py/pull/2087
+* Fix randomly failing test `DenseArrayTest::test_open_with_timestamp[False]` by @kounelisagis in https://github.com/TileDB-Inc/TileDB-Py/pull/2090
+* Fix skipif condition for test_cloud by @kounelisagis in https://github.com/TileDB-Inc/TileDB-Py/pull/2084
+* Raise error when sparse=True is passed to `tiledb.from_numpy` by @kounelisagis in https://github.com/TileDB-Inc/TileDB-Py/pull/2080
+* Documentation and CI updates by @kounelisagis in https://github.com/TileDB-Inc/TileDB-Py/pull/2078
+* Move `DenseArrayImpl` to pure Python by @kounelisagis and @nguyenv in https://github.com/TileDB-Inc/TileDB-Py/pull/2071
+* Move `PackedBuffer` and `pack_metadata_val` to pure Python, and wrap `tiledb_datatype_size` with pybind by @kounelisagis in https://github.com/TileDB-Inc/TileDB-Py/pull/2072
+* Move `SparseArrayImpl` to pure Python by @kounelisagis in https://github.com/TileDB-Inc/TileDB-Py/pull/2065
+
+## Build system changes
+
+* Add CI to test against the TileDB core library built from source with a custom version by @dudoslav and @kounelisagis in https://github.com/TileDB-Inc/TileDB-Py/pull/2091
+* Do not run CI in PR twice by @dudoslav in https://github.com/TileDB-Inc/TileDB-Py/pull/2082
+* Disable release for Python 3.8 by @dudoslav in https://github.com/TileDB-Inc/TileDB-Py/pull/2081
+* Re-enable automatic upload to pypi by @dudoslav and @kounelisagis in https://github.com/TileDB-Inc/TileDB-Py/pull/2034
+* Drop Python 3.8 support by @kounelisagis in https://github.com/TileDB-Inc/TileDB-Py/pull/2079
+
+# Release 0.32.3
+
+* TileDB-Py 0.32.3 includes TileDB Embedded [2.26.2](https://github.com/TileDB-Inc/TileDB/releases/tag/2.26.2)
+
+## Build system changes
+
+* Override tag version in manylinux container by @dudoslav in https://github.com/TileDB-Inc/TileDB-Py/pull/2077
+
+# Release 0.32.2
+
+## Improvements
+
+* Fix object_type return value by @kounelisagis in https://github.com/TileDB-Inc/TileDB-Py/pull/2073
+
 # Release 0.32.1
 
 * TileDB-Py 0.32.1 includes TileDB Embedded [2.26.1](https://github.com/TileDB-Inc/TileDB/releases/tag/2.26.1)

pyproject.toml

@@ -35,6 +35,7 @@ classifiers=[
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
 ]
 dependencies = [
   "numpy>=1.25",
@@ -75,7 +76,7 @@ sdist.include = ["tiledb/_generated_version.py"]
 TILEDB_PATH = {env="TILEDB_PATH"}
 TILEDB_VERSION = {env="TILEDB_VERSION"}
 TILEDB_HASH = {env="TILEDB_HASH"}
-TILEDB_REMOVE_DEPRECATIONS = "ON"
+TILEDB_REMOVE_DEPRECATIONS = "OFF"
 TILEDB_SERIALIZATION = "OFF"
 
 [tool.pytest.ini_options]

tiledb/array_schema.py

@@ -396,11 +396,9 @@ def current_domain(self) -> CurrentDomain:
 
             :rtype: tiledb.CurrentDomain
             """
-            curr_dom = CurrentDomain.from_pybind11(
+            return CurrentDomain.from_pybind11(
                 self._ctx, self._current_domain(self._ctx)
             )
-            curr_dom._set_domain(self.domain)
-            return curr_dom
 
         def set_current_domain(self, current_domain):
             """Set the current domain

tiledb/cc/common.cc

@@ -192,18 +192,20 @@ bool is_tdb_str(tiledb_datatype_t type) {
 }
 
 py::size_t get_ncells(py::dtype type) {
-  if (type.is(py::dtype("S")))
-    return type.itemsize() == 0 ? TILEDB_VAR_NUM : type.itemsize();
-
-  if (type.is(py::dtype("U"))) {
-    auto np_unicode_size = py::dtype("U").itemsize();
-    return type.itemsize() == 0 ? TILEDB_VAR_NUM
-                                : type.itemsize() / np_unicode_size;
-  }
-
   auto np = py::module::import("numpy");
   auto np_issubdtype = np.attr("issubdtype");
   auto np_complexfloating = np.attr("complexfloating");
+  auto np_character = np.attr("character");
+
+  py::bool_ ischaracter = np_issubdtype(type, np_character);
+  if (ischaracter) {
+    py::dtype base_dtype =
+        np.attr("dtype")(py::make_tuple(type.attr("kind"), 1));
+    if (type.itemsize() == 0)
+      return TILEDB_VAR_NUM;
+    return type.itemsize() / base_dtype.itemsize();
+  }
+
   py::bool_ iscomplexfloating = np_issubdtype(type, np_complexfloating);
   if (iscomplexfloating)
     return 2;