Skip to content

Commit

Permalink
Bulk sync main to release-1.15 (#3374)
Browse files Browse the repository at this point in the history
  • Loading branch information
johnkerl authored Nov 22, 2024
1 parent d41a025 commit a58df6a
Show file tree
Hide file tree
Showing 622 changed files with 17,553 additions and 8,773 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/backport.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ on:

jobs:
backport:
runs-on: ubuntu-latest
runs-on: ubuntu-24.04
name: Backport
steps:
- name: Backport
Expand Down
19 changes: 1 addition & 18 deletions .github/workflows/pkgdown.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ name: pkgdown

jobs:
pkgdown:
runs-on: ubuntu-latest
runs-on: ubuntu-24.04
# This is GitHub Actions magic -- there are no secrets we as package owners need to set up or manage
env:
GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
Expand All @@ -31,23 +31,6 @@ jobs:
with:
use-public-rspm: true

- name: Set additional repositories (macOS)
if: ${{ matrix.os != 'ubuntu-latest' }}
run: echo 'options(repos = c("https://tiledb-inc.r-universe.dev", getOption("repos")), timeout = 300L)' | tee -a ~/.Rprofile

- name: Set additional repositories (Linux)
if: ${{ matrix.os == 'ubuntu-latest' }}
run: |
rversion <- paste(strsplit(as.character(getRversion()), split = '\\.')[[1L]][1:2], collapse = '.')
codename <- system('. /etc/os-release; echo ${VERSION_CODENAME}', intern = TRUE)
repo <- "https://tiledb-inc.r-universe.dev"
(opt <- sprintf('options(repos = c("%s/bin/linux/%s/%s", "%s", getOption("repos")), timeout = 300L)', repo, codename, rversion, repo))
cat(opt, "\n", file = "~/.Rprofile", append = TRUE)
shell: Rscript {0}

- name: Install tiledb-r
run: cd apis/r && Rscript tools/install-tiledb-r.R

# Run this daily to surface errors as tracked at
# https://github.com/single-cell-data/TileDB-SOMA/issues/2052
- name: Install dependencies
Expand Down
14 changes: 7 additions & 7 deletions .github/workflows/python-ci-full.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,15 @@ jobs:
fail-fast: false
matrix:
# TODO: decide on Windows CI coverage
os: [ubuntu-latest, macos-latest]
# os: [ubuntu-latest, macos-latest, windows-2019]
os: [ubuntu-24.04, macos-latest]
# os: [ubuntu-24.04, macos-latest, windows-2019]
# TODO: add 3.12
# https://github.com/single-cell-data/TileDB-SOMA/issues/1849
python-version: ['3.9', '3.10', '3.11', '3.12']
include:
- runs-on: ubuntu-latest
cc: gcc-11
cxx: g++-11
- runs-on: ubuntu-24.04
cc: gcc-13
cxx: g++-13
- runs-on: macos-latest
cc: clang
cxx: clang++
Expand All @@ -39,6 +39,6 @@ jobs:
python_version: ${{ matrix.python-version }}
cc: ${{ matrix.cc }}
cxx: ${{ matrix.cxx }}
report_codecov: ${{ matrix.os == 'ubuntu-latest' && matrix.python-version == '3.11' }}
run_lint: ${{ matrix.os == 'ubuntu-latest' && matrix.python-version == '3.11' }}
report_codecov: ${{ matrix.os == 'ubuntu-24.04' && matrix.python-version == '3.11' }}
run_lint: ${{ matrix.os == 'ubuntu-24.04' && matrix.python-version == '3.11' }}
secrets: inherit
8 changes: 4 additions & 4 deletions .github/workflows/python-ci-minimal.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,12 +28,12 @@ jobs:
fail-fast: true

matrix:
os: [ubuntu-latest, macos-latest]
os: [ubuntu-24.04, macos-latest]
python-version: ['3.9', '3.12']
include:
- os: ubuntu-latest
cc: gcc-11
cxx: g++-11
- os: ubuntu-24.04
cc: gcc-13
cxx: g++-13
- os: macos-latest
cc: clang
cxx: clang++
Expand Down
24 changes: 14 additions & 10 deletions .github/workflows/python-ci-packaging.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ jobs:
# Confirm shared object copying when building the Python package
# https://github.com/single-cell-data/TileDB-SOMA/pull/1937
docker:
runs-on: ubuntu-latest
runs-on: ubuntu-24.04
name: "SO copying (docker) TILEDB_EXISTS: ${{ matrix.TILEDB_EXISTS }} TILEDBSOMA_EXISTS: ${{ matrix.TILEDBSOMA_EXISTS }}"
strategy:
fail-fast: false
Expand Down Expand Up @@ -76,7 +76,7 @@ jobs:
run: |
mkdir -p external
# Please do not edit manually -- let scripts/update-tiledb-version.py update this
wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.26.2/tiledb-linux-x86_64-2.26.2-30fc114.tar.gz
wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.27.0-rc3/tiledb-linux-x86_64-2.27.0-rc3-8d581f2.tar.gz
tar -C external -xzf tiledb-linux-x86_64-*.tar.gz
ls external/lib/
echo "LD_LIBRARY_PATH=$(pwd)/external/lib" >> $GITHUB_ENV
Expand Down Expand Up @@ -152,7 +152,7 @@ jobs:
# https://github.com/single-cell-data/TileDB-SOMA/pull/1937
# Same as job above, but running on macOS instead of in a Docker container
macos:
runs-on: macos-13
runs-on: macos-latest
name: "SO copying (macos) TILEDB_EXISTS: ${{ matrix.TILEDB_EXISTS }} TILEDBSOMA_EXISTS: ${{ matrix.TILEDBSOMA_EXISTS }}"
strategy:
fail-fast: false
Expand All @@ -168,16 +168,20 @@ jobs:
fetch-depth: 0 # for setuptools-scm
- name: Check if System Integrity Protection (SIP) is enabled
run: csrutil status
- name: Select XCode version
uses: maxim-lobanov/setup-xcode@v1
with:
xcode-version: '15.4'
- name: Install pre-built libtiledb
if: ${{ matrix.TILEDB_EXISTS == 'yes' }}
run: |
mkdir -p external
# Please do not edit manually -- let scripts/update-tiledb-version.py update this
if [ `uname -m` == "arm64" ]; then
wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.26.2/tiledb-macos-arm64-2.26.2-30fc114.tar.gz
wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.27.0-rc3/tiledb-macos-arm64-2.27.0-rc3-8d581f2.tar.gz
tar -C external -xzf tiledb-macos-arm64-*.tar.gz
else
wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.26.2/tiledb-macos-x86_64-2.26.2-30fc114.tar.gz
wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.27.0-rc3/tiledb-macos-x86_64-2.27.0-rc3-8d581f2.tar.gz
tar -C external -xzf tiledb-macos-x86_64-*.tar.gz
fi
ls external/lib/
Expand Down Expand Up @@ -259,7 +263,7 @@ jobs:
strategy:
fail-fast: false
matrix:
os: ["ubuntu-latest", "macos-latest"]
os: ["ubuntu-24.04", "macos-latest"]
steps:
- uses: actions/checkout@v4
with:
Expand All @@ -270,14 +274,14 @@ jobs:
if [ `uname -s` == "Darwin" ]; then
if [ `uname -m` == "arm64" ]; then
# Please do not edit manually -- let scripts/update-tiledb-version.py update this
wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.26.2/tiledb-macos-arm64-2.26.2-30fc114.tar.gz
wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.27.0-rc3/tiledb-macos-arm64-2.27.0-rc3-8d581f2.tar.gz
else
# Please do not edit manually -- let scripts/update-tiledb-version.py update this
wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.26.2/tiledb-macos-x86_64-2.26.2-30fc114.tar.gz
wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.27.0-rc3/tiledb-macos-x86_64-2.27.0-rc3-8d581f2.tar.gz
fi
else
# Please do not edit manually -- let scripts/update-tiledb-version.py update this
wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.26.2/tiledb-linux-x86_64-2.26.2-30fc114.tar.gz
wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.27.0-rc3/tiledb-linux-x86_64-2.27.0-rc3-8d581f2.tar.gz
fi
tar -C external -xzf tiledb-*.tar.gz
ls external/lib/
Expand Down Expand Up @@ -364,7 +368,7 @@ jobs:
strategy:
fail-fast: false
matrix:
os: ["ubuntu-latest", "macos-latest"]
os: ["ubuntu-24.04", "macos-latest"]
steps:
- uses: actions/checkout@v4
with:
Expand Down
13 changes: 3 additions & 10 deletions .github/workflows/python-ci-single.yml
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ jobs:
run: echo "inputs.os:" ${{ inputs.os }}

- name: Linux CPU info
if: ${{ inputs.os == 'ubuntu-latest' }}
if: ${{ inputs.os == 'ubuntu-24.04' }}
run: cat /proc/cpuinfo

- name: MacOS CPU info
Expand Down Expand Up @@ -104,7 +104,7 @@ jobs:
# key: libtiledbsoma-build-dist-${{ inputs.os }}-${{ inputs.python_version }}-${{ hashFiles('libtiledbsoma', 'scripts/bld') }}

- name: Install tiledbsoma
run: pip -v install -e apis/python[dev] -C "--build-option=--no-tiledb-deprecated"
run: pip -v install -e apis/python[all] -C "--build-option=--no-tiledb-deprecated"
env:
CC: ${{ inputs.cc }}
CXX: ${{ inputs.cxx }}
Expand All @@ -128,20 +128,13 @@ jobs:
env:
TILEDB_SOMA_INIT_BUFFER_BYTES: 33554432 # accommodate tiny runners

- name: Run pytests for Python without new shape
- name: Run pytests for Python
shell: bash
# Setting PYTHONPATH ensures the tests load the in-tree source code under apis/python/src
# instead of the copy we `pip install`ed to site-packages above. That's needed for the code
# coverage analysis to work.
run: PYTHONPATH=$(pwd)/apis/python/src python -m pytest --cov=apis/python/src --cov-report=xml apis/python/tests -v --durations=20 --maxfail=50

- name: Run pytests for Python with new shape
shell: bash
# Setting PYTHONPATH ensures the tests load the in-tree source code under apis/python/src
# instead of the copy we `pip install`ed to site-packages above. That's needed for the code
# coverage analysis to work.
run: export SOMA_PY_NEW_SHAPE=true; PYTHONPATH=$(pwd)/apis/python/src python -m pytest --cov=apis/python/src --cov-report=xml apis/python/tests -v --durations=20 --maxfail=50

- name: Report coverage to Codecov
if: inputs.report_codecov
uses: codecov/codecov-action@v4
Expand Down
36 changes: 24 additions & 12 deletions .github/workflows/python-packaging.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ on:
jobs:
sdist:
name: Build source distribution
runs-on: ubuntu-latest
runs-on: ubuntu-24.04
steps:
- name: Checkout TileDB-SOMA
uses: actions/checkout@v4
Expand Down Expand Up @@ -54,8 +54,8 @@ jobs:
cibw_build: [ manylinux_x86_64, macosx_x86_64, macosx_arm64 ]
include:
- cibw_build: manylinux_x86_64
os: ubuntu-latest
wheel-name: manylinux2014
os: ubuntu-24.04
wheel-name: manylinux_2_28
- cibw_build: macosx_x86_64
os: macos-latest
cibw_archs_macos: x86_64
Expand Down Expand Up @@ -84,6 +84,11 @@ jobs:
pip install setuptools
python -c 'from distutils import util; print("distutil.util.get_platform:", util.get_platform())'
python -c 'import platform; print("platform.platform()", platform.platform())'
- name: Select XCode version
if: startsWith(matrix.os, 'macos')
uses: maxim-lobanov/setup-xcode@v1
with:
xcode-version: '15.4'
- name: Build wheels
uses: pypa/[email protected]
with:
Expand All @@ -92,12 +97,19 @@ jobs:
env:
CIBW_BUILD: ${{ matrix.cibw_build }}
CIBW_BUILD_VERBOSITY: 1
CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28
CIBW_BEFORE_BUILD_LINUX: yum -y remove gcc-toolset-12\*; yum -y install gcc-toolset-13; bash -x -c 'rm -rf tiledbsoma*/dist_links/dist/lib*'
# ^ Delete lib folder that apis/python/setup.py:find_or_build() looks for in deciding to
# run CMake build or not. Otherwise it'll keep reusing the library file built in the
# first iteration of cibuildwheel's outer loop, resulting in wheels with the library
# built for the wrong python version.
CIBW_BEFORE_BUILD: bash -x -c 'rm -rf tiledbsoma*/dist_links/dist/lib*'
# ^ Delete lib folder that apis/python/setup.py:find_or_build() looks for in deciding to
# run CMake build or not. Otherwise it'll keep reusing the library file built in the
# first iteration of cibuildwheel's outer loop, resulting in wheels with the library
# built for the wrong python version.
CIBW_ARCHS_MACOS: ${{ matrix.cibw_archs_macos }}
CIBW_ENVIRONMENT_LINUX : CC=/opt/rh/gcc-toolset-13/root/usr/bin/gcc CXX=/opt/rh/gcc-toolset-13/root/usr/bin/g++
CIBW_TEST_SKIP: "*_arm64"
CMAKE_OSX_ARCHITECTURES: ${{ matrix.cibw_archs_macos }}
MACOSX_DEPLOYMENT_TARGET: "11.0"
Expand All @@ -122,15 +134,15 @@ jobs:
- undotted-version: '312'
dotted-version: '3.12'
wheel-name:
- manylinux2014
- manylinux_2_28
- macos-x86_64
- macos-arm64
include:
- wheel-name: manylinux2014
os: ubuntu-latest
- wheel-name: manylinux_2_28
os: ubuntu-24.04
arch: x86_64
cc: gcc-11
cxx: g++-11
cc: gcc-13
cxx: g++-13
- wheel-name: macos-x86_64
os: macos-13
arch: x86_64
Expand Down Expand Up @@ -166,7 +178,7 @@ jobs:
run: python -c 'import tiledbsoma; print(tiledbsoma.pytiledbsoma.__file__); tiledbsoma.show_package_versions()'
# TODO: more thorough local smoke test
- name: Smoke test in docker
if: ${{ matrix.os == 'ubuntu-latest' }}
if: ${{ matrix.os == 'ubuntu-24.04' }}
run: |
docker run -v $(pwd):/mnt python:${{ matrix.python.dotted-version }} bash -ec "
apt-get -qq update && apt-get install -y python3-pip python3-wheel
Expand All @@ -178,7 +190,7 @@ jobs:
publish-to-test-pypi:
name: Publish package to TestPyPI
needs: smoke-test
runs-on: ubuntu-latest
runs-on: ubuntu-24.04
if: github.event_name == 'workflow_dispatch'
steps:
- name: Download artifacts
Expand All @@ -202,7 +214,7 @@ jobs:
publish-to-pypi:
name: Publish package to PyPI
needs: smoke-test
runs-on: ubuntu-latest
runs-on: ubuntu-24.04
if: github.event_name == 'release'
steps:
- name: Download artifacts
Expand All @@ -223,7 +235,7 @@ jobs:
# File a bug report if anything fails, but don't file tickets for manual runs
# -- only for scheduled ones.
create_issue_on_fail:
runs-on: ubuntu-latest
runs-on: ubuntu-24.04
needs: [smoke-test, publish-to-test-pypi, publish-to-pypi]
if: (failure() || cancelled()) && github.event_name != 'workflow_dispatch'
steps:
Expand Down
Loading

0 comments on commit a58df6a

Please sign in to comment.