handle more RAPIDS version formats in update-version.sh, refactor dependencies.yaml

ci/release/update-version.sh

@@ -57,9 +57,9 @@ DEPENDENCIES=(
 )
 for DEP in "${DEPENDENCIES[@]}"; do
   for FILE in dependencies.yaml conda/environments/*.yaml; do
-    sed_runner "/-.* ${DEP}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}\.*/g" ${FILE};
+    sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}.*/g" "${FILE}"
   done
-  sed_runner "/\"${DEP}==/ s/==.*\"/==${NEXT_SHORT_TAG_PEP440}\.*\"/g" python/pyproject.toml;
+  sed_runner "/\"${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*\"/==${NEXT_SHORT_TAG_PEP440}\.*\"/g" python/pyproject.toml;
 done
 
 # rapids-cmake version

conda/environments/all_cuda-118_arch-x86_64.yaml

@@ -18,10 +18,8 @@ dependencies:
 - cupy>=12.0.0
 - cxx-compiler
 - cython>=3.0.0
-- dask
 - dask-cuda==24.4.*
 - dask-cudf==24.4.*
-- distributed
 - doxygen=1.10.0
 - fmt>=10.1.1,<11
 - gmock>=1.13.0
@@ -39,8 +37,11 @@ dependencies:
 - pytest-asyncio
 - pytest-rerunfailures
 - python>=3.9,<3.11
+- rapids-dask-dependency==24.4.*
 - rmm==24.4.*
 - scikit-build-core>=0.7.0
+- setuptools>=64.0.0
 - spdlog>=1.12.0,<1.13
+- tomli
 - ucx
 name: all_cuda-118_arch-x86_64

conda/environments/all_cuda-120_arch-x86_64.yaml

@@ -18,10 +18,8 @@ dependencies:
 - cupy>=12.0.0
 - cxx-compiler
 - cython>=3.0.0
-- dask
 - dask-cuda==24.4.*
 - dask-cudf==24.4.*
-- distributed
 - doxygen=1.10.0
 - fmt>=10.1.1,<11
 - gmock>=1.13.0
@@ -39,8 +37,11 @@ dependencies:
 - pytest-asyncio
 - pytest-rerunfailures
 - python>=3.9,<3.11
+- rapids-dask-dependency==24.4.*
 - rmm==24.4.*
 - scikit-build-core>=0.7.0
+- setuptools>=64.0.0
 - spdlog>=1.12.0,<1.13
+- tomli
 - ucx
 name: all_cuda-120_arch-x86_64

dependencies.yaml

@@ -7,17 +7,21 @@ files:
       arch: [x86_64]
     includes:
       - build_cpp
-      - build_python
+      - build_python_ucxx
+      - build_python_distributed_ucxx
       - checks
       - cuda
       - cuda_version
       - dev
       - docs
       - py_version
-      - run_python
+      - run_python_ucxx
+      - run_python_distributed_ucxx
       - test_cpp
-      - test_python
+      - test_python_ucxx
+      - test_python_distributed_ucxx
       - depends_on_cupy
+      - depends_on_rmm
       - depends_on_cudf
   test_cpp:
     output: none
@@ -29,43 +33,64 @@ files:
     includes:
       - cuda_version
       - py_version
-      - test_python
+      - test_python_ucxx
+      - test_python_distributed_ucxx
       - depends_on_cupy
       - depends_on_cudf
   checks:
     output: none
     includes:
       - checks
       - py_version
-  docs:
-    output: none
-    includes:
-      - docs
-      - py_version
-  py_build:
+  py_build_ucxx:
     output: pyproject
     pyproject_dir: python
     extras:
       table: build-system
     includes:
-      - build_python
-  py_run:
+      - build_python_ucxx
+      - depends_on_rmm
+  py_run_ucxx:
     output: pyproject
     pyproject_dir: python
     extras:
       table: project
     includes:
-      - run_python
-  py_test:
+      - run_python_ucxx
+      - depends_on_rmm
+  py_test_ucxx:
     output: pyproject
     pyproject_dir: python
     extras:
       table: project.optional-dependencies
       key: test
     includes:
-      - test_python
+      - test_python_ucxx
       - depends_on_cupy
       - depends_on_cudf
+  py_build_distributed_ucxx:
+    output: pyproject
+    pyproject_dir: python/distributed-ucxx
+    extras:
+      table: build-system
+    includes:
+      - build_python_distributed_ucxx
+  py_run_distributed_ucxx:
+    output: pyproject
+    pyproject_dir: python/distributed-ucxx
+    extras:
+      table: project
+    includes:
+      - run_python_distributed_ucxx
+      - depends_on_ucxx
+  py_test_distributed_ucxx:
+    output: pyproject
+    pyproject_dir: python/distributed-ucxx
+    extras:
+      table: project.optional-dependencies
+      key: test
+    includes:
+      - test_python_distributed_ucxx
 channels:
   - rapidsai
   - rapidsai-nightly
@@ -86,11 +111,8 @@ dependencies:
           - librmm==24.4.*
           - ninja
           - spdlog>=1.12.0,<1.13
-  build_python:
+  build_python_ucxx:
     common:
-      - output_types: [conda]
-        packages:
-          - &rmm_conda rmm==24.4.*
       - output_types: [conda, requirements, pyproject]
         packages:
           - *cmake_ver
@@ -102,16 +124,12 @@ dependencies:
       - output_types: [requirements, pyproject]
         packages:
           - scikit-build-core[pyproject]>=0.7.0
-    specific:
-      - output_types: [requirements, pyproject]
-        matrices:
-          - matrix: {cuda: "12.*"}
-            packages:
-              - rmm-cu12==24.4.*
-          - matrix: {cuda: "11.*"}
-            packages:
-              - rmm-cu11==24.4.*
-          - {matrix: null, packages: [*rmm_conda]}
+  build_python_distributed_ucxx:
+    common:
+      - output_types: [conda, requirements, pyproject]
+        packages:
+          - setuptools>=64.0.0
+          - tomli
   checks:
     common:
       - output_types: [conda, requirements]
@@ -190,48 +208,78 @@ dependencies:
           - matrix:
             packages:
               - python>=3.9,<3.11
-  run_python:
+  run_python_ucxx:
     common:
       - output_types: [conda, requirements, pyproject]
         packages:
-          - numpy>=1.21
+          - &numpy numpy>=1.21
           - pynvml>=11.4.1
       - output_types: [conda]
         packages:
           - ucx
+  run_python_distributed_ucxx:
+    common:
+      - output_types: [conda, requirements, pyproject]
+        packages:
+          - &numba numba>=0.57.1
+          - rapids-dask-dependency==24.4.*
   test_cpp:
     common:
       - output_types: conda
         packages:
           - *cmake_ver
           - *gtest
           - *gmock
-  test_python:
+  test_python_ucxx:
     common:
       - output_types: [conda, requirements, pyproject]
         packages:
           - cloudpickle
-          - dask
-          - distributed
-          - numba>=0.57.1
+          - *numba
           - pytest
           - pytest-asyncio
           - pytest-rerunfailures
+  test_python_distributed_ucxx:
+    common:
+      - output_types: [conda, requirements, pyproject]
+        packages:
+          - *numpy
+          - pytest
   depends_on_cupy:
     common:
       - output_types: conda
         packages:
-          - &cupy_conda cupy>=12.0.0
+          - cupy>=12.0.0
     specific:
       - output_types: [requirements, pyproject]
         matrices:
           - matrix: {cuda: "12.*"}
             packages:
               - cupy-cuda12x>=12.0.0
           - matrix: {cuda: "11.*"}
-            packages:
+            packages: &cupy_packages_cu11
               - cupy-cuda11x>=12.0.0
-          - {matrix: null, packages: [*cupy_conda]}
+          - {matrix: null, packages: *cupy_packages_cu11}
+  depends_on_rmm:
+    common:
+      - output_types: conda
+        packages:
+          - &rmm_conda rmm==24.4.*
+      - output_types: requirements
+        packages:
+          # pip recognizes the index as a global option for the requirements.txt file
+          - --extra-index-url=https://pypi.nvidia.com
+          - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple
+    specific:
+      - output_types: [requirements, pyproject]
+        matrices:
+          - matrix: {cuda: "12.*"}
+            packages:
+              - rmm-cu12==24.4.*
+          - matrix: {cuda: "11.*"}
+            packages:
+              - rmm-cu11==24.4.*
+          - {matrix: null, packages: [*rmm_conda]}
   depends_on_cudf:
     common:
       - output_types: conda
@@ -240,7 +288,6 @@ dependencies:
       - output_types: requirements
         packages:
           # pip recognizes the index as a global option for the requirements.txt file
-          # This index is needed for rmm, cubinlinker, ptxcompiler.
           - --extra-index-url=https://pypi.nvidia.com
           - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple
     specific:
@@ -253,3 +300,23 @@ dependencies:
             packages:
               - cudf-cu11==24.4.*
           - {matrix: null, packages: [*cudf_conda]}
+  depends_on_ucxx:
+    common:
+      - output_types: conda
+        packages:
+          - &ucxx_conda ucxx==0.37.*
+      - output_types: requirements
+        packages:
+          # pip recognizes the index as a global option for the requirements.txt file
+          - --extra-index-url=https://pypi.nvidia.com
+          - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple
+    specific:
+      - output_types: [requirements, pyproject]
+        matrices:
+          - matrix: {cuda: "12.*"}
+            packages:
+              - ucxx-cu12==0.37.*
+          - matrix: {cuda: "11.*"}
+            packages:
+              - ucxx-cu11==0.37.*
+          - {matrix: null, packages: [*ucxx_conda]}

python/distributed-ucxx/pyproject.toml

@@ -2,8 +2,8 @@
 build-backend = "setuptools.build_meta"
 requires = [
     "setuptools>=64.0.0",
-    "tomli  ; python_version < '3.11'",
-]
+    "tomli",
+] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
 
 [project]
 name = "distributed-ucxx"
@@ -16,10 +16,10 @@ authors = [
 license = { text = "Apache-2.0" }
 requires-python = ">=3.8"
 dependencies = [
-    "dask >=2023.9.2",
-    "distributed >=2023.9.2",
-    "numba >=0.54",
-]
+    "numba>=0.57.1",
+    "rapids-dask-dependency==24.4.*",
+    "ucxx==0.37.*",
+] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
 classifiers = [
     "Intended Audience :: Developers",
     "License :: OSI Approved :: Apache Software License",
@@ -39,9 +39,9 @@ docs = [
     "sphinx-rtd-theme>=0.5.1",
 ]
 test = [
+    "numpy>=1.21",
     "pytest",
-    "numpy",
-]
+] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
 
 [project.urls]
 Homepage = "https://github.com/rapidsai/ucxx"

python/pyproject.toml

@@ -24,6 +24,7 @@ requires-python = ">=3.9"
 dependencies = [
     "numpy>=1.21",
     "pynvml>=11.4.1",
+    "rmm==24.4.*",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../dependencies.yaml and run `rapids-dependency-file-generator`.
 classifiers = [
     "Intended Audience :: Developers",
@@ -41,9 +42,7 @@ classifiers = [
 test = [
     "cloudpickle",
     "cudf==24.4.*",
-    "cupy>=12.0.0",
-    "dask",
-    "distributed",
+    "cupy-cuda11x>=12.0.0",
     "numba>=0.57.1",
     "pytest",
     "pytest-asyncio",