handle more RAPIDS version formats in update-version.sh, refactor dependencies.yaml

.github/workflows/build.yaml

@@ -38,7 +38,7 @@ jobs:
     if: github.ref_type == 'branch'
     needs: conda-cpp-build
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.02
+    uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04
     with:
       arch: "amd64"
       branch: ${{ inputs.branch }}

.github/workflows/pr.yaml

@@ -33,7 +33,7 @@ jobs:
   docs-build:
     needs: conda-cpp-build
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.02
+    uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04
     with:
       build_type: pull-request
       node_type: "gpu-v100-latest-1"

ci/build_docs.sh

@@ -25,7 +25,7 @@ rapids-mamba-retry install \
   --channel "${CPP_CHANNEL}" \
   libucxx
 
-export RAPIDS_VERSION_NUMBER="24.02"
+export RAPIDS_VERSION_NUMBER="24.04"
 export RAPIDS_DOCS_DIR="$(mktemp -d)"
 
 rapids-logger "Build CPP docs"

ci/release/update-version.sh

@@ -20,8 +20,9 @@ NEXT_SHORT_TAG=${NEXT_MAJOR}.${NEXT_MINOR}
 NEXT_RAPIDS_VERSION="$(curl -sL https://version.gpuci.io/ucx-py/${NEXT_SHORT_TAG})"
 
 # Need to distutils-normalize the versions for some use cases
-NEXT_SHORT_TAG_PEP440=$(python -c "from setuptools.extern import packaging; print(packaging.version.Version('${NEXT_RAPIDS_VERSION}'))")
-echo "Next tag is ${NEXT_SHORT_TAG_PEP440}"
+NEXT_SHORT_TAG_PEP440=$(python -c "from setuptools.extern import packaging; print(packaging.version.Version('${NEXT_SHORT_TAG}'))")
+NEXT_RAPIDS_SHORT_TAG_PEP440=$(python -c "from setuptools.extern import packaging; print(packaging.version.Version('${NEXT_RAPIDS_VERSION}'))")
+echo "Next tag is ${NEXT_RAPIDS_SHORT_TAG_PEP440}"
 
 echo "Preparing release: $NEXT_FULL_TAG"
 
@@ -44,6 +45,7 @@ sed_runner "s/^version = .*/version = \"${NEXT_FULL_TAG}\"/g" python/distributed
 # bump RAPIDS libs
 sed_runner "/- librmm =/ s/=.*/=${NEXT_RAPIDS_VERSION}/g" conda/recipes/ucxx/meta.yaml
 sed_runner "/- rmm =/ s/=.*/=${NEXT_RAPIDS_VERSION}/g" conda/recipes/ucxx/meta.yaml
+sed_runner "/- rapids-dask-dependency =/ s/=.*/=${NEXT_RAPIDS_VERSION}/g" conda/recipes/ucxx/meta.yaml
 
 # doxyfile update
 sed_runner 's/PROJECT_NUMBER         = .*/PROJECT_NUMBER         = '${NEXT_FULL_TAG}'/g' cpp/doxygen/Doxyfile
@@ -54,12 +56,26 @@ DEPENDENCIES=(
   dask-cudf
   librmm
   rmm
+  rapids-dask-dependency
 )
 for DEP in "${DEPENDENCIES[@]}"; do
   for FILE in dependencies.yaml conda/environments/*.yaml; do
-    sed_runner "/-.* ${DEP}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}\.*/g" ${FILE};
+    sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_RAPIDS_SHORT_TAG_PEP440}.*/g" "${FILE}"
   done
-  sed_runner "/\"${DEP}==/ s/==.*\"/==${NEXT_SHORT_TAG_PEP440}\.*\"/g" python/pyproject.toml;
+  sed_runner "/\"${DEP}==/ s/==.*\"/==${NEXT_RAPIDS_SHORT_TAG_PEP440}\.*\"/g" python/pyproject.toml
+  sed_runner "/\"${DEP}==/ s/==.*\"/==${NEXT_RAPIDS_SHORT_TAG_PEP440}\.*\"/g" python/distributed-ucxx/pyproject.toml
+done
+
+UCXX_DEPENDENCIES=(
+  ucxx
+  distributed-ucxx
+)
+for DEP in "${UCXX_DEPENDENCIES[@]}"; do
+  for FILE in dependencies.yaml; do
+    sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}.*/g" "${FILE}"
+  done
+  sed_runner "/\"${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*\"/==${NEXT_SHORT_TAG_PEP440}\.*\"/g" python/pyproject.toml
+  sed_runner "/\"${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*\"/==${NEXT_SHORT_TAG_PEP440}\.*\"/g" python/distributed-ucxx/pyproject.toml
 done
 
 # rapids-cmake version
@@ -68,4 +84,4 @@ sed_runner 's/'"branch-.*\/RAPIDS.cmake"'/'"branch-${NEXT_RAPIDS_VERSION}\/RAPID
 for FILE in .github/workflows/*.yaml; do
   sed_runner "/shared-workflows/ s/@.*/@branch-${NEXT_RAPIDS_VERSION}/g" "${FILE}"
 done
-sed_runner "s/RAPIDS_VERSION_NUMBER=\".*/RAPIDS_VERSION_NUMBER=\"${NEXT_SHORT_TAG}\"/g" ci/build_docs.sh
+sed_runner "s/RAPIDS_VERSION_NUMBER=\".*/RAPIDS_VERSION_NUMBER=\"${NEXT_RAPIDS_VERSION}\"/g" ci/build_docs.sh

conda/environments/all_cuda-118_arch-x86_64.yaml

@@ -18,10 +18,8 @@ dependencies:
 - cupy>=12.0.0
 - cxx-compiler
 - cython>=3.0.0
-- dask
 - dask-cuda==24.4.*
 - dask-cudf==24.4.*
-- distributed
 - doxygen=1.9.1
 - fmt>=10.1.1,<11
 - gmock>=1.13.0
@@ -39,8 +37,11 @@ dependencies:
 - pytest-asyncio
 - pytest-rerunfailures
 - python>=3.9,<3.11
+- rapids-dask-dependency==24.4.*
 - rmm==24.4.*
 - scikit-build-core>=0.7.0
+- setuptools>=64.0.0
 - spdlog>=1.12.0,<1.13
+- tomli
 - ucx
 name: all_cuda-118_arch-x86_64

conda/environments/all_cuda-120_arch-x86_64.yaml

@@ -18,10 +18,8 @@ dependencies:
 - cupy>=12.0.0
 - cxx-compiler
 - cython>=3.0.0
-- dask
 - dask-cuda==24.4.*
 - dask-cudf==24.4.*
-- distributed
 - doxygen=1.9.1
 - fmt>=10.1.1,<11
 - gmock>=1.13.0
@@ -39,8 +37,11 @@ dependencies:
 - pytest-asyncio
 - pytest-rerunfailures
 - python>=3.9,<3.11
+- rapids-dask-dependency==24.4.*
 - rmm==24.4.*
 - scikit-build-core>=0.7.0
+- setuptools>=64.0.0
 - spdlog>=1.12.0,<1.13
+- tomli
 - ucx
 name: all_cuda-120_arch-x86_64

conda/recipes/ucxx/meta.yaml

@@ -271,8 +271,7 @@ outputs:
         - tomli
       run:
         - python * *_cpython
-        - dask >=2023.9.2
-        - distributed >=2023.9.2
+        - rapids-dask-dependency =24.04
         - {{ pin_subpackage('ucxx', max_pin='x.x') }}
     test:
       imports:

cpp/doxygen/Doxyfile

@@ -38,7 +38,7 @@ PROJECT_NAME           = libucxx
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = 0.36.00
+PROJECT_NUMBER         = 0.37.00
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

dependencies.yaml

@@ -7,17 +7,21 @@ files:
       arch: [x86_64]
     includes:
       - build_cpp
-      - build_python
+      - build_python_ucxx
+      - build_python_distributed_ucxx
       - checks
       - cuda
       - cuda_version
       - dev
       - docs
       - py_version
-      - run_python
+      - run_python_ucxx
+      - run_python_distributed_ucxx
       - test_cpp
-      - test_python
+      - test_python_ucxx
+      - test_python_distributed_ucxx
       - depends_on_cupy
+      - depends_on_rmm
       - depends_on_cudf
   test_cpp:
     output: none
@@ -29,7 +33,8 @@ files:
     includes:
       - cuda_version
       - py_version
-      - test_python
+      - test_python_ucxx
+      - test_python_distributed_ucxx
       - depends_on_cupy
       - depends_on_cudf
   checks:
@@ -42,30 +47,55 @@ files:
     includes:
       - docs
       - py_version
-  py_build:
+  py_build_ucxx:
     output: pyproject
     pyproject_dir: python
     extras:
       table: build-system
     includes:
-      - build_python
-  py_run:
+      - build_python_ucxx
+      - depends_on_rmm
+  py_run_ucxx:
     output: pyproject
     pyproject_dir: python
     extras:
       table: project
     includes:
-      - run_python
-  py_test:
+      - run_python_ucxx
+      - depends_on_rmm
+  py_test_ucxx:
     output: pyproject
     pyproject_dir: python
     extras:
       table: project.optional-dependencies
       key: test
     includes:
-      - test_python
+      - test_python_ucxx
       - depends_on_cupy
       - depends_on_cudf
+  py_build_distributed_ucxx:
+    output: pyproject
+    pyproject_dir: python/distributed-ucxx
+    extras:
+      table: build-system
+    includes:
+      - build_python_distributed_ucxx
+  py_run_distributed_ucxx:
+    output: pyproject
+    pyproject_dir: python/distributed-ucxx
+    extras:
+      table: project
+    includes:
+      - run_python_distributed_ucxx
+      - depends_on_ucxx
+  py_test_distributed_ucxx:
+    output: pyproject
+    pyproject_dir: python/distributed-ucxx
+    extras:
+      table: project.optional-dependencies
+      key: test
+    includes:
+      - test_python_distributed_ucxx
 channels:
   - rapidsai
   - rapidsai-nightly
@@ -86,11 +116,8 @@ dependencies:
           - librmm==24.4.*
           - ninja
           - spdlog>=1.12.0,<1.13
-  build_python:
+  build_python_ucxx:
     common:
-      - output_types: [conda]
-        packages:
-          - &rmm_conda rmm==24.4.*
       - output_types: [conda, requirements, pyproject]
         packages:
           - *cmake_ver
@@ -102,16 +129,12 @@ dependencies:
       - output_types: [requirements, pyproject]
         packages:
           - scikit-build-core[pyproject]>=0.7.0
-    specific:
-      - output_types: [requirements, pyproject]
-        matrices:
-          - matrix: {cuda: "12.*"}
-            packages:
-              - rmm-cu12==24.4.*
-          - matrix: {cuda: "11.*"}
-            packages:
-              - rmm-cu11==24.4.*
-          - {matrix: null, packages: [*rmm_conda]}
+  build_python_distributed_ucxx:
+    common:
+      - output_types: [conda, requirements, pyproject]
+        packages:
+          - setuptools>=64.0.0
+          - tomli
   checks:
     common:
       - output_types: [conda, requirements]
@@ -190,48 +213,78 @@ dependencies:
           - matrix:
             packages:
               - python>=3.9,<3.11
-  run_python:
+  run_python_ucxx:
     common:
       - output_types: [conda, requirements, pyproject]
         packages:
-          - numpy>=1.21
+          - &numpy numpy>=1.21
           - pynvml>=11.4.1
       - output_types: [conda]
         packages:
           - ucx
+  run_python_distributed_ucxx:
+    common:
+      - output_types: [conda, requirements, pyproject]
+        packages:
+          - &numba numba>=0.57.1
+          - rapids-dask-dependency==24.4.*
   test_cpp:
     common:
       - output_types: conda
         packages:
           - *cmake_ver
           - *gtest
           - *gmock
-  test_python:
+  test_python_ucxx:
     common:
       - output_types: [conda, requirements, pyproject]
         packages:
           - cloudpickle
-          - dask
-          - distributed
-          - numba>=0.57.1
+          - *numba
           - pytest
           - pytest-asyncio
           - pytest-rerunfailures
+  test_python_distributed_ucxx:
+    common:
+      - output_types: [conda, requirements, pyproject]
+        packages:
+          - *numpy
+          - pytest
   depends_on_cupy:
     common:
       - output_types: conda
         packages:
-          - &cupy_conda cupy>=12.0.0
+          - cupy>=12.0.0
     specific:
       - output_types: [requirements, pyproject]
         matrices:
           - matrix: {cuda: "12.*"}
             packages:
               - cupy-cuda12x>=12.0.0
           - matrix: {cuda: "11.*"}
-            packages:
+            packages: &cupy_packages_cu11
               - cupy-cuda11x>=12.0.0
-          - {matrix: null, packages: [*cupy_conda]}
+          - {matrix: null, packages: *cupy_packages_cu11}
+  depends_on_rmm:
+    common:
+      - output_types: conda
+        packages:
+          - &rmm_conda rmm==24.4.*
+      - output_types: requirements
+        packages:
+          # pip recognizes the index as a global option for the requirements.txt file
+          - --extra-index-url=https://pypi.nvidia.com
+          - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple
+    specific:
+      - output_types: [requirements, pyproject]
+        matrices:
+          - matrix: {cuda: "12.*"}
+            packages:
+              - rmm-cu12==24.4.*
+          - matrix: {cuda: "11.*"}
+            packages:
+              - rmm-cu11==24.4.*
+          - {matrix: null, packages: [*rmm_conda]}
   depends_on_cudf:
     common:
       - output_types: conda
@@ -240,7 +293,6 @@ dependencies:
       - output_types: requirements
         packages:
           # pip recognizes the index as a global option for the requirements.txt file
-          # This index is needed for rmm, cubinlinker, ptxcompiler.
           - --extra-index-url=https://pypi.nvidia.com
           - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple
     specific:
@@ -253,3 +305,23 @@ dependencies:
             packages:
               - cudf-cu11==24.4.*
           - {matrix: null, packages: [*cudf_conda]}
+  depends_on_ucxx:
+    common:
+      - output_types: conda
+        packages:
+          - &ucxx_conda ucxx==0.37.*
+      - output_types: requirements
+        packages:
+          # pip recognizes the index as a global option for the requirements.txt file
+          - --extra-index-url=https://pypi.nvidia.com
+          - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple
+    specific:
+      - output_types: [requirements, pyproject]
+        matrices:
+          - matrix: {cuda: "12.*"}
+            packages:
+              - ucxx-cu12==0.37.*
+          - matrix: {cuda: "11.*"}
+            packages:
+              - ucxx-cu11==0.37.*
+          - {matrix: null, packages: [*ucxx_conda]}