split up CUDA-suffixed dependencies in dependencies.yaml (#4552)

Contributes to rapidsai/build-planning#31 In short, RAPIDS DLFW builds want to produce wheels with unsuffixed dependencies, e.g. `cudf` depending on `rmm`, not `rmm-cu12`. This PR is part of a series across all of RAPIDS to try to support that type of build by setting up CUDA-suffixed and CUDA-unsuffixed dependency lists in `dependencies.yaml`. For more details, see: * rapidsai/build-planning#31 (comment) * rapidsai/cudf#16183 ## Notes for Reviewers ### Why target 24.08? This is targeting 24.08 because: 1. it should be very low-risk 2. getting these changes into 24.08 prevents the need to carry around patches for every library in DLFW builds using RAPIDS 24.08 Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Bradley Dice (https://github.com/bdice) URL: #4552
rapidsai · Jul 30, 2024 · 4fb6470 · 4fb6470
1 parent f628718
commit 4fb6470
Show file tree

Hide file tree

Showing 9 changed files with 94 additions and 43 deletions.
diff --git a/dependencies.yaml b/dependencies.yaml
@@ -493,7 +493,7 @@ dependencies:
           - requests
           - nccl>=2.9.9
           - ucx-proc=*=gpu
-          - &ucx_py ucx-py==0.39.*,>=0.0.0a0
+          - &ucx_py_unsuffixed ucx-py==0.39.*,>=0.0.0a0
       - output_types: pyproject
         packages:
             # cudf uses fsspec but is protocol independent. cugraph
@@ -504,15 +504,17 @@ dependencies:
         matrices:
           - matrix:
               cuda: "11.*"
+              cuda_suffixed: "true"
             packages:
               - &ucx_py_cu11 ucx-py-cu11==0.39.*,>=0.0.0a0
           - matrix:
               cuda: "12.*"
+              cuda_suffixed: "true"
             packages:
               - &ucx_py_cu12 ucx-py-cu12==0.39.*,>=0.0.0a0
           - matrix:
             packages:
-              - *ucx_py
+              - *ucx_py_unsuffixed
   python_run_nx_cugraph:
     common:
       - output_types: [conda, pyproject]
@@ -530,15 +532,17 @@ dependencies:
         matrices:
           - matrix:
               cuda: "11.*"
+              cuda_suffixed: "true"
             packages:
               - &cugraph_cu11 cugraph-cu11==24.8.*,>=0.0.0a0
           - matrix:
               cuda: "12.*"
+              cuda_suffixed: "true"
             packages:
               - &cugraph_cu12 cugraph-cu12==24.8.*,>=0.0.0a0
           - matrix:
             packages:
-              - &cugraph cugraph==24.8.*,>=0.0.0a0
+              - &cugraph_unsuffixed cugraph==24.8.*,>=0.0.0a0
   python_run_cugraph_pyg:
     common:
       - output_types: [conda, pyproject]
@@ -550,15 +554,17 @@ dependencies:
         matrices:
           - matrix:
               cuda: "11.*"
+              cuda_suffixed: "true"
             packages:
               - *cugraph_cu11
           - matrix:
               cuda: "12.*"
+              cuda_suffixed: "true"
             packages:
               - *cugraph_cu12
           - matrix:
             packages:
-              - *cugraph
+              - *cugraph_unsuffixed
   python_run_cugraph_service_client:
     common:
       - output_types: [conda, pyproject]
@@ -575,27 +581,29 @@ dependencies:
           - *thrift
       - output_types: conda
         packages:
-          - *ucx_py
+          - *ucx_py_unsuffixed
     specific:
       - output_types: pyproject
         matrices:
           - matrix:
               cuda: "11.*"
+              cuda_suffixed: "true"
             packages:
               - *cugraph_cu11
               - cugraph-service-client-cu11==24.8.*,>=0.0.0a0
               - *ucx_py_cu11
           - matrix:
               cuda: "12.*"
+              cuda_suffixed: "true"
             packages:
               - *cugraph_cu12
               - cugraph-service-client-cu12==24.8.*,>=0.0.0a0
               - *ucx_py_cu12
           - matrix:
             packages:
-              - *cugraph
+              - *cugraph_unsuffixed
               - cugraph-service-client==24.8.*,>=0.0.0a0
-              - *ucx_py
+              - *ucx_py_unsuffixed
   test_cpp:
     common:
       - output_types: conda
@@ -630,7 +638,7 @@ dependencies:
           - scikit-learn>=0.23.1
       - output_types: [conda]
         packages:
-          - &pylibwholegraph_conda pylibwholegraph==24.8.*,>=0.0.0a0
+          - &pylibwholegraph_unsuffixed pylibwholegraph==24.8.*,>=0.0.0a0
           - *thrift
   test_python_pylibcugraph:
     common:
@@ -648,15 +656,15 @@ dependencies:
     common:
       - output_types: [conda]
         packages:
-          - cugraph==24.8.*,>=0.0.0a0
+          - *cugraph_unsuffixed
           - pytorch>=2.0
           - pytorch-cuda==11.8
           - dgl>=1.1.0.cu*
   cugraph_pyg_dev:
     common:
       - output_types: [conda]
         packages:
-          - cugraph==24.8.*,>=0.0.0a0
+          - *cugraph_unsuffixed
           - pytorch>=2.0
           - pytorch-cuda==11.8
           - &tensordict tensordict>=0.1.2
@@ -666,7 +674,7 @@ dependencies:
     common:
       - output_types: [conda]
         packages:
-          - &pytorch_conda pytorch>=2.0,<2.2.0a0
+          - &pytorch_unsuffixed pytorch>=2.0,<2.2.0a0
 
     specific:
       - output_types: [requirements]
@@ -694,7 +702,7 @@ dependencies:
     common:
       - output_types: conda
         packages:
-          - *pylibwholegraph_conda
+          - *pylibwholegraph_unsuffixed
       - output_types: requirements
         packages:
           # pip recognizes the index as a global option for the requirements.txt file
@@ -703,19 +711,23 @@ dependencies:
     specific:
       - output_types: [requirements, pyproject]
         matrices:
-          - matrix: {cuda: "12.*"}
+          - matrix:
+              cuda: "12.*"
+              cuda_suffixed: "true"
             packages:
               - pylibwholegraph-cu12==24.8.*,>=0.0.0a0
-          - matrix: {cuda: "11.*"}
+          - matrix:
+              cuda: "11.*"
+              cuda_suffixed: "true"
             packages:
               - pylibwholegraph-cu11==24.8.*,>=0.0.0a0
-          - {matrix: null, packages: [*pylibwholegraph_conda]}
+          - {matrix: null, packages: [*pylibwholegraph_unsuffixed]}
 
   depends_on_rmm:
     common:
       - output_types: conda
         packages:
-          - &rmm_conda rmm==24.8.*,>=0.0.0a0
+          - &rmm_unsuffixed rmm==24.8.*,>=0.0.0a0
       - output_types: requirements
         packages:
           # pip recognizes the index as a global option for the requirements.txt file
@@ -724,19 +736,23 @@ dependencies:
     specific:
       - output_types: [requirements, pyproject]
         matrices:
-          - matrix: {cuda: "12.*"}
+          - matrix:
+              cuda: "12.*"
+              cuda_suffixed: "true"
             packages:
               - rmm-cu12==24.8.*,>=0.0.0a0
-          - matrix: {cuda: "11.*"}
+          - matrix:
+              cuda: "11.*"
+              cuda_suffixed: "true"
             packages:
               - rmm-cu11==24.8.*,>=0.0.0a0
-          - {matrix: null, packages: [*rmm_conda]}
+          - {matrix: null, packages: [*rmm_unsuffixed]}
 
   depends_on_cudf:
     common:
       - output_types: conda
         packages:
-          - &cudf_conda cudf==24.8.*,>=0.0.0a0
+          - &cudf_unsuffixed cudf==24.8.*,>=0.0.0a0
       - output_types: requirements
         packages:
           # pip recognizes the index as a global option for the requirements.txt file
@@ -745,19 +761,23 @@ dependencies:
     specific:
       - output_types: [requirements, pyproject]
         matrices:
-          - matrix: {cuda: "12.*"}
+          - matrix:
+              cuda: "12.*"
+              cuda_suffixed: "true"
             packages:
               - cudf-cu12==24.8.*,>=0.0.0a0
-          - matrix: {cuda: "11.*"}
+          - matrix:
+              cuda: "11.*"
+              cuda_suffixed: "true"
             packages:
               - cudf-cu11==24.8.*,>=0.0.0a0
-          - {matrix: null, packages: [*cudf_conda]}
+          - {matrix: null, packages: [*cudf_unsuffixed]}
 
   depends_on_dask_cudf:
     common:
       - output_types: conda
         packages:
-          - &dask_cudf_conda dask-cudf==24.8.*,>=0.0.0a0
+          - &dask_cudf_unsuffixed dask-cudf==24.8.*,>=0.0.0a0
       - output_types: requirements
         packages:
           # pip recognizes the index as a global option for the requirements.txt file
@@ -766,19 +786,23 @@ dependencies:
     specific:
       - output_types: [requirements, pyproject]
         matrices:
-          - matrix: {cuda: "12.*"}
+          - matrix:
+              cuda: "12.*"
+              cuda_suffixed: "true"
             packages:
               - dask-cudf-cu12==24.8.*,>=0.0.0a0
-          - matrix: {cuda: "11.*"}
+          - matrix:
+              cuda: "11.*"
+              cuda_suffixed: "true"
             packages:
               - dask-cudf-cu11==24.8.*,>=0.0.0a0
-          - {matrix: null, packages: [*dask_cudf_conda]}
+          - {matrix: null, packages: [*dask_cudf_unsuffixed]}
 
   depends_on_pylibraft:
     common:
       - output_types: conda
         packages:
-          - &pylibraft_conda pylibraft==24.8.*,>=0.0.0a0
+          - &pylibraft_unsuffixed pylibraft==24.8.*,>=0.0.0a0
       - output_types: requirements
         packages:
           # pip recognizes the index as a global option for the requirements.txt file
@@ -787,19 +811,23 @@ dependencies:
     specific:
       - output_types: [requirements, pyproject]
         matrices:
-          - matrix: {cuda: "12.*"}
+          - matrix:
+              cuda: "12.*"
+              cuda_suffixed: "true"
             packages:
               - pylibraft-cu12==24.8.*,>=0.0.0a0
-          - matrix: {cuda: "11.*"}
+          - matrix:
+              cuda: "11.*"
+              cuda_suffixed: "true"
             packages:
               - pylibraft-cu11==24.8.*,>=0.0.0a0
-          - {matrix: null, packages: [*pylibraft_conda]}
+          - {matrix: null, packages: [*pylibraft_unsuffixed]}
 
   depends_on_raft_dask:
     common:
       - output_types: conda
         packages:
-          - &raft_dask_conda raft-dask==24.8.*,>=0.0.0a0
+          - &raft_dask_unsuffixed raft-dask==24.8.*,>=0.0.0a0
       - output_types: requirements
         packages:
           # pip recognizes the index as a global option for the requirements.txt file
@@ -808,19 +836,23 @@ dependencies:
     specific:
       - output_types: [requirements, pyproject]
         matrices:
-          - matrix: {cuda: "12.*"}
+          - matrix:
+              cuda: "12.*"
+              cuda_suffixed: "true"
             packages:
               - raft-dask-cu12==24.8.*,>=0.0.0a0
-          - matrix: {cuda: "11.*"}
+          - matrix:
+              cuda: "11.*"
+              cuda_suffixed: "true"
             packages:
               - raft-dask-cu11==24.8.*,>=0.0.0a0
-          - {matrix: null, packages: [*raft_dask_conda]}
+          - {matrix: null, packages: [*raft_dask_unsuffixed]}
 
   depends_on_pylibcugraph:
     common:
       - output_types: conda
         packages:
-          - &pylibcugraph_conda pylibcugraph==24.8.*,>=0.0.0a0
+          - &pylibcugraph_unsuffixed pylibcugraph==24.8.*,>=0.0.0a0
       - output_types: requirements
         packages:
           # pip recognizes the index as a global option for the requirements.txt file
@@ -829,19 +861,23 @@ dependencies:
     specific:
       - output_types: [requirements, pyproject]
         matrices:
-          - matrix: {cuda: "12.*"}
+          - matrix:
+              cuda: "12.*"
+              cuda_suffixed: "true"
             packages:
               - pylibcugraph-cu12==24.8.*,>=0.0.0a0
-          - matrix: {cuda: "11.*"}
+          - matrix:
+              cuda: "11.*"
+              cuda_suffixed: "true"
             packages:
               - pylibcugraph-cu11==24.8.*,>=0.0.0a0
-          - {matrix: null, packages: [*pylibcugraph_conda]}
+          - {matrix: null, packages: [*pylibcugraph_unsuffixed]}
 
   depends_on_pylibcugraphops:
     common:
       - output_types: conda
         packages:
-          - &pylibcugraphops_conda pylibcugraphops==24.8.*,>=0.0.0a0
+          - &pylibcugraphops_unsuffixed pylibcugraphops==24.8.*,>=0.0.0a0
       - output_types: requirements
         packages:
           # pip recognizes the index as a global option for the requirements.txt file
@@ -850,19 +886,26 @@ dependencies:
     specific:
       - output_types: [requirements, pyproject]
         matrices:
-          - matrix: {cuda: "12.*"}
+          - matrix:
+              cuda: "12.*"
+              cuda_suffixed: "true"
             packages:
               - pylibcugraphops-cu12==24.8.*,>=0.0.0a0
-          - matrix: {cuda: "11.*"}
+          - matrix:
+              cuda: "11.*"
+              cuda_suffixed: "true"
             packages:
               - pylibcugraphops-cu11==24.8.*,>=0.0.0a0
-          - {matrix: null, packages: [*pylibcugraphops_conda]}
+          - {matrix: null, packages: [*pylibcugraphops_unsuffixed]}
 
   depends_on_cupy:
     common:
       - output_types: conda
         packages:
           - cupy>=12.0.0
+    # NOTE: This is intentionally not broken into groups by a 'cuda_suffixed' selector like
+    #       other packages with -cu{nn}x suffixes in this file.
+    #       All RAPIDS wheel builds (including in devcontainers) expect cupy to be suffixed.
     specific:
       - output_types: [requirements, pyproject]
         matrices:

diff --git a/python/cugraph-dgl/pyproject.toml b/python/cugraph-dgl/pyproject.toml
@@ -61,3 +61,4 @@ include = [
 [tool.rapids-build-backend]
 build-backend = "setuptools.build_meta"
 dependencies-file = "../../dependencies.yaml"
+matrix-entry = "cuda_suffixed=true"
diff --git a/python/cugraph-equivariant/pyproject.toml b/python/cugraph-equivariant/pyproject.toml
@@ -69,3 +69,4 @@ include = [
 [tool.rapids-build-backend]
 build-backend = "setuptools.build_meta"
 dependencies-file = "../../dependencies.yaml"
+matrix-entry = "cuda_suffixed=true"
diff --git a/python/cugraph-pyg/pyproject.toml b/python/cugraph-pyg/pyproject.toml
@@ -67,3 +67,4 @@ include = [
 [tool.rapids-build-backend]
 build-backend = "setuptools.build_meta"
 dependencies-file = "../../dependencies.yaml"
+matrix-entry = "cuda_suffixed=true"
diff --git a/python/cugraph-service/client/pyproject.toml b/python/cugraph-service/client/pyproject.toml
@@ -49,3 +49,4 @@ include = [
 build-backend = "setuptools.build_meta"
 dependencies-file = "../../../dependencies.yaml"
 disable-cuda = true
+matrix-entry = "cuda_suffixed=true"
diff --git a/python/cugraph-service/server/pyproject.toml b/python/cugraph-service/server/pyproject.toml
@@ -77,3 +77,4 @@ include = [
 [tool.rapids-build-backend]
 build-backend = "setuptools.build_meta"
 dependencies-file = "../../../dependencies.yaml"
+matrix-entry = "cuda_suffixed=true"
diff --git a/python/cugraph/pyproject.toml b/python/cugraph/pyproject.toml
@@ -86,3 +86,4 @@ requires = [
     "rmm==24.8.*,>=0.0.0a0",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
 dependencies-file = "../../dependencies.yaml"
+matrix-entry = "cuda_suffixed=true"