From 4b1b93b5291a0b7f6d1db914d3be6ea21d58c377 Mon Sep 17 00:00:00 2001 From: Divye Gala Date: Fri, 12 May 2023 18:38:55 -0400 Subject: [PATCH] Support CUDA 12.0 for pip wheels (#3544) Authors: - Divye Gala (https://github.com/divyegala) - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - AJ Schmidt (https://github.com/ajschmidt8) - Brad Rees (https://github.com/BradReesWork) URL: https://github.com/rapidsai/cugraph/pull/3544 --- .github/workflows/build.yaml | 11 ++++++----- .github/workflows/pr.yaml | 17 +++++++---------- .github/workflows/test.yaml | 5 +---- ci/release/apply_wheel_modifications.sh | 5 +++++ .../environments/all_cuda-118_arch-x86_64.yaml | 4 ++-- dependencies.yaml | 4 ++-- python/cugraph-dgl/pyproject.toml | 4 ++-- python/cugraph-pyg/pyproject.toml | 4 ++-- python/cugraph-service/server/pyproject.toml | 4 ++-- python/cugraph/pyproject.toml | 4 ++-- python/pylibcugraph/pyproject.toml | 2 +- 11 files changed, 32 insertions(+), 32 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 850e58d4767..1c03d0d66db 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -65,7 +65,7 @@ jobs: run_script: "ci/build_docs.sh" wheel-build-pylibcugraph: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@manylinux_v2 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-120-pip with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -85,7 +85,7 @@ jobs: wheel-publish-pylibcugraph: needs: wheel-build-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@manylinux_v2 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@cuda-120-pip with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -95,7 +95,7 @@ jobs: wheel-build-cugraph: needs: wheel-publish-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@manylinux_v2 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-120-pip with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -111,11 +111,12 @@ jobs: extra-repo-sha: branch-23.06 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY - skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/python/pylibcugraph/cugraph-ops/" + before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_${{ '${PIP_CU_VERSION}' }} rapids-download-wheels-from-s3 /local-wheelhouse" + skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/python/cugraph/cugraph-ops/" wheel-publish-cugraph: needs: wheel-build-cugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@manylinux_v2 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@cuda-120-pip with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 624967a4afc..943099c36f5 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -78,7 +78,7 @@ jobs: wheel-build-pylibcugraph: needs: checks secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@manylinux_v2 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-120-pip with: build_type: pull-request package-name: pylibcugraph @@ -90,18 +90,16 @@ jobs: wheel-tests-pylibcugraph: needs: wheel-build-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@manylinux_v2 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120-pip with: build_type: pull-request package-name: pylibcugraph - # On arm also need to install cupy from the specific webpage. - test-before-arm64: "pip install 'cupy-cuda11x<12.0.0' -f https://pip.cupy.dev/aarch64" test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets pytest ./python/pylibcugraph/pylibcugraph/tests" test-smoketest: "python ci/wheel_smoke_test_pylibcugraph.py" wheel-build-cugraph: needs: wheel-tests-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@manylinux_v2 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-120-pip with: build_type: pull-request package-name: cugraph @@ -109,19 +107,18 @@ jobs: extra-repo: rapidsai/cugraph-ops extra-repo-sha: branch-23.06 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY - before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph && python -m pip install --no-deps ./local-pylibcugraph/pylibcugraph*.whl" + before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_${{ '${PIP_CU_VERSION}' }} rapids-download-wheels-from-s3 /local-wheelhouse" skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/python/cugraph/cugraph-ops/" wheel-tests-cugraph: needs: wheel-build-cugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@manylinux_v2 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120-pip with: build_type: pull-request package-name: cugraph # Always want to test against latest dask/distributed. - test-before-amd64: "cd ./datasets && bash ./get_test_data.sh && cd - && RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install git+https://github.com/dask/dask.git@2023.3.2 git+https://github.com/dask/distributed.git@2023.3.2.1 git+https://github.com/rapidsai/dask-cuda.git@branch-23.06" + test-before-amd64: "cd ./datasets && bash ./get_test_data.sh && cd - && RAPIDS_PY_WHEEL_NAME=pylibcugraph_${{ '${PIP_CU_VERSION}' }} rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install git+https://github.com/dask/dask.git@2023.3.2 git+https://github.com/dask/distributed.git@2023.3.2.1 git+https://github.com/rapidsai/dask-cuda.git@branch-23.06" # Skip dataset downloads on arm to save CI time -- arm only runs smoke tests. - # On arm also need to install cupy from the specific site. - test-before-arm64: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install 'cupy-cuda11x<12.0.0' -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@2023.3.2 git+https://github.com/dask/distributed.git@2023.3.2.1 git+https://github.com/rapidsai/dask-cuda.git@branch-23.06" + test-before-arm64: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_${{ '${PIP_CU_VERSION}' }} rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install git+https://github.com/dask/dask.git@2023.3.2 git+https://github.com/dask/distributed.git@2023.3.2.1 git+https://github.com/rapidsai/dask-cuda.git@branch-23.06" test-unittest: "RAPIDS_DATASET_ROOT_DIR=/__w/cugraph/cugraph/datasets pytest -m sg ./python/cugraph/cugraph/tests" test-smoketest: "python ci/wheel_smoke_test_cugraph.py" diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 33fbc8614aa..6896142e677 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -39,8 +39,6 @@ jobs: date: ${{ inputs.date }} sha: ${{ inputs.sha }} package-name: pylibcugraph - # On arm also need to install cupy from the specific webpage. - test-before-arm64: "pip install 'cupy-cuda11x<12.0.0' -f https://pip.cupy.dev/aarch64" test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets pytest ./python/pylibcugraph/pylibcugraph/tests" wheel-tests-cugraph: secrets: inherit @@ -53,6 +51,5 @@ jobs: package-name: cugraph # Always want to test against latest dask/distributed. test-before-amd64: "cd ./datasets && bash ./get_test_data.sh && cd - && pip install git+https://github.com/dask/dask.git@2023.3.2 git+https://github.com/dask/distributed.git@2023.3.2.1 git+https://github.com/rapidsai/dask-cuda.git@branch-23.06" - # On arm also need to install cupy from the specific webpage. - test-before-arm64: "cd ./datasets && bash ./get_test_data.sh && cd - && pip install 'cupy-cuda11x<12.0.0' -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@2023.3.2 git+https://github.com/dask/distributed.git@2023.3.2.1 git+https://github.com/rapidsai/dask-cuda.git@branch-23.06" + test-before-arm64: "cd ./datasets && bash ./get_test_data.sh && cd - && pip install git+https://github.com/dask/dask.git@2023.3.2 git+https://github.com/dask/distributed.git@2023.3.2.1 git+https://github.com/rapidsai/dask-cuda.git@branch-23.06" test-unittest: "RAPIDS_DATASET_ROOT_DIR=/__w/cugraph/cugraph/datasets pytest -m sg ./python/cugraph/cugraph/tests" diff --git a/ci/release/apply_wheel_modifications.sh b/ci/release/apply_wheel_modifications.sh index ed291077494..7f8a9767abe 100755 --- a/ci/release/apply_wheel_modifications.sh +++ b/ci/release/apply_wheel_modifications.sh @@ -29,3 +29,8 @@ sed -i "s/raft-dask/raft-dask${CUDA_SUFFIX}/g" python/cugraph/pyproject.toml sed -i "s/pylibcugraph/pylibcugraph${CUDA_SUFFIX}/g" python/cugraph/pyproject.toml sed -i "s/pylibraft/pylibraft${CUDA_SUFFIX}/g" python/cugraph/pyproject.toml sed -i "s/ucx-py/ucx-py${CUDA_SUFFIX}/g" python/cugraph/pyproject.toml + +if [[ $CUDA_SUFFIX == "-cu12" ]]; then + sed -i "s/cupy-cuda11x/cupy-cuda12x/g" python/cugraph/pyproject.toml + sed -i "s/numba[<=>\.,0-9]*/numba>=0.57/g" python/cugraph/pyproject.toml +fi diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index cf496bd1deb..903dcf98c8e 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -37,8 +37,8 @@ dependencies: - networkx>=2.5.1 - ninja - notebook>=0.5.0 -- numba>=0.56.2 -- numpy>=1.21 +- numba>=0.56.4,<0.57 +- numpy>=1.21,<1.24 - numpydoc - nvcc_linux-64=11.8 - openmpi diff --git a/dependencies.yaml b/dependencies.yaml index a8b0dd172cf..46803bb8173 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -303,7 +303,7 @@ dependencies: - &distributed distributed==2023.3.2.1 - &dask_cuda dask-cuda==23.6.* - &dask_cudf dask-cudf==23.6.* - - &numba numba>=0.56.2 + - &numba numba>=0.56.4,<0.57 - raft-dask==23.6.* - *rmm - &ucx_py ucx-py==0.32.* @@ -329,7 +329,7 @@ dependencies: - output_types: [conda, pyproject] packages: - *numba - - &numpy numpy>=1.21 + - &numpy numpy>=1.21,<1.24 # Temporarily upper bound numpy to avoid overflow deprecations - output_types: [pyproject] packages: - &cugraph cugraph==23.6.* diff --git a/python/cugraph-dgl/pyproject.toml b/python/cugraph-dgl/pyproject.toml index 59b1642c34f..4f54b76c9e6 100644 --- a/python/cugraph-dgl/pyproject.toml +++ b/python/cugraph-dgl/pyproject.toml @@ -20,8 +20,8 @@ license = { text = "Apache 2.0" } requires-python = ">=3.9" dependencies = [ "cugraph==23.6.*", - "numba>=0.56.2", - "numpy>=1.21", + "numba>=0.56.4,<0.57", + "numpy>=1.21,<1.24", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ "Intended Audience :: Developers", diff --git a/python/cugraph-pyg/pyproject.toml b/python/cugraph-pyg/pyproject.toml index b80cbd4bd02..d32bdf50dc8 100644 --- a/python/cugraph-pyg/pyproject.toml +++ b/python/cugraph-pyg/pyproject.toml @@ -27,8 +27,8 @@ classifiers = [ ] dependencies = [ "cugraph==23.6.*", - "numba>=0.56.2", - "numpy>=1.21", + "numba>=0.56.4,<0.57", + "numpy>=1.21,<1.24", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project.urls] diff --git a/python/cugraph-service/server/pyproject.toml b/python/cugraph-service/server/pyproject.toml index 6944dd22043..b4457c0c2d8 100644 --- a/python/cugraph-service/server/pyproject.toml +++ b/python/cugraph-service/server/pyproject.toml @@ -27,7 +27,7 @@ dependencies = [ "dask-cudf==23.6.*", "dask==2023.3.2", "distributed==2023.3.2.1", - "numpy>=1.21", + "numpy>=1.21,<1.24", "rmm==23.6.*", "thriftpy2", "ucx-py==0.32.*", @@ -45,7 +45,7 @@ test = [ "aiohttp", "fsspec[http]>=0.6.0", "networkx>=2.5.1", - "numpy>=1.21", + "numpy>=1.21,<1.24", "pandas", "pytest", "pytest-benchmark", diff --git a/python/cugraph/pyproject.toml b/python/cugraph/pyproject.toml index c3b534e2fde..f31f54c1f3d 100644 --- a/python/cugraph/pyproject.toml +++ b/python/cugraph/pyproject.toml @@ -35,7 +35,7 @@ dependencies = [ "dask-cudf==23.6.*", "dask==2023.3.2", "distributed==2023.3.2.1", - "numba>=0.56.2", + "numba>=0.56.4,<0.57", "pylibcugraph==23.6.*", "raft-dask==23.6.*", "rmm==23.6.*", @@ -53,7 +53,7 @@ test = [ "aiohttp", "fsspec[http]>=0.6.0", "networkx>=2.5.1", - "numpy>=1.21", + "numpy>=1.21,<1.24", "pandas", "pytest", "pytest-benchmark", diff --git a/python/pylibcugraph/pyproject.toml b/python/pylibcugraph/pyproject.toml index ac4786653b5..982c435efb5 100644 --- a/python/pylibcugraph/pyproject.toml +++ b/python/pylibcugraph/pyproject.toml @@ -42,7 +42,7 @@ classifiers = [ test = [ "cudf==23.6.*", "networkx>=2.5.1", - "numpy>=1.21", + "numpy>=1.21,<1.24", "pandas", "pytest", "pytest-benchmark",