From 50b83c8e496a4940c1f8b3770e80da4359bb913d Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Sun, 31 Mar 2024 06:35:01 +0000 Subject: [PATCH 01/63] Get libcugraph building --- cpp/cmake/thirdparty/get_raft.cmake | 2 +- python/libcugraph/CMakeLists.txt | 87 ++++++++++++++++++++++++ python/libcugraph/LICENSE | 1 + python/libcugraph/README.md | 1 + python/libcugraph/libcugraph/VERSION | 1 + python/libcugraph/libcugraph/__init__.py | 17 +++++ python/libcugraph/libcugraph/_version.py | 26 +++++++ python/libcugraph/pyproject.toml | 51 ++++++++++++++ 8 files changed, 185 insertions(+), 1 deletion(-) create mode 100644 python/libcugraph/CMakeLists.txt create mode 120000 python/libcugraph/LICENSE create mode 120000 python/libcugraph/README.md create mode 100644 python/libcugraph/libcugraph/VERSION create mode 100644 python/libcugraph/libcugraph/__init__.py create mode 100644 python/libcugraph/libcugraph/_version.py create mode 100644 python/libcugraph/pyproject.toml diff --git a/cpp/cmake/thirdparty/get_raft.cmake b/cpp/cmake/thirdparty/get_raft.cmake index 8f56372c81..6274005733 100644 --- a/cpp/cmake/thirdparty/get_raft.cmake +++ b/cpp/cmake/thirdparty/get_raft.cmake @@ -39,7 +39,7 @@ function(find_and_configure_raft) endif() rapids_cpm_find(raft ${PKG_VERSION} - GLOBAL_TARGETS raft::raft + GLOBAL_TARGETS raft::raft raft::raft_lib BUILD_EXPORT_SET cugraph-exports INSTALL_EXPORT_SET cugraph-exports COMPONENTS ${RAFT_COMPONENTS} diff --git a/python/libcugraph/CMakeLists.txt b/python/libcugraph/CMakeLists.txt new file mode 100644 index 0000000000..c99413dc2a --- /dev/null +++ b/python/libcugraph/CMakeLists.txt @@ -0,0 +1,87 @@ +# ============================================================================= +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + +cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR) + +include(../../rapids_config.cmake) + +include(rapids-cuda) +rapids_cuda_init_architectures(libcugraph-python) + +project( + libcugraph-python + VERSION "${RAPIDS_VERSION}" + LANGUAGES CXX CUDA +) + +# TODO: Temporarily not using cugraph-ops for wheels +option(USE_CUGRAPH_OPS "Enable all functions that call cugraph-ops" OFF) + +if(NOT USE_CUGRAPH_OPS) + message(STATUS "Disabling libcugraph functions that reference cugraph-ops") + add_compile_definitions(NO_CUGRAPH_OPS) +endif() + +set(BUILD_TESTS OFF) +set(BUILD_CUGRAPH_MG_TESTS OFF) +set(BUILD_CUGRAPH_OPS_CPP_TESTS OFF) +set(CUDA_STATIC_RUNTIME ON) + +add_subdirectory(../../cpp cugraph-cpp) + +# Register a library that is installed to site-packages for rpath linking +function(rapids_cython_register_site_packages_library target) + # TODO: Is this solution robust? Should we also check for other configs, or the default IMPORTED_LOCATION? + get_property(target_library TARGET "${target}" PROPERTY IMPORTED_LOCATION_RELEASE) + cmake_path(GET target_library PARENT_PATH library_dir) + + find_package(Python REQUIRED) + # TODO: Should we support other install locations like user site? + cmake_path(IS_PREFIX Python_SITELIB "${library_dir}" library_is_in_site) + # We cannot support libraries that are not installed to site-packages + # TODO: We do need to still allow building in such cases. For now the easiest + # thing I can think of is just a status message here because a warning seems + # pretty loud for standard uses cases like conda or devcontainers where this + # won't link. + if(NOT library_is_in_site) + message(STATUS "Library ${target} is not installed to site-packages. This is not supported.") + return() + endif() + + cmake_path(RELATIVE_PATH library_dir BASE_DIRECTORY "${Python_SITELIB}" + OUTPUT_VARIABLE library_relative_path) + set_property(GLOBAL PROPERTY "rapids_cython_registered_libraries_${target}" "${library_relative_path}") +endfunction() + +#find_package(raft "${RAPIDS_VERSION}" REQUIRED COMPONENTS compiled) +set(CUGRAPH_COMPILE_RAFT_LIB ON) +include(../../cpp/cmake/thirdparty/get_raft.cmake) +if (NOT TARGET raft::raft_lib) + message(FATAL_ERROR "raft::raft_lib target not found. Please make sure to build raft before building libcugraph.") +endif() +rapids_cython_register_site_packages_library(raft::raft_lib) + +function(rapids_cython_link_to_site_packages_library target linked_target) + get_property(library_relative_path GLOBAL PROPERTY "rapids_cython_registered_libraries_${linked_target}") + if(NOT library_relative_path) + message(STATUS "Library ${linked_target} is not registered as a site-packages library, not updating RPATH of ${target}") + return() + endif() + + cmake_path(RELATIVE_PATH PROJECT_SOURCE_DIR BASE_DIRECTORY "${CMAKE_CURRENT_LIST_DIR}" + OUTPUT_VARIABLE path_to_site) + set_property(TARGET "${target}" APPEND PROPERTY INSTALL_RPATH "\$ORIGIN/${path_to_site}/${library_relative_path}") +endfunction() + +rapids_cython_link_to_site_packages_library(cugraph raft::raft_lib) diff --git a/python/libcugraph/LICENSE b/python/libcugraph/LICENSE new file mode 120000 index 0000000000..30cff7403d --- /dev/null +++ b/python/libcugraph/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/python/libcugraph/README.md b/python/libcugraph/README.md new file mode 120000 index 0000000000..fe84005413 --- /dev/null +++ b/python/libcugraph/README.md @@ -0,0 +1 @@ +../../README.md \ No newline at end of file diff --git a/python/libcugraph/libcugraph/VERSION b/python/libcugraph/libcugraph/VERSION new file mode 100644 index 0000000000..0bff6981a3 --- /dev/null +++ b/python/libcugraph/libcugraph/VERSION @@ -0,0 +1 @@ +24.06.00 diff --git a/python/libcugraph/libcugraph/__init__.py b/python/libcugraph/libcugraph/__init__.py new file mode 100644 index 0000000000..26ae4adb26 --- /dev/null +++ b/python/libcugraph/libcugraph/__init__.py @@ -0,0 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is simply used to make librmm a real package rather than a namespace +# package to work around https://github.com/scikit-build/scikit-build-core/issues/682. +# Since we have it, we may as well also set up some helpful metadata. +from libcugraph._version import __git_commit__, __version__ diff --git a/python/libcugraph/libcugraph/_version.py b/python/libcugraph/libcugraph/_version.py new file mode 100644 index 0000000000..cb96e5d750 --- /dev/null +++ b/python/libcugraph/libcugraph/_version.py @@ -0,0 +1,26 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import importlib.resources + +# Read VERSION file from the module that is symlinked to VERSION file +# in the root of the repo at build time or copied to the moudle at +# installation. VERSION is a separate file that allows CI build-time scripts +# to update version info (including commit hashes) without modifying +# source files. +__version__ = ( + importlib.resources.files("libcugraph").joinpath("VERSION").read_text().strip() +) +__git_commit__ = "" diff --git a/python/libcugraph/pyproject.toml b/python/libcugraph/pyproject.toml new file mode 100644 index 0000000000..623c1b1abd --- /dev/null +++ b/python/libcugraph/pyproject.toml @@ -0,0 +1,51 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +[build-system] +requires = [ + "cmake>=3.26.4", + "cython>=3.0.0", + "ninja", + "pylibraft==24.6.*", + "rmm==24.6.*", + "scikit-build-core[pyproject]>=0.7.0", + "setuptools>=61.0.0", + "wheel", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +build-backend = "scikit_build_core.build" + +[project] +name = "libcugraph" +dynamic = ["version"] +description = "libcugraph - The libcugraph cuGraph C/C++/CUDA library" +readme = { file = "README.md", content-type = "text/markdown" } +authors = [ + { name = "NVIDIA Corporation" }, +] +license = { text = "Apache 2.0" } +requires-python = ">=3.9" +dependencies = [ + "pylibraft==24.6.*", + "rmm==24.6.*", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +classifiers = [ + "Intended Audience :: Developers", + "Programming Language :: C++", + "Environment :: GPU :: NVIDIA CUDA", +] + +[project.urls] +Homepage = "https://github.com/rapidsai/cugraph" +Documentation = "https://docs.rapids.ai/api/cugraph/stable/" + +[tool.scikit-build] +build-dir = "build/{wheel_tag}" +cmake.build-type = "Release" +cmake.minimum-version = "3.26.4" +ninja.make-fallback = true +sdist.reproducible = true +wheel.packages = ["libcugraph"] + +[tool.scikit-build.metadata.version] +provider = "scikit_build_core.metadata.regex" +input = "libcugraph/VERSION" +regex = "(?P.*)" From fdbaf78962d7f4d33ee0a0afc39bb23107afcdb8 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Sun, 31 Mar 2024 06:37:18 +0000 Subject: [PATCH 02/63] Add entry point --- python/libcugraph/pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/libcugraph/pyproject.toml b/python/libcugraph/pyproject.toml index 623c1b1abd..544300de51 100644 --- a/python/libcugraph/pyproject.toml +++ b/python/libcugraph/pyproject.toml @@ -37,6 +37,9 @@ classifiers = [ Homepage = "https://github.com/rapidsai/cugraph" Documentation = "https://docs.rapids.ai/api/cugraph/stable/" +[project.entry-points."cmake.prefix"] +libcugraph = "libcugraph" + [tool.scikit-build] build-dir = "build/{wheel_tag}" cmake.build-type = "Release" From a88178497d7993f9a9810e548532dec2d3c170a6 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Sun, 31 Mar 2024 06:46:35 +0000 Subject: [PATCH 03/63] Update dependencies.yaml --- dependencies.yaml | 38 ++++++++++++++++++++++++++++-- python/cugraph/pyproject.toml | 4 ++-- python/libcugraph/pyproject.toml | 11 ++++----- python/pylibcugraph/pyproject.toml | 4 ++-- 4 files changed, 44 insertions(+), 13 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 85fb1344ce..8a7b5906f4 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -64,6 +64,22 @@ files: - test_python_common - test_python_cugraph - test_python_pylibcugraph + py_build_libcugraph: + output: pyproject + pyproject_dir: python/libcugraph + extras: + table: build-system + includes: + - common_build + - libcugraph_build + - cpp_wheels + py_run_libcugraph: + output: pyproject + pyproject_dir: python/libcugraph + extras: + table: project + includes: + - cpp_wheels py_build_cugraph: output: pyproject pyproject_dir: python/cugraph @@ -71,11 +87,11 @@ files: table: build-system includes: - common_build - - python_build_wheel - depends_on_rmm - depends_on_pylibraft - depends_on_pylibcugraph - python_build_cythonize + - libcugraph_wheel py_run_cugraph: output: pyproject pyproject_dir: python/cugraph @@ -89,6 +105,7 @@ files: - depends_on_pylibcugraph - depends_on_cupy - python_run_cugraph + - libcugraph_wheel py_test_cugraph: output: pyproject pyproject_dir: python/cugraph @@ -105,10 +122,10 @@ files: table: build-system includes: - common_build - - python_build_wheel - depends_on_rmm - depends_on_pylibraft - python_build_cythonize + - libcugraph_wheel py_run_pylibcugraph: output: pyproject pyproject_dir: python/pylibcugraph @@ -117,6 +134,7 @@ files: includes: - depends_on_rmm - depends_on_pylibraft + - libcugraph_wheel py_test_pylibcugraph: output: pyproject pyproject_dir: python/pylibcugraph @@ -442,6 +460,22 @@ dependencies: - output_types: [pyproject, requirements] packages: - scikit-build-core[pyproject]>=0.7.0 + libcugraph_wheel: + common: + - output_types: [pyproject, requirements] + packages: + - libcugraph==24.6.* + libcugraph_build: + common: + - output_types: [pyproject, requirements] + packages: + - scikit-build-core[pyproject]>=0.7.0 + cpp_wheels: + common: + - output_types: [requirements, pyproject] + packages: + - librmm==24.6.* + - libraft==24.6.* python_run_cugraph: common: - output_types: [conda, pyproject] diff --git a/python/cugraph/pyproject.toml b/python/cugraph/pyproject.toml index b29d6f80ff..cf7938b4e6 100644 --- a/python/cugraph/pyproject.toml +++ b/python/cugraph/pyproject.toml @@ -5,13 +5,12 @@ requires = [ "cmake>=3.26.4", "cython>=3.0.0", + "libcugraph==24.6.*", "ninja", "pylibcugraph==24.6.*", "pylibraft==24.6.*", "rmm==24.6.*", "scikit-build-core[pyproject]>=0.7.0", - "setuptools>=61.0.0", - "wheel", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. build-backend = "scikit_build_core.build" @@ -34,6 +33,7 @@ dependencies = [ "dask-cuda==24.6.*", "dask-cudf==24.6.*", "fsspec[http]>=0.6.0", + "libcugraph==24.6.*", "numba>=0.57", "numpy>=1.23,<2.0a0", "pylibcugraph==24.6.*", diff --git a/python/libcugraph/pyproject.toml b/python/libcugraph/pyproject.toml index 544300de51..420a8142d3 100644 --- a/python/libcugraph/pyproject.toml +++ b/python/libcugraph/pyproject.toml @@ -3,13 +3,10 @@ [build-system] requires = [ "cmake>=3.26.4", - "cython>=3.0.0", + "libraft==24.6.*", + "librmm==24.6.*", "ninja", - "pylibraft==24.6.*", - "rmm==24.6.*", "scikit-build-core[pyproject]>=0.7.0", - "setuptools>=61.0.0", - "wheel", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. build-backend = "scikit_build_core.build" @@ -24,8 +21,8 @@ authors = [ license = { text = "Apache 2.0" } requires-python = ">=3.9" dependencies = [ - "pylibraft==24.6.*", - "rmm==24.6.*", + "libraft==24.6.*", + "librmm==24.6.*", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ "Intended Audience :: Developers", diff --git a/python/pylibcugraph/pyproject.toml b/python/pylibcugraph/pyproject.toml index 19eafe55b7..3e77c742a5 100644 --- a/python/pylibcugraph/pyproject.toml +++ b/python/pylibcugraph/pyproject.toml @@ -5,12 +5,11 @@ requires = [ "cmake>=3.26.4", "cython>=3.0.0", + "libcugraph==24.6.*", "ninja", "pylibraft==24.6.*", "rmm==24.6.*", "scikit-build-core[pyproject]>=0.7.0", - "setuptools>=61.0.0", - "wheel", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. build-backend = "scikit_build_core.build" @@ -28,6 +27,7 @@ authors = [ license = { text = "Apache 2.0" } requires-python = ">=3.9" dependencies = [ + "libcugraph==24.6.*", "pylibraft==24.6.*", "rmm==24.6.*", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From e1ef21573ef2784b0e32dccd69a2fb647af2738e Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Sun, 31 Mar 2024 07:32:08 +0000 Subject: [PATCH 04/63] Build pylibcugraph wheel against libcugraph wheel --- python/libcugraph/pyproject.toml | 1 + python/pylibcugraph/CMakeLists.txt | 71 ++++++++++--------- .../pylibcugraph/pylibcugraph/CMakeLists.txt | 3 + .../pylibcugraph/components/CMakeLists.txt | 5 +- .../internal_types/CMakeLists.txt | 5 +- .../pylibcugraph/testing/CMakeLists.txt | 5 +- 6 files changed, 55 insertions(+), 35 deletions(-) diff --git a/python/libcugraph/pyproject.toml b/python/libcugraph/pyproject.toml index 420a8142d3..beb676c8e7 100644 --- a/python/libcugraph/pyproject.toml +++ b/python/libcugraph/pyproject.toml @@ -44,6 +44,7 @@ cmake.minimum-version = "3.26.4" ninja.make-fallback = true sdist.reproducible = true wheel.packages = ["libcugraph"] +wheel.install-dir = "libcugraph" [tool.scikit-build.metadata.version] provider = "scikit_build_core.metadata.regex" diff --git a/python/pylibcugraph/CMakeLists.txt b/python/pylibcugraph/CMakeLists.txt index f43b7db127..4e468dbade 100644 --- a/python/pylibcugraph/CMakeLists.txt +++ b/python/pylibcugraph/CMakeLists.txt @@ -16,7 +16,7 @@ cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR) include(../../rapids_config.cmake) -# We always need CUDA for cuML because the raft dependency brings in a +# We always need CUDA for pylibcugraph because the raft dependency brings in a # header-only cuco dependency that enables CUDA unconditionally. include(rapids-cuda) rapids_cuda_init_architectures(pylibcugraph-python) @@ -27,12 +27,8 @@ project( LANGUAGES CXX CUDA ) -################################################################################ -# - User Options -------------------------------------------------------------- -option(FIND_CUGRAPH_CPP "Search for existing CUGRAPH C++ installations before defaulting to local files" - OFF -) -option(USE_CUGRAPH_OPS "Enable all functions that call cugraph-ops" ON) +# TODO: Temporarily not using cugraph-ops for wheels +option(USE_CUGRAPH_OPS "Enable all functions that call cugraph-ops" OFF) if(NOT USE_CUGRAPH_OPS) message(STATUS "Disabling libcugraph functions that reference cugraph-ops") @@ -40,36 +36,47 @@ if(NOT USE_CUGRAPH_OPS) endif() # If the user requested it we attempt to find CUGRAPH. -if(FIND_CUGRAPH_CPP) - find_package(cugraph "${RAPIDS_VERSION}" REQUIRED) -else() - set(cugraph_FOUND OFF) -endif() +find_package(cugraph "${RAPIDS_VERSION}" REQUIRED) -include(rapids-cython-core) +# Register a library that is installed to site-packages for rpath linking +function(rapids_cython_register_site_packages_library target) + # TODO: Is this solution robust? Should we also check for other configs, or the default IMPORTED_LOCATION? + get_property(target_library TARGET "${target}" PROPERTY IMPORTED_LOCATION_RELEASE) + cmake_path(GET target_library PARENT_PATH library_dir) -if (NOT cugraph_FOUND) - set(BUILD_TESTS OFF) - set(BUILD_CUGRAPH_MG_TESTS OFF) - set(BUILD_CUGRAPH_OPS_CPP_TESTS OFF) - set(CUDA_STATIC_RUNTIME ON) - set(USE_RAFT_STATIC ON) - set(CUGRAPH_COMPILE_RAFT_LIB ON) - set(CUGRAPH_USE_CUGRAPH_OPS_STATIC ON) - set(CUGRAPH_EXCLUDE_CUGRAPH_OPS_FROM_ALL ON) - set(ALLOW_CLONE_CUGRAPH_OPS ON) + find_package(Python REQUIRED) + # TODO: Should we support other install locations like user site? + cmake_path(IS_PREFIX Python_SITELIB "${library_dir}" library_is_in_site) + # We cannot support libraries that are not installed to site-packages + # TODO: We do need to still allow building in such cases. For now the easiest + # thing I can think of is just a status message here because a warning seems + # pretty loud for standard uses cases like conda or devcontainers where this + # won't link. + if(NOT library_is_in_site) + message(STATUS "Library ${target} is not installed to site-packages. This is not supported.") + return() + endif() - add_subdirectory(../../cpp cugraph-cpp EXCLUDE_FROM_ALL) + cmake_path(RELATIVE_PATH library_dir BASE_DIRECTORY "${Python_SITELIB}" + OUTPUT_VARIABLE library_relative_path) + set_property(GLOBAL PROPERTY "rapids_cython_registered_libraries_${target}" "${library_relative_path}") +endfunction() - set(cython_lib_dir pylibcugraph) - install(TARGETS cugraph DESTINATION ${cython_lib_dir}) - install(TARGETS cugraph_c DESTINATION ${cython_lib_dir}) -endif() +rapids_cython_register_site_packages_library(cugraph::cugraph) + +function(rapids_cython_link_to_site_packages_library target linked_target) + get_property(library_relative_path GLOBAL PROPERTY "rapids_cython_registered_libraries_${linked_target}") + if(NOT library_relative_path) + message(STATUS "Library ${linked_target} is not registered as a site-packages library, not updating RPATH of ${target}") + return() + endif() + cmake_path(RELATIVE_PATH PROJECT_SOURCE_DIR BASE_DIRECTORY "${CMAKE_CURRENT_LIST_DIR}" + OUTPUT_VARIABLE path_to_site) + set_property(TARGET "${target}" APPEND PROPERTY INSTALL_RPATH "\$ORIGIN/${path_to_site}/${library_relative_path}") +endfunction() + +include(rapids-cython-core) rapids_cython_init() add_subdirectory(pylibcugraph) - -if(DEFINED cython_lib_dir) - rapids_cython_add_rpath_entries(TARGET cugraph PATHS "${cython_lib_dir}") -endif() diff --git a/python/pylibcugraph/pylibcugraph/CMakeLists.txt b/python/pylibcugraph/pylibcugraph/CMakeLists.txt index 7cc9014594..119ae1800b 100644 --- a/python/pylibcugraph/pylibcugraph/CMakeLists.txt +++ b/python/pylibcugraph/pylibcugraph/CMakeLists.txt @@ -67,3 +67,6 @@ rapids_cython_create_modules( LINKED_LIBRARIES ${linked_libraries} ASSOCIATED_TARGETS cugraph ) +foreach(target IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) + rapids_cython_link_to_site_packages_library(${target} cugraph::cugraph) +endforeach() diff --git a/python/pylibcugraph/pylibcugraph/components/CMakeLists.txt b/python/pylibcugraph/pylibcugraph/components/CMakeLists.txt index 0d8997e3da..6ff544ac72 100644 --- a/python/pylibcugraph/pylibcugraph/components/CMakeLists.txt +++ b/python/pylibcugraph/pylibcugraph/components/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -23,3 +23,6 @@ rapids_cython_create_modules( LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS cugraph ) +foreach(target IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) + rapids_cython_link_to_site_packages_library(${target} cugraph::cugraph) +endforeach() diff --git a/python/pylibcugraph/pylibcugraph/internal_types/CMakeLists.txt b/python/pylibcugraph/pylibcugraph/internal_types/CMakeLists.txt index 1ca169c586..b296650564 100644 --- a/python/pylibcugraph/pylibcugraph/internal_types/CMakeLists.txt +++ b/python/pylibcugraph/pylibcugraph/internal_types/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -23,3 +23,6 @@ rapids_cython_create_modules( LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS cugraph ) +foreach(target IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) + rapids_cython_link_to_site_packages_library(${target} cugraph::cugraph) +endforeach() diff --git a/python/pylibcugraph/pylibcugraph/testing/CMakeLists.txt b/python/pylibcugraph/pylibcugraph/testing/CMakeLists.txt index 8bf8969b1a..6f7a92e6c0 100644 --- a/python/pylibcugraph/pylibcugraph/testing/CMakeLists.txt +++ b/python/pylibcugraph/pylibcugraph/testing/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -23,3 +23,6 @@ rapids_cython_create_modules( LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS cugraph ) +foreach(target IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) + rapids_cython_link_to_site_packages_library(${target} cugraph::cugraph) +endforeach() From a9789db6c51cfaacb5bc51855f60bc047f5de79a Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Sun, 31 Mar 2024 07:32:32 +0000 Subject: [PATCH 05/63] Add missing cupy dep --- dependencies.yaml | 1 + python/pylibcugraph/pyproject.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/dependencies.yaml b/dependencies.yaml index 8a7b5906f4..cb9dc753d3 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -132,6 +132,7 @@ files: extras: table: project includes: + - depends_on_cupy - depends_on_rmm - depends_on_pylibraft - libcugraph_wheel diff --git a/python/pylibcugraph/pyproject.toml b/python/pylibcugraph/pyproject.toml index 3e77c742a5..7121f4412c 100644 --- a/python/pylibcugraph/pyproject.toml +++ b/python/pylibcugraph/pyproject.toml @@ -27,6 +27,7 @@ authors = [ license = { text = "Apache 2.0" } requires-python = ">=3.9" dependencies = [ + "cupy-cuda11x>=12.0.0", "libcugraph==24.6.*", "pylibraft==24.6.*", "rmm==24.6.*", From 990c3eb14471ca4938fa892396f1cc5b65737523 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Sun, 31 Mar 2024 14:59:20 +0000 Subject: [PATCH 06/63] Fix cugraph CMake to properly support using dynamic raft with static cudart --- cpp/CMakeLists.txt | 48 ++++++++++++++++++---------------------------- 1 file changed, 19 insertions(+), 29 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 0240e2b892..6ad514d339 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -339,48 +339,38 @@ target_include_directories(cugraph "$" ) -set(COMPILED_RAFT_LIB "") if(CUDA_STATIC_RUNTIME) get_target_property(_includes raft::raft INTERFACE_INCLUDE_DIRECTORIES) target_include_directories(cugraph PUBLIC ${_includes}) # Add CTK include paths because we make our CTK library links private below target_include_directories(cugraph SYSTEM PUBLIC ${CUDAToolkit_INCLUDE_DIRS}) - if(CUGRAPH_COMPILE_RAFT_LIB) +endif() + +set(COMPILED_RAFT_LIB "") +if(CUGRAPH_COMPILE_RAFT_LIB) + if(USE_RAFT_STATIC) set(COMPILED_RAFT_LIB raft::compiled_static) - endif() -else() - if(CUGRAPH_COMPILE_RAFT_LIB) + else() set(COMPILED_RAFT_LIB raft::compiled) endif() endif() ################################################################################ # - link libraries ------------------------------------------------------------- +target_link_libraries(cugraph + PUBLIC + rmm::rmm + $<$>:raft::raft> + $<$>:${COMPILED_RAFT_LIB}> + PRIVATE + $<$:raft::raft> + $<$:${COMPILED_RAFT_LIB}> + cuco::cuco + NCCL::NCCL +) + if (USE_CUGRAPH_OPS) - target_link_libraries(cugraph - PUBLIC - rmm::rmm - cugraph-ops::cugraph-ops++ - $<$>:raft::raft> - $<$>:${COMPILED_RAFT_LIB}> - PRIVATE - $<$:raft::raft> - $<$:${COMPILED_RAFT_LIB}> - cuco::cuco - NCCL::NCCL - ) -else() - target_link_libraries(cugraph - PUBLIC - rmm::rmm - $<$>:raft::raft> - $<$>:${COMPILED_RAFT_LIB}> - PRIVATE - $<$:raft::raft> - $<$:${COMPILED_RAFT_LIB}> - cuco::cuco - NCCL::NCCL - ) + target_link_libraries(cugraph PUBLIC cugraph-ops::cugraph-ops++) endif() ################################################################################ From 0b0e3a98f57ed852f78e93cd5c19d7fcd10564cf Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 1 Apr 2024 00:33:25 +0000 Subject: [PATCH 07/63] Switch from RPATH hacking to dynamic loading with ctypes --- python/cugraph/CMakeLists.txt | 37 ++------------ python/libcugraph/CMakeLists.txt | 49 +------------------ python/libcugraph/libcugraph/__init__.py | 1 + python/libcugraph/libcugraph/load.py | 43 ++++++++++++++++ python/pylibcugraph/CMakeLists.txt | 39 --------------- .../pylibcugraph/pylibcugraph/CMakeLists.txt | 3 -- python/pylibcugraph/pylibcugraph/__init__.py | 12 +++++ .../pylibcugraph/components/CMakeLists.txt | 5 +- .../internal_types/CMakeLists.txt | 5 +- .../pylibcugraph/testing/CMakeLists.txt | 5 +- 10 files changed, 66 insertions(+), 133 deletions(-) create mode 100644 python/libcugraph/libcugraph/load.py diff --git a/python/cugraph/CMakeLists.txt b/python/cugraph/CMakeLists.txt index dfccf02d04..8254da60ec 100644 --- a/python/cugraph/CMakeLists.txt +++ b/python/cugraph/CMakeLists.txt @@ -27,11 +27,6 @@ project( LANGUAGES CXX CUDA ) -################################################################################ -# - User Options -------------------------------------------------------------- -option(FIND_CUGRAPH_CPP "Search for existing CUGRAPH C++ installations before defaulting to local files" - OFF -) option(USE_CUGRAPH_OPS "Enable all functions that call cugraph-ops" ON) if(NOT USE_CUGRAPH_OPS) @@ -39,31 +34,13 @@ if(NOT USE_CUGRAPH_OPS) add_compile_definitions(NO_CUGRAPH_OPS) endif() -# If the user requested it, we attempt to find CUGRAPH. -if(FIND_CUGRAPH_CPP) - find_package(cugraph "${RAPIDS_VERSION}" REQUIRED) -else() - set(cugraph_FOUND OFF) -endif() - -include(rapids-cython-core) - -if(NOT cugraph_FOUND) - set(BUILD_TESTS OFF) - set(BUILD_CUGRAPH_MG_TESTS OFF) - set(BUILD_CUGRAPH_OPS_CPP_TESTS OFF) - set(CUDA_STATIC_RUNTIME ON) - set(USE_RAFT_STATIC ON) - set(CUGRAPH_COMPILE_RAFT_LIB ON) - set(CUGRAPH_USE_CUGRAPH_OPS_STATIC ON) - set(CUGRAPH_EXCLUDE_CUGRAPH_OPS_FROM_ALL ON) - set(ALLOW_CLONE_CUGRAPH_OPS ON) +find_package(cugraph "${RAPIDS_VERSION}" REQUIRED) - add_subdirectory(../../cpp cugraph-cpp EXCLUDE_FROM_ALL) +# TODO: Wait until this is implemented in rapids-cmake. At that point also +# should have the llinking automated as part of rapids_cython_create_modules. +rapids_cython_register_site_packages_library(cugraph::cugraph) - set(cython_lib_dir cugraph) - install(TARGETS cugraph DESTINATION ${cython_lib_dir}) -endif() +include(rapids-cython-core) rapids_cython_init() @@ -76,7 +53,3 @@ add_subdirectory(cugraph/linear_assignment) add_subdirectory(cugraph/structure) add_subdirectory(cugraph/tree) add_subdirectory(cugraph/utilities) - -if(DEFINED cython_lib_dir) - rapids_cython_add_rpath_entries(TARGET cugraph PATHS "${cython_lib_dir}") -endif() diff --git a/python/libcugraph/CMakeLists.txt b/python/libcugraph/CMakeLists.txt index c99413dc2a..ae08234059 100644 --- a/python/libcugraph/CMakeLists.txt +++ b/python/libcugraph/CMakeLists.txt @@ -25,7 +25,8 @@ project( LANGUAGES CXX CUDA ) -# TODO: Temporarily not using cugraph-ops for wheels +# TODO: Temporarily not using cugraph-ops for wheels while testing the new +# libcugraph wheel approach option(USE_CUGRAPH_OPS "Enable all functions that call cugraph-ops" OFF) if(NOT USE_CUGRAPH_OPS) @@ -39,49 +40,3 @@ set(BUILD_CUGRAPH_OPS_CPP_TESTS OFF) set(CUDA_STATIC_RUNTIME ON) add_subdirectory(../../cpp cugraph-cpp) - -# Register a library that is installed to site-packages for rpath linking -function(rapids_cython_register_site_packages_library target) - # TODO: Is this solution robust? Should we also check for other configs, or the default IMPORTED_LOCATION? - get_property(target_library TARGET "${target}" PROPERTY IMPORTED_LOCATION_RELEASE) - cmake_path(GET target_library PARENT_PATH library_dir) - - find_package(Python REQUIRED) - # TODO: Should we support other install locations like user site? - cmake_path(IS_PREFIX Python_SITELIB "${library_dir}" library_is_in_site) - # We cannot support libraries that are not installed to site-packages - # TODO: We do need to still allow building in such cases. For now the easiest - # thing I can think of is just a status message here because a warning seems - # pretty loud for standard uses cases like conda or devcontainers where this - # won't link. - if(NOT library_is_in_site) - message(STATUS "Library ${target} is not installed to site-packages. This is not supported.") - return() - endif() - - cmake_path(RELATIVE_PATH library_dir BASE_DIRECTORY "${Python_SITELIB}" - OUTPUT_VARIABLE library_relative_path) - set_property(GLOBAL PROPERTY "rapids_cython_registered_libraries_${target}" "${library_relative_path}") -endfunction() - -#find_package(raft "${RAPIDS_VERSION}" REQUIRED COMPONENTS compiled) -set(CUGRAPH_COMPILE_RAFT_LIB ON) -include(../../cpp/cmake/thirdparty/get_raft.cmake) -if (NOT TARGET raft::raft_lib) - message(FATAL_ERROR "raft::raft_lib target not found. Please make sure to build raft before building libcugraph.") -endif() -rapids_cython_register_site_packages_library(raft::raft_lib) - -function(rapids_cython_link_to_site_packages_library target linked_target) - get_property(library_relative_path GLOBAL PROPERTY "rapids_cython_registered_libraries_${linked_target}") - if(NOT library_relative_path) - message(STATUS "Library ${linked_target} is not registered as a site-packages library, not updating RPATH of ${target}") - return() - endif() - - cmake_path(RELATIVE_PATH PROJECT_SOURCE_DIR BASE_DIRECTORY "${CMAKE_CURRENT_LIST_DIR}" - OUTPUT_VARIABLE path_to_site) - set_property(TARGET "${target}" APPEND PROPERTY INSTALL_RPATH "\$ORIGIN/${path_to_site}/${library_relative_path}") -endfunction() - -rapids_cython_link_to_site_packages_library(cugraph raft::raft_lib) diff --git a/python/libcugraph/libcugraph/__init__.py b/python/libcugraph/libcugraph/__init__.py index 26ae4adb26..ad367c7ae0 100644 --- a/python/libcugraph/libcugraph/__init__.py +++ b/python/libcugraph/libcugraph/__init__.py @@ -15,3 +15,4 @@ # package to work around https://github.com/scikit-build/scikit-build-core/issues/682. # Since we have it, we may as well also set up some helpful metadata. from libcugraph._version import __git_commit__, __version__ +from libcugraph.load import load_library diff --git a/python/libcugraph/libcugraph/load.py b/python/libcugraph/libcugraph/load.py new file mode 100644 index 0000000000..95364fc6ff --- /dev/null +++ b/python/libcugraph/libcugraph/load.py @@ -0,0 +1,43 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import ctypes +import os + +import libraft + + +def load_library(): + # libraft must be loaded before libcugraph since libcugraph references symbols in + # libraft. + libraft.load_library() + + # Dynamically load libcugraph.so. Prefer a system library if one is present to + # avoid clobbering symbols that other packages might expect, but if no + # other library is present use the one in the wheel. + try: + libcugraph_lib = ctypes.CDLL("libcugraph.so", ctypes.RTLD_GLOBAL) + except OSError: + libcugraph_lib = ctypes.CDLL( + # TODO: Do we always know it will be lib64? Should we consider + # finding a way for CMake to export the path for us to find here? + os.path.join(os.path.dirname(__file__), "lib64", "libcugraph.so"), + ctypes.RTLD_GLOBAL, + ) + + # The caller almost never needs to do anything with this library, but no + # harm in offering the option since this object at least provides a handle + # to inspect where libcugraph was loaded from. + return libcugraph_lib diff --git a/python/pylibcugraph/CMakeLists.txt b/python/pylibcugraph/CMakeLists.txt index 4e468dbade..465a2fd8c7 100644 --- a/python/pylibcugraph/CMakeLists.txt +++ b/python/pylibcugraph/CMakeLists.txt @@ -35,47 +35,8 @@ if(NOT USE_CUGRAPH_OPS) add_compile_definitions(NO_CUGRAPH_OPS) endif() -# If the user requested it we attempt to find CUGRAPH. find_package(cugraph "${RAPIDS_VERSION}" REQUIRED) -# Register a library that is installed to site-packages for rpath linking -function(rapids_cython_register_site_packages_library target) - # TODO: Is this solution robust? Should we also check for other configs, or the default IMPORTED_LOCATION? - get_property(target_library TARGET "${target}" PROPERTY IMPORTED_LOCATION_RELEASE) - cmake_path(GET target_library PARENT_PATH library_dir) - - find_package(Python REQUIRED) - # TODO: Should we support other install locations like user site? - cmake_path(IS_PREFIX Python_SITELIB "${library_dir}" library_is_in_site) - # We cannot support libraries that are not installed to site-packages - # TODO: We do need to still allow building in such cases. For now the easiest - # thing I can think of is just a status message here because a warning seems - # pretty loud for standard uses cases like conda or devcontainers where this - # won't link. - if(NOT library_is_in_site) - message(STATUS "Library ${target} is not installed to site-packages. This is not supported.") - return() - endif() - - cmake_path(RELATIVE_PATH library_dir BASE_DIRECTORY "${Python_SITELIB}" - OUTPUT_VARIABLE library_relative_path) - set_property(GLOBAL PROPERTY "rapids_cython_registered_libraries_${target}" "${library_relative_path}") -endfunction() - -rapids_cython_register_site_packages_library(cugraph::cugraph) - -function(rapids_cython_link_to_site_packages_library target linked_target) - get_property(library_relative_path GLOBAL PROPERTY "rapids_cython_registered_libraries_${linked_target}") - if(NOT library_relative_path) - message(STATUS "Library ${linked_target} is not registered as a site-packages library, not updating RPATH of ${target}") - return() - endif() - - cmake_path(RELATIVE_PATH PROJECT_SOURCE_DIR BASE_DIRECTORY "${CMAKE_CURRENT_LIST_DIR}" - OUTPUT_VARIABLE path_to_site) - set_property(TARGET "${target}" APPEND PROPERTY INSTALL_RPATH "\$ORIGIN/${path_to_site}/${library_relative_path}") -endfunction() - include(rapids-cython-core) rapids_cython_init() diff --git a/python/pylibcugraph/pylibcugraph/CMakeLists.txt b/python/pylibcugraph/pylibcugraph/CMakeLists.txt index 119ae1800b..7cc9014594 100644 --- a/python/pylibcugraph/pylibcugraph/CMakeLists.txt +++ b/python/pylibcugraph/pylibcugraph/CMakeLists.txt @@ -67,6 +67,3 @@ rapids_cython_create_modules( LINKED_LIBRARIES ${linked_libraries} ASSOCIATED_TARGETS cugraph ) -foreach(target IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) - rapids_cython_link_to_site_packages_library(${target} cugraph::cugraph) -endforeach() diff --git a/python/pylibcugraph/pylibcugraph/__init__.py b/python/pylibcugraph/pylibcugraph/__init__.py index dcdef05e10..dd64a51d58 100644 --- a/python/pylibcugraph/pylibcugraph/__init__.py +++ b/python/pylibcugraph/pylibcugraph/__init__.py @@ -11,6 +11,18 @@ # See the License for the specific language governing permissions and # limitations under the License. +# If libcugraph was installed as a wheel, we must request it to load the library +# symbols. Otherwise, we assume that the library was installed in a system path that ld +# can find. +try: + import libcugraph +except ModuleNotFoundError: + pass +else: + libcugraph.load_library() + del libcugraph + + from pylibcugraph.components._connectivity import ( strongly_connected_components, ) diff --git a/python/pylibcugraph/pylibcugraph/components/CMakeLists.txt b/python/pylibcugraph/pylibcugraph/components/CMakeLists.txt index 6ff544ac72..0d8997e3da 100644 --- a/python/pylibcugraph/pylibcugraph/components/CMakeLists.txt +++ b/python/pylibcugraph/pylibcugraph/components/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -23,6 +23,3 @@ rapids_cython_create_modules( LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS cugraph ) -foreach(target IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) - rapids_cython_link_to_site_packages_library(${target} cugraph::cugraph) -endforeach() diff --git a/python/pylibcugraph/pylibcugraph/internal_types/CMakeLists.txt b/python/pylibcugraph/pylibcugraph/internal_types/CMakeLists.txt index b296650564..1ca169c586 100644 --- a/python/pylibcugraph/pylibcugraph/internal_types/CMakeLists.txt +++ b/python/pylibcugraph/pylibcugraph/internal_types/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -23,6 +23,3 @@ rapids_cython_create_modules( LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS cugraph ) -foreach(target IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) - rapids_cython_link_to_site_packages_library(${target} cugraph::cugraph) -endforeach() diff --git a/python/pylibcugraph/pylibcugraph/testing/CMakeLists.txt b/python/pylibcugraph/pylibcugraph/testing/CMakeLists.txt index 6f7a92e6c0..8bf8969b1a 100644 --- a/python/pylibcugraph/pylibcugraph/testing/CMakeLists.txt +++ b/python/pylibcugraph/pylibcugraph/testing/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -23,6 +23,3 @@ rapids_cython_create_modules( LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS cugraph ) -foreach(target IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) - rapids_cython_link_to_site_packages_library(${target} cugraph::cugraph) -endforeach() From a6c9cb1de233e7fac5ae132073479b8e1268b6fb Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 1 Apr 2024 03:04:56 +0000 Subject: [PATCH 08/63] Also load libcugraph_c --- python/libcugraph/libcugraph/load.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/python/libcugraph/libcugraph/load.py b/python/libcugraph/libcugraph/load.py index 95364fc6ff..ad4cccbbf9 100644 --- a/python/libcugraph/libcugraph/load.py +++ b/python/libcugraph/libcugraph/load.py @@ -37,7 +37,20 @@ def load_library(): ctypes.RTLD_GLOBAL, ) - # The caller almost never needs to do anything with this library, but no + # Dynamically load libcugraph_c.so. Prefer a system library if one is present to + # avoid clobbering symbols that other packages might expect, but if no + # other library is present use the one in the wheel. + try: + libcugraph_c_lib = ctypes.CDLL("libcugraph_c.so", ctypes.RTLD_GLOBAL) + except OSError: + libcugraph_c_lib = ctypes.CDLL( + # TODO: Do we always know it will be lib64? Should we consider + # finding a way for CMake to export the path for us to find here? + os.path.join(os.path.dirname(__file__), "lib64", "libcugraph_c.so"), + ctypes.RTLD_GLOBAL, + ) + + # The caller almost never needs to do anything with these libraries, but no # harm in offering the option since this object at least provides a handle # to inspect where libcugraph was loaded from. - return libcugraph_lib + return libcugraph_lib, libcugraph_c_lib From 4ac08db8ad618f76fdfe32280c80b2a4b5dff2a4 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 1 Apr 2024 19:08:36 +0000 Subject: [PATCH 09/63] Try fixing non-relocatable bundling of raft headers --- cpp/CMakeLists.txt | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 6ad514d339..307721c754 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -339,10 +339,12 @@ target_include_directories(cugraph "$" ) -if(CUDA_STATIC_RUNTIME) +if(USE_RAFT_STATIC) get_target_property(_includes raft::raft INTERFACE_INCLUDE_DIRECTORIES) target_include_directories(cugraph PUBLIC ${_includes}) - # Add CTK include paths because we make our CTK library links private below +endif() + +if(CUDA_STATIC_RUNTIME) target_include_directories(cugraph SYSTEM PUBLIC ${CUDAToolkit_INCLUDE_DIRS}) endif() @@ -458,10 +460,12 @@ target_include_directories(cugraph_c "$" ) -if(CUDA_STATIC_RUNTIME) +if(USE_RAFT_STATIC) get_target_property(_includes raft::raft INTERFACE_INCLUDE_DIRECTORIES) target_include_directories(cugraph_c PUBLIC ${_includes}) - # Add CTK include paths because we make our CTK library links private below +endif() + +if(CUDA_STATIC_RUNTIME) target_include_directories(cugraph_c SYSTEM PUBLIC ${CUDAToolkit_INCLUDE_DIRS}) set(_ctk_static_suffix "_static") endif() From 396e7b3af2b997e9239d702e82e15d89f6da5758 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 1 Apr 2024 22:58:03 +0000 Subject: [PATCH 10/63] Enable GHA --- .github/workflows/pr.yaml | 17 +++++++++++-- ci/build_wheel_cpp.sh | 53 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 2 deletions(-) create mode 100755 ci/build_wheel_cpp.sh diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index c04e0e879d..7f945cf401 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -20,6 +20,7 @@ jobs: - conda-python-build - conda-python-tests - docs-build + - wheel-build-cpp - wheel-build-pylibcugraph - wheel-tests-pylibcugraph - wheel-build-cugraph @@ -93,10 +94,22 @@ jobs: arch: "amd64" container_image: "rapidsai/ci-conda:cuda11.8.0-ubuntu22.04-py3.10" run_script: "ci/build_docs.sh" - wheel-build-pylibcugraph: + wheel-build-cpp: needs: checks secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 + with: + matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) + build_type: pull-request + script: ci/build_wheel_cpp.sh + extra-repo: rapidsai/cugraph-ops + extra-repo-sha: branch-24.06 + extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY + node_type: cpu32 + wheel-build-pylibcugraph: + needs: wheel-build-cpp + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 with: build_type: pull-request script: ci/build_wheel_pylibcugraph.sh @@ -112,7 +125,7 @@ jobs: build_type: pull-request script: ci/test_wheel_pylibcugraph.sh wheel-build-cugraph: - needs: wheel-tests-pylibcugraph + needs: wheel-build-pylibcugraph secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 with: diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh new file mode 100755 index 0000000000..a25cb68a43 --- /dev/null +++ b/ci/build_wheel_cpp.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# Copyright (c) 2024, NVIDIA CORPORATION. + +set -euo pipefail + +package_name="libcugraph" +package_dir="python/libcugraph" + +source rapids-configure-sccache +source rapids-date-string + +version=$(rapids-generate-version) +git_commit=$(git rev-parse HEAD) + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + +artifact_name=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_REPOSITORY=rmm PYTHON_VERSION="3.11" rapids-package-name wheel_python) +commit=$(git ls-remote https://github.com/rapidsai/rmm.git refs/heads/pull-request/1512 | cut -c1-7) +librmm_wheelhouse=$(rapids-get-artifact "ci/rmm/pull-request/1512/${commit}/${artifact_name}") + +artifact_name=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_REPOSITORY=raft PYTHON_VERSION="3.11" rapids-package-name wheel_python) +commit=$(git ls-remote https://github.com/rapidsai/raft.git refs/heads/pull-request/1512 | cut -c1-7) +libraft_wheelhouse=$(rapids-get-artifact "ci/raft/pull-request/1512/${commit}/${artifact_name}") + +# This is the version of the suffix with a preceding hyphen. It's used +# everywhere except in the final wheel name. +PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" + +pyproject_file="${package_dir}/pyproject.toml" +version_file="${package_dir}/${version_package_name}/_version.py" + +sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} +echo "${version}" > VERSION +sed -i "/^__git_commit__ / s/= .*/= \"${git_commit}\"/g" ${version_file} + +# For nightlies we want to ensure that we're pulling in alphas as well. The +# easiest way to do so is to augment the spec with a constraint containing a +# min alpha version that doesn't affect the version bounds but does allow usage +# of alpha versions for that dependency without --pre +alpha_spec='' +if ! rapids-is-release-build; then + alpha_spec=',>=0.0.0a0' +fi + +for dep in librmm libraft; do + sed -r -i "s/${dep}==(.*)\"/${dep}${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} +done + +cd "${package_dir}" + +PIP_FIND_LINKS="${librmm_wheelhouse};${libraft_wheelhouse}" python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check + +RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-upload-wheels-to-s3 dist From 2c6617e53acc317bcdd23874608b414a30765ce2 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 1 Apr 2024 23:12:32 +0000 Subject: [PATCH 11/63] Fix PR number for raft --- ci/build_wheel_cpp.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index a25cb68a43..80960701e8 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -19,8 +19,8 @@ commit=$(git ls-remote https://github.com/rapidsai/rmm.git refs/heads/pull-reque librmm_wheelhouse=$(rapids-get-artifact "ci/rmm/pull-request/1512/${commit}/${artifact_name}") artifact_name=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_REPOSITORY=raft PYTHON_VERSION="3.11" rapids-package-name wheel_python) -commit=$(git ls-remote https://github.com/rapidsai/raft.git refs/heads/pull-request/1512 | cut -c1-7) -libraft_wheelhouse=$(rapids-get-artifact "ci/raft/pull-request/1512/${commit}/${artifact_name}") +commit=$(git ls-remote https://github.com/rapidsai/raft.git refs/heads/pull-request/2251 | cut -c1-7) +libraft_wheelhouse=$(rapids-get-artifact "ci/raft/pull-request/2251/${commit}/${artifact_name}") # This is the version of the suffix with a preceding hyphen. It's used # everywhere except in the final wheel name. From 24c6c4ce3196b4d44f409b79ec110a9f25b82a12 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 1 Apr 2024 23:20:36 +0000 Subject: [PATCH 12/63] Fix py variables --- ci/build_wheel_cpp.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index 80960701e8..da8f076c4a 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -14,11 +14,11 @@ git_commit=$(git rev-parse HEAD) RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -artifact_name=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_REPOSITORY=rmm PYTHON_VERSION="3.11" rapids-package-name wheel_python) +artifact_name=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_REPOSITORY=rmm RAPIDS_PY_VERSION="3.11" rapids-package-name wheel_python) commit=$(git ls-remote https://github.com/rapidsai/rmm.git refs/heads/pull-request/1512 | cut -c1-7) librmm_wheelhouse=$(rapids-get-artifact "ci/rmm/pull-request/1512/${commit}/${artifact_name}") -artifact_name=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_REPOSITORY=raft PYTHON_VERSION="3.11" rapids-package-name wheel_python) +artifact_name=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_REPOSITORY=raft RAPIDS_PY_VERSION="3.11" rapids-package-name wheel_python) commit=$(git ls-remote https://github.com/rapidsai/raft.git refs/heads/pull-request/2251 | cut -c1-7) libraft_wheelhouse=$(rapids-get-artifact "ci/raft/pull-request/2251/${commit}/${artifact_name}") From 8ca6820d332b5ec027ba84d86a44bd6010610947 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 1 Apr 2024 23:31:16 +0000 Subject: [PATCH 13/63] Use known raft commit --- ci/build_wheel_cpp.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index da8f076c4a..fd59c3b2b2 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -19,7 +19,8 @@ commit=$(git ls-remote https://github.com/rapidsai/rmm.git refs/heads/pull-reque librmm_wheelhouse=$(rapids-get-artifact "ci/rmm/pull-request/1512/${commit}/${artifact_name}") artifact_name=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_REPOSITORY=raft RAPIDS_PY_VERSION="3.11" rapids-package-name wheel_python) -commit=$(git ls-remote https://github.com/rapidsai/raft.git refs/heads/pull-request/2251 | cut -c1-7) +#commit=$(git ls-remote https://github.com/rapidsai/raft.git refs/heads/pull-request/2251 | cut -c1-7) +commit="a107fa5" libraft_wheelhouse=$(rapids-get-artifact "ci/raft/pull-request/2251/${commit}/${artifact_name}") # This is the version of the suffix with a preceding hyphen. It's used From ddf474b29b503ab6b87b5044cd63e6fe98ce8ed5 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 1 Apr 2024 23:39:11 +0000 Subject: [PATCH 14/63] Fix variable name --- ci/build_wheel_cpp.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index fd59c3b2b2..abe7cc35c9 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -28,7 +28,7 @@ libraft_wheelhouse=$(rapids-get-artifact "ci/raft/pull-request/2251/${commit}/${ PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" pyproject_file="${package_dir}/pyproject.toml" -version_file="${package_dir}/${version_package_name}/_version.py" +version_file="${package_dir}/${package_name}/_version.py" sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} echo "${version}" > VERSION From eaafcc638a92d3a00fb1f5efd90b26832fe7c930 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 1 Apr 2024 23:39:56 +0000 Subject: [PATCH 15/63] Turn off pure setting --- ci/build_wheel_cpp.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index abe7cc35c9..d330629e50 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -51,4 +51,4 @@ cd "${package_dir}" PIP_FIND_LINKS="${librmm_wheelhouse};${libraft_wheelhouse}" python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check -RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-upload-wheels-to-s3 dist +RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 dist From 1e274c953faa70c07e7084f5ca197e768f1cc341 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 1 Apr 2024 23:48:34 +0000 Subject: [PATCH 16/63] Use space rather than semicolon-separated find links --- ci/build_wheel_cpp.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index d330629e50..410d6f6fa6 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -49,6 +49,6 @@ done cd "${package_dir}" -PIP_FIND_LINKS="${librmm_wheelhouse};${libraft_wheelhouse}" python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check +PIP_FIND_LINKS="${librmm_wheelhouse} ${libraft_wheelhouse}" python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 dist From a0ba13b285bccd9a7de045d76e8665676e66bc72 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 2 Apr 2024 00:58:43 +0000 Subject: [PATCH 17/63] Audit the C++ wheel --- ci/build_wheel_cpp.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index 410d6f6fa6..90e1510774 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -51,4 +51,10 @@ cd "${package_dir}" PIP_FIND_LINKS="${librmm_wheelhouse} ${libraft_wheelhouse}" python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check -RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 dist +# Don't repair raft into this wheel. Everything else is fair game. +# TODO: Check if this works transitively, i.e. if raft links to something that +# it bundles in then will cugraph try to pull that into its own libs? If so, +# we'll need to programatically exclude raft's dependencies from cugraph's. +mkdir -p final_dist +python -m auditwheel repair --exclude libraft.so -w final_dist dist/* +RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist From 14ed178fd105bf79751f82636b34b2d5d6a49032 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 2 Apr 2024 01:04:49 +0000 Subject: [PATCH 18/63] Remove legacy code --- python/cugraph/CMakeLists.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/python/cugraph/CMakeLists.txt b/python/cugraph/CMakeLists.txt index 8254da60ec..63942a845b 100644 --- a/python/cugraph/CMakeLists.txt +++ b/python/cugraph/CMakeLists.txt @@ -36,10 +36,6 @@ endif() find_package(cugraph "${RAPIDS_VERSION}" REQUIRED) -# TODO: Wait until this is implemented in rapids-cmake. At that point also -# should have the llinking automated as part of rapids_cython_create_modules. -rapids_cython_register_site_packages_library(cugraph::cugraph) - include(rapids-cython-core) rapids_cython_init() From 1d404827ac1a76d682aa4483ea4b3a8c0add3d7d Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 2 Apr 2024 01:09:24 +0000 Subject: [PATCH 19/63] Pull all artifacts needed to build wheels --- ci/build_wheel.sh | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 587c5fb38e..c9d2830b40 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -14,6 +14,23 @@ version=$(rapids-generate-version) git_commit=$(git rev-parse HEAD) RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_VERSION="3.11" rapids-download-wheels-from-s3 /tmp/libcugraph_dist + +artifact_name=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_REPOSITORY=rmm RAPIDS_PY_VERSION="3.11" rapids-package-name wheel_python) +commit=$(git ls-remote https://github.com/rapidsai/rmm.git refs/heads/pull-request/1512 | cut -c1-7) +librmm_wheelhouse=$(rapids-get-artifact "ci/rmm/pull-request/1512/${commit}/${artifact_name}") + +artifact_name=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_REPOSITORY=raft RAPIDS_PY_VERSION="3.11" rapids-package-name wheel_python) +#commit=$(git ls-remote https://github.com/rapidsai/raft.git refs/heads/pull-request/2251 | cut -c1-7) +commit="a107fa5" +libraft_wheelhouse=$(rapids-get-artifact "ci/raft/pull-request/2251/${commit}/${artifact_name}") + +artifact_name=$(RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_REPOSITORY=raft rapids-package-name wheel_python) +# I have an older raft commit with working cpp, but still waiting on Python so +# just pull the head and hope it's ready. +commit=$(git ls-remote https://github.com/rapidsai/raft.git refs/heads/pull-request/2251 | cut -c1-7) +#commit="a107fa5" +pylibraft_wheelhouse=$(rapids-get-artifact "ci/raft/pull-request/2251/${commit}/${artifact_name}") # This is the version of the suffix with a preceding hyphen. It's used # everywhere except in the final wheel name. From 84ad199aa01d4f9f46a1340c002673bfb69b44a1 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 2 Apr 2024 01:35:26 +0000 Subject: [PATCH 20/63] Make wheel py3 --- python/libcugraph/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/python/libcugraph/pyproject.toml b/python/libcugraph/pyproject.toml index beb676c8e7..f18bf0cd0b 100644 --- a/python/libcugraph/pyproject.toml +++ b/python/libcugraph/pyproject.toml @@ -45,6 +45,7 @@ ninja.make-fallback = true sdist.reproducible = true wheel.packages = ["libcugraph"] wheel.install-dir = "libcugraph" +wheel.py-api = "py3" [tool.scikit-build.metadata.version] provider = "scikit_build_core.metadata.regex" From 618ad3cd114379cf82b6c66c1d165ff617960a55 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 2 Apr 2024 19:01:16 +0000 Subject: [PATCH 21/63] Clean up commit pulls --- ci/build_wheel.sh | 6 +----- ci/build_wheel_cpp.sh | 3 +-- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index c9d2830b40..f9f700e5dc 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -21,15 +21,11 @@ commit=$(git ls-remote https://github.com/rapidsai/rmm.git refs/heads/pull-reque librmm_wheelhouse=$(rapids-get-artifact "ci/rmm/pull-request/1512/${commit}/${artifact_name}") artifact_name=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_REPOSITORY=raft RAPIDS_PY_VERSION="3.11" rapids-package-name wheel_python) -#commit=$(git ls-remote https://github.com/rapidsai/raft.git refs/heads/pull-request/2251 | cut -c1-7) -commit="a107fa5" +commit=$(git ls-remote https://github.com/rapidsai/raft.git refs/heads/pull-request/2251 | cut -c1-7) libraft_wheelhouse=$(rapids-get-artifact "ci/raft/pull-request/2251/${commit}/${artifact_name}") artifact_name=$(RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_REPOSITORY=raft rapids-package-name wheel_python) -# I have an older raft commit with working cpp, but still waiting on Python so -# just pull the head and hope it's ready. commit=$(git ls-remote https://github.com/rapidsai/raft.git refs/heads/pull-request/2251 | cut -c1-7) -#commit="a107fa5" pylibraft_wheelhouse=$(rapids-get-artifact "ci/raft/pull-request/2251/${commit}/${artifact_name}") # This is the version of the suffix with a preceding hyphen. It's used diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index 90e1510774..b8613b0abf 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -19,8 +19,7 @@ commit=$(git ls-remote https://github.com/rapidsai/rmm.git refs/heads/pull-reque librmm_wheelhouse=$(rapids-get-artifact "ci/rmm/pull-request/1512/${commit}/${artifact_name}") artifact_name=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_REPOSITORY=raft RAPIDS_PY_VERSION="3.11" rapids-package-name wheel_python) -#commit=$(git ls-remote https://github.com/rapidsai/raft.git refs/heads/pull-request/2251 | cut -c1-7) -commit="a107fa5" +commit=$(git ls-remote https://github.com/rapidsai/raft.git refs/heads/pull-request/2251 | cut -c1-7) libraft_wheelhouse=$(rapids-get-artifact "ci/raft/pull-request/2251/${commit}/${artifact_name}") # This is the version of the suffix with a preceding hyphen. It's used From 3fd2377a23591af08d07d81d3c47890551e52c78 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 2 Apr 2024 19:01:31 +0000 Subject: [PATCH 22/63] Make sure to update the libcugraph suffix --- ci/build_wheel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index f9f700e5dc..8a1d9675e6 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -53,7 +53,7 @@ if ! rapids-is-release-build; then alpha_spec=',>=0.0.0a0' fi -for dep in rmm cudf cugraph raft-dask pylibcugraph pylibcugraphops pylibraft ucx-py; do +for dep in rmm cudf cugraph libcugraph raft-dask pylibcugraph pylibcugraphops pylibraft ucx-py; do sed -r -i "s/${dep}==(.*)\"/${dep}${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} done From 587d6f4b96dbe1eb9651f12859644a6c0b5eeb9b Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 2 Apr 2024 19:02:00 +0000 Subject: [PATCH 23/63] Download all requirements in test job --- ci/test_wheel.sh | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index cda40d92c7..6f910248c5 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -21,9 +21,28 @@ if [[ "${package_name}" == "nx-cugraph" ]]; then else RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist fi + +artifact_name=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_REPOSITORY=rmm RAPIDS_PY_VERSION="3.11" rapids-package-name wheel_python) +commit=$(git ls-remote https://github.com/rapidsai/rmm.git refs/heads/pull-request/1512 | cut -c1-7) +librmm_wheelhouse=$(rapids-get-artifact "ci/rmm/pull-request/1512/${commit}/${artifact_name}") + +artifact_name=$(RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_REPOSITORY=rmm rapids-package-name wheel_python) +commit=$(git ls-remote https://github.com/rapidsai/rmm.git refs/heads/pull-request/1512 | cut -c1-7) +rmm_wheelhouse=$(rapids-get-artifact "ci/rmm/pull-request/1512/${commit}/${artifact_name}") + +artifact_name=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_REPOSITORY=raft RAPIDS_PY_VERSION="3.11" rapids-package-name wheel_python) +commit=$(git ls-remote https://github.com/rapidsai/raft.git refs/heads/pull-request/2251 | cut -c1-7) +libraft_wheelhouse=$(rapids-get-artifact "ci/raft/pull-request/2251/${commit}/${artifact_name}") + +artifact_name=$(RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_REPOSITORY=raft rapids-package-name wheel_python) +commit=$(git ls-remote https://github.com/rapidsai/raft.git refs/heads/pull-request/2251 | cut -c1-7) +pylibraft_wheelhouse=$(rapids-get-artifact "ci/raft/pull-request/2251/${commit}/${artifact_name}") + +RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_VERSION="3.11" rapids-download-wheels-from-s3 /tmp/libcugraph_dist + # use 'ls' to expand wildcard before adding `[extra]` requires for pip # pip creates wheels using python package names -python -m pip install $(ls ./dist/${python_package_name}*.whl)[test] +python -m pip install $(ls ./dist/${python_package_name}*.whl)[test] --find-links /tmp/libcugraph_dist --find-links ${librmm_wheelhouse} --find-links ${rmm_wheelhouse} --find-links ${libraft_wheelhouse} --find-links ${pylibraft_wheelhouse} # Run smoke tests for aarch64 pull requests arch=$(uname -m) From 47ef4a3d082eda60ee279d1442abe7550d1c4a7a Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 2 Apr 2024 20:55:13 +0000 Subject: [PATCH 24/63] Add PIP_FIND_LINKS --- ci/build_wheel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 8a1d9675e6..7d93899471 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -69,7 +69,7 @@ fi cd "${package_dir}" -python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check +PIP_FIND_LINKS="/tmp/libcugraph_dist ${librmm_wheelhouse} ${libraft_wheelhouse} ${pylibraft_wheelhouse}" python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check # pure-python packages should be marked as pure, and not have auditwheel run on them. if [[ ${package_name} == "nx-cugraph" ]] || \ From 11db8d58a4c2b8f83236c7892265525630764f97 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 3 Apr 2024 00:28:11 +0000 Subject: [PATCH 25/63] Update to use latest tools --- ci/build_wheel.sh | 25 +++++++++++-------------- ci/build_wheel_cpp.sh | 16 ++++++++-------- ci/test_wheel.sh | 30 ++++++++++++------------------ 3 files changed, 31 insertions(+), 40 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 7d93899471..b89b6c70e3 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -7,6 +7,11 @@ package_name=$1 package_dir=$2 underscore_package_name=$(echo "${package_name}" | tr "-" "_") +if [[ ! -d "/tmp/gha-tools" ]]; then + git clone https://github.com/msarahan/gha-tools.git -b get-pr-wheel-artifact /tmp/gha-tools +fi +export PATH="/tmp/gha-tools/tools:${PATH}" + source rapids-configure-sccache source rapids-date-string @@ -14,19 +19,11 @@ version=$(rapids-generate-version) git_commit=$(git rev-parse HEAD) RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_VERSION="3.11" rapids-download-wheels-from-s3 /tmp/libcugraph_dist - -artifact_name=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_REPOSITORY=rmm RAPIDS_PY_VERSION="3.11" rapids-package-name wheel_python) -commit=$(git ls-remote https://github.com/rapidsai/rmm.git refs/heads/pull-request/1512 | cut -c1-7) -librmm_wheelhouse=$(rapids-get-artifact "ci/rmm/pull-request/1512/${commit}/${artifact_name}") - -artifact_name=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_REPOSITORY=raft RAPIDS_PY_VERSION="3.11" rapids-package-name wheel_python) -commit=$(git ls-remote https://github.com/rapidsai/raft.git refs/heads/pull-request/2251 | cut -c1-7) -libraft_wheelhouse=$(rapids-get-artifact "ci/raft/pull-request/2251/${commit}/${artifact_name}") +RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcugraph_dist -artifact_name=$(RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_REPOSITORY=raft rapids-package-name wheel_python) -commit=$(git ls-remote https://github.com/rapidsai/raft.git refs/heads/pull-request/2251 | cut -c1-7) -pylibraft_wheelhouse=$(rapids-get-artifact "ci/raft/pull-request/2251/${commit}/${artifact_name}") +librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1512 cpp) +libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2251 cpp) +libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2251 python) # This is the version of the suffix with a preceding hyphen. It's used # everywhere except in the final wheel name. @@ -76,9 +73,9 @@ if [[ ${package_name} == "nx-cugraph" ]] || \ [[ ${package_name} == "cugraph-dgl" ]] || \ [[ ${package_name} == "cugraph-pyg" ]] || \ [[ ${package_name} == "cugraph-equivariant" ]]; then - RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-upload-wheels-to-s3 dist + RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-upload-wheels-to-s3 python dist else mkdir -p final_dist python -m auditwheel repair -w final_dist dist/* - RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist + RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python final_dist fi diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index b8613b0abf..5fe7436309 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -6,6 +6,11 @@ set -euo pipefail package_name="libcugraph" package_dir="python/libcugraph" +if [[ ! -d "/tmp/gha-tools" ]]; then + git clone https://github.com/msarahan/gha-tools.git -b get-pr-wheel-artifact /tmp/gha-tools +fi +export PATH="/tmp/gha-tools/tools:${PATH}" + source rapids-configure-sccache source rapids-date-string @@ -14,13 +19,8 @@ git_commit=$(git rev-parse HEAD) RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -artifact_name=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_REPOSITORY=rmm RAPIDS_PY_VERSION="3.11" rapids-package-name wheel_python) -commit=$(git ls-remote https://github.com/rapidsai/rmm.git refs/heads/pull-request/1512 | cut -c1-7) -librmm_wheelhouse=$(rapids-get-artifact "ci/rmm/pull-request/1512/${commit}/${artifact_name}") - -artifact_name=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_REPOSITORY=raft RAPIDS_PY_VERSION="3.11" rapids-package-name wheel_python) -commit=$(git ls-remote https://github.com/rapidsai/raft.git refs/heads/pull-request/2251 | cut -c1-7) -libraft_wheelhouse=$(rapids-get-artifact "ci/raft/pull-request/2251/${commit}/${artifact_name}") +librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1512 cpp) +libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2251 cpp) # This is the version of the suffix with a preceding hyphen. It's used # everywhere except in the final wheel name. @@ -56,4 +56,4 @@ PIP_FIND_LINKS="${librmm_wheelhouse} ${libraft_wheelhouse}" python -m pip wheel # we'll need to programatically exclude raft's dependencies from cugraph's. mkdir -p final_dist python -m auditwheel repair --exclude libraft.so -w final_dist dist/* -RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist +RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp final_dist diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index 6f910248c5..880ad31d27 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -3,6 +3,11 @@ set -eoxu pipefail +if [[ ! -d "/tmp/gha-tools" ]]; then + git clone https://github.com/msarahan/gha-tools.git -b get-pr-wheel-artifact /tmp/gha-tools +fi +export PATH="/tmp/gha-tools/tools:${PATH}" + # TODO: Enable dask query planning (by default) once some bugs are fixed. # xref: https://github.com/rapidsai/cudf/issues/15027 export DASK_DATAFRAME__QUERY_PLANNING=False @@ -17,28 +22,17 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" # nx-cugraph is a pure wheel, which is part of generating the download path if [[ "${package_name}" == "nx-cugraph" ]]; then - RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist + RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 python ./dist else - RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist + RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist fi -artifact_name=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_REPOSITORY=rmm RAPIDS_PY_VERSION="3.11" rapids-package-name wheel_python) -commit=$(git ls-remote https://github.com/rapidsai/rmm.git refs/heads/pull-request/1512 | cut -c1-7) -librmm_wheelhouse=$(rapids-get-artifact "ci/rmm/pull-request/1512/${commit}/${artifact_name}") - -artifact_name=$(RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_REPOSITORY=rmm rapids-package-name wheel_python) -commit=$(git ls-remote https://github.com/rapidsai/rmm.git refs/heads/pull-request/1512 | cut -c1-7) -rmm_wheelhouse=$(rapids-get-artifact "ci/rmm/pull-request/1512/${commit}/${artifact_name}") - -artifact_name=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_REPOSITORY=raft RAPIDS_PY_VERSION="3.11" rapids-package-name wheel_python) -commit=$(git ls-remote https://github.com/rapidsai/raft.git refs/heads/pull-request/2251 | cut -c1-7) -libraft_wheelhouse=$(rapids-get-artifact "ci/raft/pull-request/2251/${commit}/${artifact_name}") - -artifact_name=$(RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_REPOSITORY=raft rapids-package-name wheel_python) -commit=$(git ls-remote https://github.com/rapidsai/raft.git refs/heads/pull-request/2251 | cut -c1-7) -pylibraft_wheelhouse=$(rapids-get-artifact "ci/raft/pull-request/2251/${commit}/${artifact_name}") +librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1512 cpp) +rmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1512 python) +libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2251 cpp) +libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2251 python) -RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_VERSION="3.11" rapids-download-wheels-from-s3 /tmp/libcugraph_dist +RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcugraph_dist # use 'ls' to expand wildcard before adding `[extra]` requires for pip # pip creates wheels using python package names From b6bdb3875708f35765ceeb8c6843333ff0381c5d Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 3 Apr 2024 05:09:57 +0000 Subject: [PATCH 26/63] Fix typo --- ci/build_wheel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index b89b6c70e3..9bc4839551 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -23,7 +23,7 @@ RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheel librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1512 cpp) libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2251 cpp) -libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2251 python) +pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2251 python) # This is the version of the suffix with a preceding hyphen. It's used # everywhere except in the final wheel name. From e4c026f9be8767dbb61b44200c9d97b759b67fd7 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 3 Apr 2024 05:11:20 +0000 Subject: [PATCH 27/63] Add excludes to auditwheel --- ci/build_wheel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 9bc4839551..cb8e9c90f1 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -76,6 +76,6 @@ if [[ ${package_name} == "nx-cugraph" ]] || \ RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-upload-wheels-to-s3 python dist else mkdir -p final_dist - python -m auditwheel repair -w final_dist dist/* + python -m auditwheel repair -w final_dist --exclude libcugraph.so --exclude libcugraph_c.so --exclude libraft.so dist/* RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python final_dist fi From 0f7cbb1e09f24e67c67e5de5b9d2aa24a843fd77 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 3 Apr 2024 05:27:12 +0000 Subject: [PATCH 28/63] Fix one more typo --- ci/test_wheel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index 880ad31d27..e62eb46ace 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -30,7 +30,7 @@ fi librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1512 cpp) rmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1512 python) libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2251 cpp) -libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2251 python) +pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2251 python) RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcugraph_dist From 175c21be76db173e916ec51b54e56e23cec57563 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 4 Apr 2024 01:52:58 +0000 Subject: [PATCH 29/63] Reenable cugraph-ops --- python/libcugraph/CMakeLists.txt | 7 ++++--- python/pylibcugraph/CMakeLists.txt | 3 +-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/python/libcugraph/CMakeLists.txt b/python/libcugraph/CMakeLists.txt index ae08234059..99eb5a1868 100644 --- a/python/libcugraph/CMakeLists.txt +++ b/python/libcugraph/CMakeLists.txt @@ -25,9 +25,7 @@ project( LANGUAGES CXX CUDA ) -# TODO: Temporarily not using cugraph-ops for wheels while testing the new -# libcugraph wheel approach -option(USE_CUGRAPH_OPS "Enable all functions that call cugraph-ops" OFF) +option(USE_CUGRAPH_OPS "Enable all functions that call cugraph-ops" ON) if(NOT USE_CUGRAPH_OPS) message(STATUS "Disabling libcugraph functions that reference cugraph-ops") @@ -38,5 +36,8 @@ set(BUILD_TESTS OFF) set(BUILD_CUGRAPH_MG_TESTS OFF) set(BUILD_CUGRAPH_OPS_CPP_TESTS OFF) set(CUDA_STATIC_RUNTIME ON) +set(CUGRAPH_USE_CUGRAPH_OPS_STATIC ON) +set(CUGRAPH_EXCLUDE_CUGRAPH_OPS_FROM_ALL ON) +set(ALLOW_CLONE_CUGRAPH_OPS ON) add_subdirectory(../../cpp cugraph-cpp) diff --git a/python/pylibcugraph/CMakeLists.txt b/python/pylibcugraph/CMakeLists.txt index 465a2fd8c7..3e7067d4e0 100644 --- a/python/pylibcugraph/CMakeLists.txt +++ b/python/pylibcugraph/CMakeLists.txt @@ -27,8 +27,7 @@ project( LANGUAGES CXX CUDA ) -# TODO: Temporarily not using cugraph-ops for wheels -option(USE_CUGRAPH_OPS "Enable all functions that call cugraph-ops" OFF) +option(USE_CUGRAPH_OPS "Enable all functions that call cugraph-ops" ON) if(NOT USE_CUGRAPH_OPS) message(STATUS "Disabling libcugraph functions that reference cugraph-ops") From f49c8b689a0f9ab01203ecaaf2a7981fbd7c4dee Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 4 Apr 2024 02:10:34 +0000 Subject: [PATCH 30/63] Point CMake to the cugraph-ops clone --- ci/build_wheel_cpp.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index 5fe7436309..0cb531944b 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -48,7 +48,7 @@ done cd "${package_dir}" -PIP_FIND_LINKS="${librmm_wheelhouse} ${libraft_wheelhouse}" python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check +SKBUILD_CMAKE_ARGS="-DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/cugraph-ops/" PIP_FIND_LINKS="${librmm_wheelhouse} ${libraft_wheelhouse}" python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check # Don't repair raft into this wheel. Everything else is fair game. # TODO: Check if this works transitively, i.e. if raft links to something that From 57a0d43909ead285d588058f29ba946fee899db4 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 4 Apr 2024 16:28:45 +0000 Subject: [PATCH 31/63] Use libcugraphops wheel --- dependencies.yaml | 1 + python/libcugraph/CMakeLists.txt | 4 ---- python/libcugraph/libcugraph/load.py | 6 ++++-- python/libcugraph/pyproject.toml | 2 ++ 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index cb9dc753d3..829cca966f 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -477,6 +477,7 @@ dependencies: packages: - librmm==24.6.* - libraft==24.6.* + - libcugraphops==24.6.* python_run_cugraph: common: - output_types: [conda, pyproject] diff --git a/python/libcugraph/CMakeLists.txt b/python/libcugraph/CMakeLists.txt index 99eb5a1868..429df34ff5 100644 --- a/python/libcugraph/CMakeLists.txt +++ b/python/libcugraph/CMakeLists.txt @@ -34,10 +34,6 @@ endif() set(BUILD_TESTS OFF) set(BUILD_CUGRAPH_MG_TESTS OFF) -set(BUILD_CUGRAPH_OPS_CPP_TESTS OFF) set(CUDA_STATIC_RUNTIME ON) -set(CUGRAPH_USE_CUGRAPH_OPS_STATIC ON) -set(CUGRAPH_EXCLUDE_CUGRAPH_OPS_FROM_ALL ON) -set(ALLOW_CLONE_CUGRAPH_OPS ON) add_subdirectory(../../cpp cugraph-cpp) diff --git a/python/libcugraph/libcugraph/load.py b/python/libcugraph/libcugraph/load.py index ad4cccbbf9..192ca1290d 100644 --- a/python/libcugraph/libcugraph/load.py +++ b/python/libcugraph/libcugraph/load.py @@ -17,12 +17,14 @@ import os import libraft +import libcugraphops def load_library(): - # libraft must be loaded before libcugraph since libcugraph references symbols in - # libraft. + # libraft and libcugraphops must be loaded before libcugraph since libcugraph + # references their symbols. libraft.load_library() + libcugraphops.load_library() # Dynamically load libcugraph.so. Prefer a system library if one is present to # avoid clobbering symbols that other packages might expect, but if no diff --git a/python/libcugraph/pyproject.toml b/python/libcugraph/pyproject.toml index f18bf0cd0b..3aa048f0b6 100644 --- a/python/libcugraph/pyproject.toml +++ b/python/libcugraph/pyproject.toml @@ -3,6 +3,7 @@ [build-system] requires = [ "cmake>=3.26.4", + "libcugraphops==24.6.*", "libraft==24.6.*", "librmm==24.6.*", "ninja", @@ -21,6 +22,7 @@ authors = [ license = { text = "Apache 2.0" } requires-python = ">=3.9" dependencies = [ + "libcugraphops==24.6.*", "libraft==24.6.*", "librmm==24.6.*", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From eed9f857a7865acd4402019cefaebdc0f629f362 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 4 Apr 2024 16:29:42 +0000 Subject: [PATCH 32/63] Note that we don't need an extra repo anymore --- .github/workflows/pr.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 7f945cf401..805a52a876 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -102,6 +102,8 @@ jobs: matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) build_type: pull-request script: ci/build_wheel_cpp.sh + # TODO: We should be able to remove this now (here and in build.yaml) + # because we're using a pre-built wheel and don't need the source. extra-repo: rapidsai/cugraph-ops extra-repo-sha: branch-24.06 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY From cced123b35b83e7e15b18c7aee42f64f684252fc Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 4 Apr 2024 16:33:00 +0000 Subject: [PATCH 33/63] Download libcugraphops wheel --- ci/build_wheel.sh | 3 ++- ci/build_wheel_cpp.sh | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index cb8e9c90f1..e54f338851 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -23,6 +23,7 @@ RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheel librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1512 cpp) libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2251 cpp) +libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp) pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2251 python) # This is the version of the suffix with a preceding hyphen. It's used @@ -66,7 +67,7 @@ fi cd "${package_dir}" -PIP_FIND_LINKS="/tmp/libcugraph_dist ${librmm_wheelhouse} ${libraft_wheelhouse} ${pylibraft_wheelhouse}" python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check +PIP_FIND_LINKS="/tmp/libcugraph_dist ${librmm_wheelhouse} ${libraft_wheelhouse} ${libcugraphops_wheelhouse} ${pylibraft_wheelhouse}" python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check # pure-python packages should be marked as pure, and not have auditwheel run on them. if [[ ${package_name} == "nx-cugraph" ]] || \ diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index 0cb531944b..c6974227be 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -21,6 +21,7 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1512 cpp) libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2251 cpp) +libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp) # This is the version of the suffix with a preceding hyphen. It's used # everywhere except in the final wheel name. @@ -42,13 +43,13 @@ if ! rapids-is-release-build; then alpha_spec=',>=0.0.0a0' fi -for dep in librmm libraft; do +for dep in librmm libraft libcugraphops; do sed -r -i "s/${dep}==(.*)\"/${dep}${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} done cd "${package_dir}" -SKBUILD_CMAKE_ARGS="-DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/cugraph-ops/" PIP_FIND_LINKS="${librmm_wheelhouse} ${libraft_wheelhouse}" python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check +PIP_FIND_LINKS="${librmm_wheelhouse} ${libraft_wheelhouse} ${libcugraphops_wheelhouse}" python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check # Don't repair raft into this wheel. Everything else is fair game. # TODO: Check if this works transitively, i.e. if raft links to something that From 8c650ffaf3b09d529faee5703a9af17edd690a13 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 4 Apr 2024 16:34:34 +0000 Subject: [PATCH 34/63] Stop cloning extra repo --- .github/workflows/build.yaml | 6 ------ .github/workflows/pr.yaml | 11 ----------- ci/release/update-version.sh | 2 -- 3 files changed, 19 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index f50dd64681..b15528636c 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -76,9 +76,6 @@ jobs: sha: ${{ inputs.sha }} date: ${{ inputs.date }} script: ci/build_wheel_pylibcugraph.sh - extra-repo: rapidsai/cugraph-ops - extra-repo-sha: branch-24.06 - extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY node_type: cpu32 wheel-publish-pylibcugraph: needs: wheel-build-pylibcugraph @@ -100,9 +97,6 @@ jobs: sha: ${{ inputs.sha }} date: ${{ inputs.date }} script: ci/build_wheel_cugraph.sh - extra-repo: rapidsai/cugraph-ops - extra-repo-sha: branch-24.06 - extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY wheel-publish-cugraph: needs: wheel-build-cugraph secrets: inherit diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 805a52a876..06a7b27c17 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -102,11 +102,6 @@ jobs: matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) build_type: pull-request script: ci/build_wheel_cpp.sh - # TODO: We should be able to remove this now (here and in build.yaml) - # because we're using a pre-built wheel and don't need the source. - extra-repo: rapidsai/cugraph-ops - extra-repo-sha: branch-24.06 - extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY node_type: cpu32 wheel-build-pylibcugraph: needs: wheel-build-cpp @@ -115,9 +110,6 @@ jobs: with: build_type: pull-request script: ci/build_wheel_pylibcugraph.sh - extra-repo: rapidsai/cugraph-ops - extra-repo-sha: branch-24.06 - extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY node_type: cpu32 wheel-tests-pylibcugraph: needs: wheel-build-pylibcugraph @@ -133,9 +125,6 @@ jobs: with: build_type: pull-request script: ci/build_wheel_cugraph.sh - extra-repo: rapidsai/cugraph-ops - extra-repo-sha: branch-24.06 - extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY wheel-tests-cugraph: needs: wheel-build-cugraph secrets: inherit diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index ce2c14a01f..c9d61d3f33 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -91,8 +91,6 @@ sed_runner "/^ucx_py_version:$/ {n;s/.*/ - \"${NEXT_UCX_PY_VERSION}.*\"/}" cond # CI files for FILE in .github/workflows/*.yaml; do sed_runner "/shared-workflows/ s/@.*/@branch-${NEXT_SHORT_TAG}/g" "${FILE}" - # Wheel builds clone cugraph-ops, update its branch - sed_runner "s/extra-repo-sha: branch-.*/extra-repo-sha: branch-${NEXT_SHORT_TAG}/g" "${FILE}" # Wheel builds install dask-cuda from source, update its branch sed_runner "s/dask-cuda.git@branch-[0-9][0-9].[0-9][0-9]/dask-cuda.git@branch-${NEXT_SHORT_TAG}/g" "${FILE}" done From 8c8c7c4804be11c7a9d1687712fa03e7f7962344 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 4 Apr 2024 16:59:05 +0000 Subject: [PATCH 35/63] Specify the commit so that we don't need to ls the remote --- ci/build_wheel.sh | 2 +- ci/build_wheel_cpp.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index e54f338851..52367b1012 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -23,7 +23,7 @@ RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheel librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1512 cpp) libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2251 cpp) -libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp) +libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp a3ac1f3) pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2251 python) # This is the version of the suffix with a preceding hyphen. It's used diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index c6974227be..cb10548e79 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -21,7 +21,7 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1512 cpp) libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2251 cpp) -libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp) +libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp a3ac1f3) # This is the version of the suffix with a preceding hyphen. It's used # everywhere except in the final wheel name. From ec4fb459417ddb6866ff9a505b2819a6690e881a Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 4 Apr 2024 17:55:40 +0000 Subject: [PATCH 36/63] Also exclude libcugraph-ops from auditwheel --- ci/build_wheel.sh | 2 +- ci/build_wheel_cpp.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 52367b1012..8fa504c22d 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -77,6 +77,6 @@ if [[ ${package_name} == "nx-cugraph" ]] || \ RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-upload-wheels-to-s3 python dist else mkdir -p final_dist - python -m auditwheel repair -w final_dist --exclude libcugraph.so --exclude libcugraph_c.so --exclude libraft.so dist/* + python -m auditwheel repair -w final_dist --exclude libcugraph.so --exclude libcugraph_c.so --exclude libraft.so --exclude libcugraph-ops++.so dist/* RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python final_dist fi diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index cb10548e79..b79043d08b 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -56,5 +56,5 @@ PIP_FIND_LINKS="${librmm_wheelhouse} ${libraft_wheelhouse} ${libcugraphops_wheel # it bundles in then will cugraph try to pull that into its own libs? If so, # we'll need to programatically exclude raft's dependencies from cugraph's. mkdir -p final_dist -python -m auditwheel repair --exclude libraft.so -w final_dist dist/* +python -m auditwheel repair --exclude libraft.so --exclude libcugraph-ops++.so -w final_dist dist/* RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp final_dist From d044445a2b32e47c0bdd7dc5d8017f662eff98ea Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 4 Apr 2024 19:02:57 +0000 Subject: [PATCH 37/63] Also download libcugraphops when running tests --- ci/test_wheel.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index e62eb46ace..5cd08f2adf 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -31,12 +31,13 @@ librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapid rmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1512 python) libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2251 cpp) pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2251 python) +libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp a3ac1f3) RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcugraph_dist # use 'ls' to expand wildcard before adding `[extra]` requires for pip # pip creates wheels using python package names -python -m pip install $(ls ./dist/${python_package_name}*.whl)[test] --find-links /tmp/libcugraph_dist --find-links ${librmm_wheelhouse} --find-links ${rmm_wheelhouse} --find-links ${libraft_wheelhouse} --find-links ${pylibraft_wheelhouse} +python -m pip install $(ls ./dist/${python_package_name}*.whl)[test] --find-links /tmp/libcugraph_dist --find-links ${librmm_wheelhouse} --find-links ${rmm_wheelhouse} --find-links ${libraft_wheelhouse} --find-links ${pylibraft_wheelhouse} --find-links ${libcugraphops_wheelhouse} # Run smoke tests for aarch64 pull requests arch=$(uname -m) From a80132f4ceffc9764e9b7db3448f8621645baf61 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 9 Apr 2024 23:37:07 +0000 Subject: [PATCH 38/63] Use new rapids-cmake to hide necessary CCCL symbols --- rapids_config.cmake | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rapids_config.cmake b/rapids_config.cmake index 50b1054b7b..6ebdd3b47f 100644 --- a/rapids_config.cmake +++ b/rapids_config.cmake @@ -25,6 +25,8 @@ else() "Could not determine RAPIDS version. Contents of VERSION file:\n${_rapids_version_formatted}") endif() +set(rapids-cmake-repo vyasr/rapids-cmake) +set(rapids-cmake-branch "fix/cccl_kernel_pointer_hidden") if(NOT EXISTS "${CMAKE_CURRENT_BINARY_DIR}/CUGRAPH_RAPIDS-${RAPIDS_VERSION_MAJOR_MINOR}.cmake") file( DOWNLOAD From d4c3742e39194fc56c60cf2e426421be2b69ee3f Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 12 Apr 2024 00:25:16 +0000 Subject: [PATCH 39/63] Some cleanup --- cpp/cmake/thirdparty/get_raft.cmake | 2 +- python/cugraph/cugraph/__init__.py | 12 ++++++++++++ rapids_config.cmake | 2 -- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/cpp/cmake/thirdparty/get_raft.cmake b/cpp/cmake/thirdparty/get_raft.cmake index 6274005733..8f56372c81 100644 --- a/cpp/cmake/thirdparty/get_raft.cmake +++ b/cpp/cmake/thirdparty/get_raft.cmake @@ -39,7 +39,7 @@ function(find_and_configure_raft) endif() rapids_cpm_find(raft ${PKG_VERSION} - GLOBAL_TARGETS raft::raft raft::raft_lib + GLOBAL_TARGETS raft::raft BUILD_EXPORT_SET cugraph-exports INSTALL_EXPORT_SET cugraph-exports COMPONENTS ${RAFT_COMPONENTS} diff --git a/python/cugraph/cugraph/__init__.py b/python/cugraph/cugraph/__init__.py index ba7e23df80..55c4d55c08 100644 --- a/python/cugraph/cugraph/__init__.py +++ b/python/cugraph/cugraph/__init__.py @@ -11,6 +11,18 @@ # See the License for the specific language governing permissions and # limitations under the License. +# If libcugraph was installed as a wheel, we must request it to load the library +# symbols. Otherwise, we assume that the library was installed in a system path that ld +# can find. +try: + import libcugraph +except ModuleNotFoundError: + pass +else: + libcugraph.load_library() + del libcugraph + + from cugraph.community import ( ecg, induced_subgraph, diff --git a/rapids_config.cmake b/rapids_config.cmake index 6ebdd3b47f..50b1054b7b 100644 --- a/rapids_config.cmake +++ b/rapids_config.cmake @@ -25,8 +25,6 @@ else() "Could not determine RAPIDS version. Contents of VERSION file:\n${_rapids_version_formatted}") endif() -set(rapids-cmake-repo vyasr/rapids-cmake) -set(rapids-cmake-branch "fix/cccl_kernel_pointer_hidden") if(NOT EXISTS "${CMAKE_CURRENT_BINARY_DIR}/CUGRAPH_RAPIDS-${RAPIDS_VERSION_MAJOR_MINOR}.cmake") file( DOWNLOAD From 1a24565dd99ed1926656db648c02f1aa54748b23 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 12 Apr 2024 00:48:17 +0000 Subject: [PATCH 40/63] Try restructuring CI --- .github/workflows/pr.yaml | 96 +++++++++++++++++------------------ ci/build_wheel_python.sh | 102 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 150 insertions(+), 48 deletions(-) create mode 100644 ci/build_wheel_python.sh diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 06a7b27c17..9bfe630f4b 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -21,17 +21,17 @@ jobs: - conda-python-tests - docs-build - wheel-build-cpp - - wheel-build-pylibcugraph + - wheel-build-python - wheel-tests-pylibcugraph - - wheel-build-cugraph + #- wheel-build-cugraph - wheel-tests-cugraph - - wheel-build-nx-cugraph + #- wheel-build-nx-cugraph - wheel-tests-nx-cugraph - - wheel-build-cugraph-dgl + #- wheel-build-cugraph-dgl - wheel-tests-cugraph-dgl - - wheel-build-cugraph-pyg + #- wheel-build-cugraph-pyg - wheel-tests-cugraph-pyg - - wheel-build-cugraph-equivariant + #- wheel-build-cugraph-equivariant - wheel-tests-cugraph-equivariant - devcontainer secrets: inherit @@ -103,87 +103,87 @@ jobs: build_type: pull-request script: ci/build_wheel_cpp.sh node_type: cpu32 - wheel-build-pylibcugraph: + wheel-build-python: needs: wheel-build-cpp secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 with: build_type: pull-request - script: ci/build_wheel_pylibcugraph.sh + script: ci/build_wheel_python.sh node_type: cpu32 wheel-tests-pylibcugraph: - needs: wheel-build-pylibcugraph + needs: wheel-build-python secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.06 with: build_type: pull-request script: ci/test_wheel_pylibcugraph.sh - wheel-build-cugraph: - needs: wheel-build-pylibcugraph - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 - with: - build_type: pull-request - script: ci/build_wheel_cugraph.sh + #wheel-build-cugraph: + # needs: wheel-build-pylibcugraph + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 + # with: + # build_type: pull-request + # script: ci/build_wheel_cugraph.sh wheel-tests-cugraph: - needs: wheel-build-cugraph + needs: wheel-build-python secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.06 with: build_type: pull-request script: ci/test_wheel_cugraph.sh - wheel-build-nx-cugraph: - needs: wheel-tests-pylibcugraph - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 - with: - build_type: pull-request - script: ci/build_wheel_nx-cugraph.sh + #wheel-build-nx-cugraph: + # needs: wheel-tests-pylibcugraph + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 + # with: + # build_type: pull-request + # script: ci/build_wheel_nx-cugraph.sh wheel-tests-nx-cugraph: - needs: wheel-build-nx-cugraph + needs: wheel-build-python secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.06 with: build_type: pull-request script: ci/test_wheel_nx-cugraph.sh - wheel-build-cugraph-dgl: - needs: wheel-tests-cugraph - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 - with: - build_type: pull-request - script: ci/build_wheel_cugraph-dgl.sh + #wheel-build-cugraph-dgl: + # needs: wheel-tests-cugraph + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 + # with: + # build_type: pull-request + # script: ci/build_wheel_cugraph-dgl.sh wheel-tests-cugraph-dgl: - needs: wheel-build-cugraph-dgl + needs: wheel-build-python secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.06 with: build_type: pull-request script: ci/test_wheel_cugraph-dgl.sh matrix_filter: map(select(.ARCH == "amd64")) - wheel-build-cugraph-pyg: - needs: wheel-tests-cugraph - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 - with: - build_type: pull-request - script: ci/build_wheel_cugraph-pyg.sh + #wheel-build-cugraph-pyg: + # needs: wheel-tests-cugraph + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 + # with: + # build_type: pull-request + # script: ci/build_wheel_cugraph-pyg.sh wheel-tests-cugraph-pyg: - needs: wheel-build-cugraph-pyg + needs: wheel-build-python secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.06 with: build_type: pull-request script: ci/test_wheel_cugraph-pyg.sh matrix_filter: map(select(.ARCH == "amd64")) - wheel-build-cugraph-equivariant: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 - with: - build_type: pull-request - script: ci/build_wheel_cugraph-equivariant.sh + #wheel-build-cugraph-equivariant: + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 + # with: + # build_type: pull-request + # script: ci/build_wheel_cugraph-equivariant.sh wheel-tests-cugraph-equivariant: - needs: wheel-build-cugraph-equivariant + needs: wheel-build-python secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.06 with: diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh new file mode 100644 index 0000000000..0e22181190 --- /dev/null +++ b/ci/build_wheel_python.sh @@ -0,0 +1,102 @@ +#!/bin/bash +# Copyright (c) 2023-2024, NVIDIA CORPORATION. + +set -euo pipefail + +if [[ ! -d "/tmp/gha-tools" ]]; then + git clone https://github.com/msarahan/gha-tools.git -b get-pr-wheel-artifact /tmp/gha-tools +fi +export PATH="/tmp/gha-tools/tools:${PATH}" + +source rapids-configure-sccache +source rapids-date-string + +version=$(rapids-generate-version) +git_commit=$(git rev-parse HEAD) + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcugraph_dist + +librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) +libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp) +libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp a3ac1f3) +pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 python) + +wheelhouses=("${librmm_wheelhouse}" "${libraft_wheelhouse}" "${libcugraphops_wheelhouse}" "${pylibraft_wheelhouse}") + +# This is the version of the suffix with a preceding hyphen. It's used +# everywhere except in the final wheel name. +PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" + +# For nightlies we want to ensure that we're pulling in alphas as well. The +# easiest way to do so is to augment the spec with a constraint containing a +# min alpha version that doesn't affect the version bounds but does allow usage +# of alpha versions for that dependency without --pre +alpha_spec='' +if ! rapids-is-release-build; then + alpha_spec=',>=0.0.0a0' +fi + +build_wheel () { + local package_name="${1}" + local package_dir="${2}" + local underscore_package_name=$(echo "${package_name}" | tr "-" "_") + local version_package_name="$underscore_package_name" + if [[ "${version_package_name}" = "nx_cugraph" ]]; then + version_package_name="_nx_cugraph" + fi + + local package_dir="python/${package_name}" + local pyproject_file="${package_dir}/pyproject.toml" + local version_file="${package_dir}/${version_package_name}/_version.py" + + sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} + echo "${version}" > VERSION + sed -i "/^__git_commit__ / s/= .*/= \"${git_commit}\"/g" ${version_file} + + for dep in rmm cudf cugraph libcugraph raft-dask pylibcugraph pylibcugraphops pylibraft ucx-py; do + sed -r -i "s/${dep}==(.*)\"/${dep}${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} + done + + # dask-cuda & rapids-dask-dependency doesn't get a suffix, but it does get an alpha spec. + for dep in dask-cuda rapids-dask-dependency; do + sed -r -i "s/${dep}==(.*)\"/${dep}==\1${alpha_spec}\"/g" ${pyproject_file} + done + + if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then + sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file} + fi + + pushd "${package_dir}" + + local find_links="" + # Iterate over the array + for wheelhouse in "${wheelhouses[@]}"; do + find_links+="--find-links ${wheelhouse} " + done + + python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check ${find_links} + + # pure-python packages should be marked as pure, and not have auditwheel run on them. + if [[ ${package_name} == "nx-cugraph" ]] || \ + [[ ${package_name} == "cugraph-dgl" ]] || \ + [[ ${package_name} == "cugraph-pyg" ]] || \ + [[ ${package_name} == "cugraph-equivariant" ]]; then + RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-upload-wheels-to-s3 python dist + else + mkdir -p final_dist + python -m auditwheel repair -w final_dist --exclude libcugraph.so --exclude libcugraph_c.so --exclude libraft.so --exclude libcugraph-ops++.so dist/* + RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python final_dist + fi + + # Append ${PWD}/final_dist to the list of wheelhouses for the next package + wheelhouses+=("${PWD}/final_dist") + popd +} + +build_wheel pylibcugraph python/pylibcugraph +build_wheel cugraph python/cugraph +build_wheel nx-cugraph python/nx-cugraph +build_wheel cugraph-dgl python/cugraph-dgl +build_wheel cugraph-pyg python/cugraph-pyg +build_wheel cugraph-equivariant python/cugraph-equivariant From 703a99a43f3dd3cf017b60d46802bc6dee150654 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 11 Apr 2024 20:15:39 -0700 Subject: [PATCH 41/63] Apply suggestions from code review --- ci/build_wheel_cpp.sh | 4 ++-- ci/test_wheel.sh | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index b79043d08b..c47305a7bc 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -19,8 +19,8 @@ git_commit=$(git rev-parse HEAD) RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1512 cpp) -libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2251 cpp) +librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) +libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp) libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp a3ac1f3) # This is the version of the suffix with a preceding hyphen. It's used diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index 5cd08f2adf..4794b54033 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -27,10 +27,10 @@ else RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist fi -librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1512 cpp) -rmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1512 python) -libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2251 cpp) -pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2251 python) +librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) +rmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 python) +libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp) +pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 python) libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp a3ac1f3) RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcugraph_dist From d986c1c2b20de4ff9a05d0ef7a048b009064a80a Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 11 Apr 2024 20:26:07 -0700 Subject: [PATCH 42/63] Apply suggestions from code review --- ci/build_wheel.sh | 2 +- ci/build_wheel_cpp.sh | 2 +- ci/build_wheel_python.sh | 2 +- ci/test_wheel.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 8fa504c22d..53ed4ab03a 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -23,7 +23,7 @@ RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheel librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1512 cpp) libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2251 cpp) -libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp a3ac1f3) +libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp f843746) pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2251 python) # This is the version of the suffix with a preceding hyphen. It's used diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index c47305a7bc..04363479f3 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -21,7 +21,7 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp) -libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp a3ac1f3) +libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp f843746) # This is the version of the suffix with a preceding hyphen. It's used # everywhere except in the final wheel name. diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index 0e22181190..2fd0714253 100644 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -19,7 +19,7 @@ RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheel librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp) -libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp a3ac1f3) +libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp f843746) pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 python) wheelhouses=("${librmm_wheelhouse}" "${libraft_wheelhouse}" "${libcugraphops_wheelhouse}" "${pylibraft_wheelhouse}") diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index 4794b54033..e30ac8987a 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -31,7 +31,7 @@ librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapid rmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 python) libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp) pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 python) -libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp a3ac1f3) +libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp f843746) RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcugraph_dist From ab1fc042f9b103186fb7877d4f774fc0c0a02a50 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 11 Apr 2024 20:31:37 -0700 Subject: [PATCH 43/63] Update ci/test_wheel.sh --- ci/test_wheel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index e30ac8987a..359afac648 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -31,7 +31,7 @@ librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapid rmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 python) libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp) pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 python) -libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp f843746) +libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp f843746) RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcugraph_dist From 7bd2b44e36ba7e68e63c9998113d914b9dddf455 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 12 Apr 2024 12:14:58 +0000 Subject: [PATCH 44/63] Add exec perms --- ci/build_wheel_python.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 ci/build_wheel_python.sh diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh old mode 100644 new mode 100755 From f3051c384d6c9f996d49f684ee0c02baeaee3355 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 12 Apr 2024 06:30:30 -0700 Subject: [PATCH 45/63] Update ci/build_wheel_python.sh --- ci/build_wheel_python.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index 2fd0714253..71a1fd9119 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -22,7 +22,7 @@ libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rap libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp f843746) pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 python) -wheelhouses=("${librmm_wheelhouse}" "${libraft_wheelhouse}" "${libcugraphops_wheelhouse}" "${pylibraft_wheelhouse}") +wheelhouses=("${librmm_wheelhouse}" "${libraft_wheelhouse}" "${libcugraphops_wheelhouse}" "${pylibraft_wheelhouse}" "/tmp/libcugraph_dist") # This is the version of the suffix with a preceding hyphen. It's used # everywhere except in the final wheel name. From 2baffb710723bfb0bb00ddd57c84be0310158b5f Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 16 Apr 2024 23:54:05 +0000 Subject: [PATCH 46/63] Delete now superfluous build scripts --- ci/build_wheel.sh | 82 --------------------------- ci/build_wheel_cugraph-dgl.sh | 6 -- ci/build_wheel_cugraph-equivariant.sh | 6 -- ci/build_wheel_cugraph-pyg.sh | 6 -- ci/build_wheel_cugraph.sh | 17 ------ ci/build_wheel_nx-cugraph.sh | 6 -- ci/build_wheel_pylibcugraph.sh | 8 --- 7 files changed, 131 deletions(-) delete mode 100755 ci/build_wheel.sh delete mode 100755 ci/build_wheel_cugraph-dgl.sh delete mode 100755 ci/build_wheel_cugraph-equivariant.sh delete mode 100755 ci/build_wheel_cugraph-pyg.sh delete mode 100755 ci/build_wheel_cugraph.sh delete mode 100755 ci/build_wheel_nx-cugraph.sh delete mode 100755 ci/build_wheel_pylibcugraph.sh diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh deleted file mode 100755 index 53ed4ab03a..0000000000 --- a/ci/build_wheel.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/bin/bash -# Copyright (c) 2023-2024, NVIDIA CORPORATION. - -set -euo pipefail - -package_name=$1 -package_dir=$2 -underscore_package_name=$(echo "${package_name}" | tr "-" "_") - -if [[ ! -d "/tmp/gha-tools" ]]; then - git clone https://github.com/msarahan/gha-tools.git -b get-pr-wheel-artifact /tmp/gha-tools -fi -export PATH="/tmp/gha-tools/tools:${PATH}" - -source rapids-configure-sccache -source rapids-date-string - -version=$(rapids-generate-version) -git_commit=$(git rev-parse HEAD) - -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcugraph_dist - -librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1512 cpp) -libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2251 cpp) -libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp f843746) -pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2251 python) - -# This is the version of the suffix with a preceding hyphen. It's used -# everywhere except in the final wheel name. -PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" - -# Patch project metadata files to include the CUDA version suffix and version override. -version_package_name="$underscore_package_name" -if [[ "${version_package_name}" = "nx_cugraph" ]]; then - version_package_name="_nx_cugraph" -fi -pyproject_file="${package_dir}/pyproject.toml" -version_file="${package_dir}/${version_package_name}/_version.py" - -sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} -echo "${version}" > VERSION -sed -i "/^__git_commit__ / s/= .*/= \"${git_commit}\"/g" ${version_file} - -# For nightlies we want to ensure that we're pulling in alphas as well. The -# easiest way to do so is to augment the spec with a constraint containing a -# min alpha version that doesn't affect the version bounds but does allow usage -# of alpha versions for that dependency without --pre -alpha_spec='' -if ! rapids-is-release-build; then - alpha_spec=',>=0.0.0a0' -fi - -for dep in rmm cudf cugraph libcugraph raft-dask pylibcugraph pylibcugraphops pylibraft ucx-py; do - sed -r -i "s/${dep}==(.*)\"/${dep}${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} -done - -# dask-cuda & rapids-dask-dependency doesn't get a suffix, but it does get an alpha spec. -for dep in dask-cuda rapids-dask-dependency; do - sed -r -i "s/${dep}==(.*)\"/${dep}==\1${alpha_spec}\"/g" ${pyproject_file} -done - - -if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then - sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file} -fi - -cd "${package_dir}" - -PIP_FIND_LINKS="/tmp/libcugraph_dist ${librmm_wheelhouse} ${libraft_wheelhouse} ${libcugraphops_wheelhouse} ${pylibraft_wheelhouse}" python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check - -# pure-python packages should be marked as pure, and not have auditwheel run on them. -if [[ ${package_name} == "nx-cugraph" ]] || \ - [[ ${package_name} == "cugraph-dgl" ]] || \ - [[ ${package_name} == "cugraph-pyg" ]] || \ - [[ ${package_name} == "cugraph-equivariant" ]]; then - RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-upload-wheels-to-s3 python dist -else - mkdir -p final_dist - python -m auditwheel repair -w final_dist --exclude libcugraph.so --exclude libcugraph_c.so --exclude libraft.so --exclude libcugraph-ops++.so dist/* - RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python final_dist -fi diff --git a/ci/build_wheel_cugraph-dgl.sh b/ci/build_wheel_cugraph-dgl.sh deleted file mode 100755 index d62f810cba..0000000000 --- a/ci/build_wheel_cugraph-dgl.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -# Copyright (c) 2024, NVIDIA CORPORATION. - -set -euo pipefail - -./ci/build_wheel.sh cugraph-dgl python/cugraph-dgl diff --git a/ci/build_wheel_cugraph-equivariant.sh b/ci/build_wheel_cugraph-equivariant.sh deleted file mode 100755 index fcc8e0f774..0000000000 --- a/ci/build_wheel_cugraph-equivariant.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -# Copyright (c) 2024, NVIDIA CORPORATION. - -set -euo pipefail - -./ci/build_wheel.sh cugraph-equivariant python/cugraph-equivariant diff --git a/ci/build_wheel_cugraph-pyg.sh b/ci/build_wheel_cugraph-pyg.sh deleted file mode 100755 index 97baa243f7..0000000000 --- a/ci/build_wheel_cugraph-pyg.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -# Copyright (c) 2024, NVIDIA CORPORATION. - -set -euo pipefail - -./ci/build_wheel.sh cugraph-pyg python/cugraph-pyg diff --git a/ci/build_wheel_cugraph.sh b/ci/build_wheel_cugraph.sh deleted file mode 100755 index ffd6445f8d..0000000000 --- a/ci/build_wheel_cugraph.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -# Copyright (c) 2023, NVIDIA CORPORATION. - -set -euo pipefail - -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" - -# Download the pylibcugraph wheel built in the previous step and make it -# available for pip to find. We must use PIP_FIND_LINKS because the package -# must be made available to the isolated build step, and there is no way to -# manually install it into that environment. -RAPIDS_PY_WHEEL_NAME=pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX} rapids-download-wheels-from-s3 ./local-pylibcugraph -export PIP_FIND_LINKS=$(pwd)/local-pylibcugraph - -export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_CUGRAPH_CPP=OFF;-DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/cugraph-ops/" - -./ci/build_wheel.sh cugraph python/cugraph diff --git a/ci/build_wheel_nx-cugraph.sh b/ci/build_wheel_nx-cugraph.sh deleted file mode 100755 index 4481de1283..0000000000 --- a/ci/build_wheel_nx-cugraph.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -# Copyright (c) 2023, NVIDIA CORPORATION. - -set -euo pipefail - -./ci/build_wheel.sh nx-cugraph python/nx-cugraph diff --git a/ci/build_wheel_pylibcugraph.sh b/ci/build_wheel_pylibcugraph.sh deleted file mode 100755 index 7c5a729942..0000000000 --- a/ci/build_wheel_pylibcugraph.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -# Copyright (c) 2023, NVIDIA CORPORATION. - -set -euo pipefail - -export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_CUGRAPH_CPP=OFF;-DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/cugraph-ops/" - -./ci/build_wheel.sh pylibcugraph python/pylibcugraph From 7bcfb8bb5833e3528057a8585f5aa32de980872f Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 17 Apr 2024 00:11:35 +0000 Subject: [PATCH 47/63] Update remaining testing scripts --- ci/test_wheel_cugraph-dgl.sh | 12 +++++++++--- ci/test_wheel_cugraph-pyg.sh | 12 +++++++++--- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/ci/test_wheel_cugraph-dgl.sh b/ci/test_wheel_cugraph-dgl.sh index 827ad48711..2aea95fb37 100755 --- a/ci/test_wheel_cugraph-dgl.sh +++ b/ci/test_wheel_cugraph-dgl.sh @@ -8,18 +8,24 @@ package_dir="python/cugraph-dgl" python_package_name=$(echo ${package_name}|sed 's/-/_/g') -mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" # Download wheels built during this job. RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps RAPIDS_PY_WHEEL_NAME="cugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps -python -m pip install ./local-deps/*.whl + +librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) +rmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 python) +libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp) +pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 python) +libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp f843746) + +RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcugraph_dist # use 'ls' to expand wildcard before adding `[extra]` requires for pip RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist # pip creates wheels using python package names -python -m pip install $(ls ./dist/${python_package_name}*.whl)[test] +python -m pip install $(ls ./dist/${python_package_name}*.whl)[test] --find-links /tmp/libcugraph_dist --find-links ${librmm_wheelhouse} --find-links ${rmm_wheelhouse} --find-links ${libraft_wheelhouse} --find-links ${pylibraft_wheelhouse} --find-links ${libcugraphops_wheelhouse} --find-links ./local-deps PKG_CUDA_VER="$(echo ${CUDA_VERSION} | cut -d '.' -f1,2 | tr -d '.')" diff --git a/ci/test_wheel_cugraph-pyg.sh b/ci/test_wheel_cugraph-pyg.sh index e98bf4ab56..b2581327c9 100755 --- a/ci/test_wheel_cugraph-pyg.sh +++ b/ci/test_wheel_cugraph-pyg.sh @@ -8,18 +8,24 @@ package_dir="python/cugraph-pyg" python_package_name=$(echo ${package_name}|sed 's/-/_/g') -mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" # Download wheels built during this job. RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps RAPIDS_PY_WHEEL_NAME="cugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps -python -m pip install ./local-deps/*.whl + +librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) +rmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 python) +libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp) +pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 python) +libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp f843746) + +RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcugraph_dist # use 'ls' to expand wildcard before adding `[extra]` requires for pip RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist # pip creates wheels using python package names -python -m pip install $(ls ./dist/${python_package_name}*.whl)[test] +python -m pip install $(ls ./dist/${python_package_name}*.whl)[test] --find-links /tmp/libcugraph_dist --find-links ${librmm_wheelhouse} --find-links ${rmm_wheelhouse} --find-links ${libraft_wheelhouse} --find-links ${pylibraft_wheelhouse} --find-links ${libcugraphops_wheelhouse} --find-links ./local-deps # RAPIDS_DATASET_ROOT_DIR is used by test scripts export RAPIDS_DATASET_ROOT_DIR="$(realpath datasets)" From 449e81214f69f7febce7052f178d67292faff244 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 17 Apr 2024 00:15:59 +0000 Subject: [PATCH 48/63] Remove unused build jobs --- .github/workflows/pr.yaml | 39 --------------------------------------- 1 file changed, 39 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 9bfe630f4b..5efd8e6abb 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -23,15 +23,10 @@ jobs: - wheel-build-cpp - wheel-build-python - wheel-tests-pylibcugraph - #- wheel-build-cugraph - wheel-tests-cugraph - #- wheel-build-nx-cugraph - wheel-tests-nx-cugraph - #- wheel-build-cugraph-dgl - wheel-tests-cugraph-dgl - #- wheel-build-cugraph-pyg - wheel-tests-cugraph-pyg - #- wheel-build-cugraph-equivariant - wheel-tests-cugraph-equivariant - devcontainer secrets: inherit @@ -118,13 +113,6 @@ jobs: with: build_type: pull-request script: ci/test_wheel_pylibcugraph.sh - #wheel-build-cugraph: - # needs: wheel-build-pylibcugraph - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 - # with: - # build_type: pull-request - # script: ci/build_wheel_cugraph.sh wheel-tests-cugraph: needs: wheel-build-python secrets: inherit @@ -132,13 +120,6 @@ jobs: with: build_type: pull-request script: ci/test_wheel_cugraph.sh - #wheel-build-nx-cugraph: - # needs: wheel-tests-pylibcugraph - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 - # with: - # build_type: pull-request - # script: ci/build_wheel_nx-cugraph.sh wheel-tests-nx-cugraph: needs: wheel-build-python secrets: inherit @@ -146,13 +127,6 @@ jobs: with: build_type: pull-request script: ci/test_wheel_nx-cugraph.sh - #wheel-build-cugraph-dgl: - # needs: wheel-tests-cugraph - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 - # with: - # build_type: pull-request - # script: ci/build_wheel_cugraph-dgl.sh wheel-tests-cugraph-dgl: needs: wheel-build-python secrets: inherit @@ -161,13 +135,6 @@ jobs: build_type: pull-request script: ci/test_wheel_cugraph-dgl.sh matrix_filter: map(select(.ARCH == "amd64")) - #wheel-build-cugraph-pyg: - # needs: wheel-tests-cugraph - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 - # with: - # build_type: pull-request - # script: ci/build_wheel_cugraph-pyg.sh wheel-tests-cugraph-pyg: needs: wheel-build-python secrets: inherit @@ -176,12 +143,6 @@ jobs: build_type: pull-request script: ci/test_wheel_cugraph-pyg.sh matrix_filter: map(select(.ARCH == "amd64")) - #wheel-build-cugraph-equivariant: - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 - # with: - # build_type: pull-request - # script: ci/build_wheel_cugraph-equivariant.sh wheel-tests-cugraph-equivariant: needs: wheel-build-python secrets: inherit From f41a3c61b372dccf3cfaab0c8257858e8c354af7 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 17 Apr 2024 01:51:50 +0000 Subject: [PATCH 49/63] Some cleanup --- ci/build_wheel_python.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index 71a1fd9119..2b8276e8fb 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -28,6 +28,8 @@ wheelhouses=("${librmm_wheelhouse}" "${libraft_wheelhouse}" "${libcugraphops_whe # everywhere except in the final wheel name. PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" +echo "${version}" > VERSION + # For nightlies we want to ensure that we're pulling in alphas as well. The # easiest way to do so is to augment the spec with a constraint containing a # min alpha version that doesn't affect the version bounds but does allow usage @@ -39,7 +41,6 @@ fi build_wheel () { local package_name="${1}" - local package_dir="${2}" local underscore_package_name=$(echo "${package_name}" | tr "-" "_") local version_package_name="$underscore_package_name" if [[ "${version_package_name}" = "nx_cugraph" ]]; then @@ -51,14 +52,13 @@ build_wheel () { local version_file="${package_dir}/${version_package_name}/_version.py" sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} - echo "${version}" > VERSION sed -i "/^__git_commit__ / s/= .*/= \"${git_commit}\"/g" ${version_file} for dep in rmm cudf cugraph libcugraph raft-dask pylibcugraph pylibcugraphops pylibraft ucx-py; do sed -r -i "s/${dep}==(.*)\"/${dep}${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} done - # dask-cuda & rapids-dask-dependency doesn't get a suffix, but it does get an alpha spec. + # dask-cuda & rapids-dask-dependency don't get a suffix, but they do get an alpha spec. for dep in dask-cuda rapids-dask-dependency; do sed -r -i "s/${dep}==(.*)\"/${dep}==\1${alpha_spec}\"/g" ${pyproject_file} done @@ -94,9 +94,9 @@ build_wheel () { popd } -build_wheel pylibcugraph python/pylibcugraph -build_wheel cugraph python/cugraph -build_wheel nx-cugraph python/nx-cugraph -build_wheel cugraph-dgl python/cugraph-dgl -build_wheel cugraph-pyg python/cugraph-pyg -build_wheel cugraph-equivariant python/cugraph-equivariant +build_wheel pylibcugraph +build_wheel cugraph +build_wheel nx-cugraph +build_wheel cugraph-dgl +build_wheel cugraph-pyg +build_wheel cugraph-equivariant From 857c495d7d7eb1a52af3b16328816f1056b9f7c1 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 17 Apr 2024 06:08:57 -0700 Subject: [PATCH 50/63] Apply suggestions from code review --- ci/build_wheel_cpp.sh | 2 +- ci/build_wheel_python.sh | 2 +- ci/test_wheel.sh | 2 +- ci/test_wheel_cugraph-dgl.sh | 2 +- ci/test_wheel_cugraph-pyg.sh | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index 04363479f3..5bbea1b4a6 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -21,7 +21,7 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp) -libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp f843746) +libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp c8c3bce) # This is the version of the suffix with a preceding hyphen. It's used # everywhere except in the final wheel name. diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index 2b8276e8fb..3ad0dcc2f5 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -19,7 +19,7 @@ RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheel librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp) -libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp f843746) +libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp c8c3bce) pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 python) wheelhouses=("${librmm_wheelhouse}" "${libraft_wheelhouse}" "${libcugraphops_wheelhouse}" "${pylibraft_wheelhouse}" "/tmp/libcugraph_dist") diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index 359afac648..561f922672 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -31,7 +31,7 @@ librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapid rmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 python) libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp) pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 python) -libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp f843746) +libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp c8c3bce) RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcugraph_dist diff --git a/ci/test_wheel_cugraph-dgl.sh b/ci/test_wheel_cugraph-dgl.sh index 2aea95fb37..d15de37ec8 100755 --- a/ci/test_wheel_cugraph-dgl.sh +++ b/ci/test_wheel_cugraph-dgl.sh @@ -18,7 +18,7 @@ librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapid rmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 python) libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp) pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 python) -libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp f843746) +libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp c8c3bce) RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcugraph_dist diff --git a/ci/test_wheel_cugraph-pyg.sh b/ci/test_wheel_cugraph-pyg.sh index b2581327c9..0e0b5dc608 100755 --- a/ci/test_wheel_cugraph-pyg.sh +++ b/ci/test_wheel_cugraph-pyg.sh @@ -18,7 +18,7 @@ librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapid rmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 python) libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp) pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 python) -libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp f843746) +libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp c8c3bce) RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcugraph_dist From 6f47f562a4ea05b2b8d57e9a5e4620ab620ae64c Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 17 Apr 2024 14:49:17 +0000 Subject: [PATCH 51/63] Remove some more superfluous clones --- ci/build_wheel_cpp.sh | 5 ----- ci/build_wheel_python.sh | 5 ----- ci/test_wheel.sh | 5 ----- 3 files changed, 15 deletions(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index 5bbea1b4a6..98095fe605 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -6,11 +6,6 @@ set -euo pipefail package_name="libcugraph" package_dir="python/libcugraph" -if [[ ! -d "/tmp/gha-tools" ]]; then - git clone https://github.com/msarahan/gha-tools.git -b get-pr-wheel-artifact /tmp/gha-tools -fi -export PATH="/tmp/gha-tools/tools:${PATH}" - source rapids-configure-sccache source rapids-date-string diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index 3ad0dcc2f5..c5bb4289a0 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -3,11 +3,6 @@ set -euo pipefail -if [[ ! -d "/tmp/gha-tools" ]]; then - git clone https://github.com/msarahan/gha-tools.git -b get-pr-wheel-artifact /tmp/gha-tools -fi -export PATH="/tmp/gha-tools/tools:${PATH}" - source rapids-configure-sccache source rapids-date-string diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index 561f922672..0068b26224 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -3,11 +3,6 @@ set -eoxu pipefail -if [[ ! -d "/tmp/gha-tools" ]]; then - git clone https://github.com/msarahan/gha-tools.git -b get-pr-wheel-artifact /tmp/gha-tools -fi -export PATH="/tmp/gha-tools/tools:${PATH}" - # TODO: Enable dask query planning (by default) once some bugs are fixed. # xref: https://github.com/rapidsai/cudf/issues/15027 export DASK_DATAFRAME__QUERY_PLANNING=False From be0896b34e88a88ef7f97593ce94fb2a8204d55b Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 17 Apr 2024 17:29:58 +0000 Subject: [PATCH 52/63] Improve handling of wheelhouses, grouping packages, etc --- ci/build_wheel_cpp.sh | 16 ++++------ ci/build_wheel_python.sh | 66 ++++++++++++++++++---------------------- 2 files changed, 36 insertions(+), 46 deletions(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index 98095fe605..0eb8073a0e 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -14,10 +14,6 @@ git_commit=$(git rev-parse HEAD) RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) -libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp) -libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp c8c3bce) - # This is the version of the suffix with a preceding hyphen. It's used # everywhere except in the final wheel name. PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" @@ -44,12 +40,12 @@ done cd "${package_dir}" -PIP_FIND_LINKS="${librmm_wheelhouse} ${libraft_wheelhouse} ${libcugraphops_wheelhouse}" python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check +librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) +libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp 0435e6b) +libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp c8c3bce) + +python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check --find-links ${librmm_wheelhouse} --find-links ${libraft_wheelhouse} --find-links ${libcugraphops_wheelhouse} -# Don't repair raft into this wheel. Everything else is fair game. -# TODO: Check if this works transitively, i.e. if raft links to something that -# it bundles in then will cugraph try to pull that into its own libs? If so, -# we'll need to programatically exclude raft's dependencies from cugraph's. mkdir -p final_dist python -m auditwheel repair --exclude libraft.so --exclude libcugraph-ops++.so -w final_dist dist/* -RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp final_dist +RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp final_dist diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index c5bb4289a0..391bb5c611 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -10,14 +10,25 @@ version=$(rapids-generate-version) git_commit=$(git rev-parse HEAD) RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcugraph_dist +CPP_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcugraph_dist) -librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) -libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp) -libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp c8c3bce) -pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 python) +librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) +libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp 0435e6b) +libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp c8c3bce) +pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 python) -wheelhouses=("${librmm_wheelhouse}" "${libraft_wheelhouse}" "${libcugraphops_wheelhouse}" "${pylibraft_wheelhouse}" "/tmp/libcugraph_dist") +PYTHON_WHEELHOUSE="${PWD}/dist/" +PYTHON_PURE_WHEELHOUSE="${PWD}/pure_dist/" +PYTHON_AUDITED_WHEELHOUSE="${PWD}/final_dist/" + +wheelhouses=("${librmm_wheelhouse}" "${libraft_wheelhouse}" "${libcugraphops_wheelhouse}" "${pylibraft_wheelhouse}" "${CPP_WHEELHOUSE}" "${PYTHON_WHEELHOUSE}" "${PYTHON_PURE_WHEELHOUSE}") +mkdir -p "${PYTHON_AUDITED_WHEELHOUSE}" + +FIND_LINKS="" +# Iterate over the array +for wheelhouse in "${WHEELHOUSES[@]}"; do + FIND_LINKS+="--find-links ${wheelhouse} " +done # This is the version of the suffix with a preceding hyphen. It's used # everywhere except in the final wheel name. @@ -36,6 +47,8 @@ fi build_wheel () { local package_name="${1}" + local current_wheelhouse="${2}" + local underscore_package_name=$(echo "${package_name}" | tr "-" "_") local version_package_name="$underscore_package_name" if [[ "${version_package_name}" = "nx_cugraph" ]]; then @@ -62,36 +75,17 @@ build_wheel () { sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file} fi - pushd "${package_dir}" - - local find_links="" - # Iterate over the array - for wheelhouse in "${wheelhouses[@]}"; do - find_links+="--find-links ${wheelhouse} " - done + python -m pip wheel "${package_dir}" -w ${current_wheelhouse} -vvv --no-deps --disable-pip-version-check ${FIND_LINKS} +} - python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check ${find_links} - - # pure-python packages should be marked as pure, and not have auditwheel run on them. - if [[ ${package_name} == "nx-cugraph" ]] || \ - [[ ${package_name} == "cugraph-dgl" ]] || \ - [[ ${package_name} == "cugraph-pyg" ]] || \ - [[ ${package_name} == "cugraph-equivariant" ]]; then - RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-upload-wheels-to-s3 python dist - else - mkdir -p final_dist - python -m auditwheel repair -w final_dist --exclude libcugraph.so --exclude libcugraph_c.so --exclude libraft.so --exclude libcugraph-ops++.so dist/* - RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python final_dist - fi +build_wheel pylibcugraph "${PYTHON_WHEELHOUSE}" +build_wheel cugraph "${PYTHON_WHEELHOUSE}" +build_wheel nx-cugraph "${PYTHON_PURE_WHEELHOUSE}" +build_wheel cugraph-dgl "${PYTHON_PURE_WHEELHOUSE}" +build_wheel cugraph-pyg "${PYTHON_PURE_WHEELHOUSE}" +build_wheel cugraph-equivariant "${PYTHON_PURE_WHEELHOUSE}" - # Append ${PWD}/final_dist to the list of wheelhouses for the next package - wheelhouses+=("${PWD}/final_dist") - popd -} +RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-upload-wheels-to-s3 python ${PYTHON_PURE_WHEELHOUSE} -build_wheel pylibcugraph -build_wheel cugraph -build_wheel nx-cugraph -build_wheel cugraph-dgl -build_wheel cugraph-pyg -build_wheel cugraph-equivariant +python -m auditwheel repair -w "${PYTHON_AUDITED_WHEELHOUSE}" --exclude libcugraph.so --exclude libcugraph_c.so --exclude libraft.so --exclude libcugraph-ops++.so ${PYTHON_WHEELHOUSE}/* +RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python ${PYTHON_AUDITED_WHEELHOUSE} From 121c23c339a52e25ac70a652b1e60af4a7d4bcb5 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 17 Apr 2024 17:47:23 +0000 Subject: [PATCH 53/63] Update cugraph-ops hash --- ci/build_wheel_cpp.sh | 2 +- ci/build_wheel_python.sh | 2 +- ci/test_wheel.sh | 2 +- ci/test_wheel_cugraph-dgl.sh | 2 +- ci/test_wheel_cugraph-pyg.sh | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index 0eb8073a0e..1f63605681 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -42,7 +42,7 @@ cd "${package_dir}" librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp 0435e6b) -libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp c8c3bce) +libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp 7c6f068) python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check --find-links ${librmm_wheelhouse} --find-links ${libraft_wheelhouse} --find-links ${libcugraphops_wheelhouse} diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index 391bb5c611..bc36dedf08 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -14,7 +14,7 @@ CPP_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-download librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp 0435e6b) -libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp c8c3bce) +libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp 7c6f068) pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 python) PYTHON_WHEELHOUSE="${PWD}/dist/" diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index 0068b26224..cbb1464c05 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -26,7 +26,7 @@ librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapid rmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 python) libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp) pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 python) -libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp c8c3bce) +libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp 7c6f068) RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcugraph_dist diff --git a/ci/test_wheel_cugraph-dgl.sh b/ci/test_wheel_cugraph-dgl.sh index d15de37ec8..3b9a340043 100755 --- a/ci/test_wheel_cugraph-dgl.sh +++ b/ci/test_wheel_cugraph-dgl.sh @@ -18,7 +18,7 @@ librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapid rmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 python) libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp) pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 python) -libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp c8c3bce) +libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp 7c6f068) RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcugraph_dist diff --git a/ci/test_wheel_cugraph-pyg.sh b/ci/test_wheel_cugraph-pyg.sh index 0e0b5dc608..d3fb1cb484 100755 --- a/ci/test_wheel_cugraph-pyg.sh +++ b/ci/test_wheel_cugraph-pyg.sh @@ -18,7 +18,7 @@ librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapid rmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 python) libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp) pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 python) -libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp c8c3bce) +libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp 7c6f068) RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcugraph_dist From bc6f85e528ab61c1879de0798a8e0ffbb9171113 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 17 Apr 2024 18:58:03 +0000 Subject: [PATCH 54/63] Fix hash again --- ci/build_wheel_cpp.sh | 2 +- ci/build_wheel_python.sh | 2 +- ci/test_wheel.sh | 2 +- ci/test_wheel_cugraph-dgl.sh | 2 +- ci/test_wheel_cugraph-pyg.sh | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index 1f63605681..79cbb11bea 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -42,7 +42,7 @@ cd "${package_dir}" librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp 0435e6b) -libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp 7c6f068) +libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp e7c6f06) python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check --find-links ${librmm_wheelhouse} --find-links ${libraft_wheelhouse} --find-links ${libcugraphops_wheelhouse} diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index bc36dedf08..cde60333f4 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -14,7 +14,7 @@ CPP_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-download librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp 0435e6b) -libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp 7c6f068) +libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp e7c6f06) pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 python) PYTHON_WHEELHOUSE="${PWD}/dist/" diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index cbb1464c05..d3a78bb1e2 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -26,7 +26,7 @@ librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapid rmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 python) libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp) pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 python) -libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp 7c6f068) +libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp e7c6f06) RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcugraph_dist diff --git a/ci/test_wheel_cugraph-dgl.sh b/ci/test_wheel_cugraph-dgl.sh index 3b9a340043..dc7e2d228d 100755 --- a/ci/test_wheel_cugraph-dgl.sh +++ b/ci/test_wheel_cugraph-dgl.sh @@ -18,7 +18,7 @@ librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapid rmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 python) libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp) pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 python) -libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp 7c6f068) +libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp e7c6f06) RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcugraph_dist diff --git a/ci/test_wheel_cugraph-pyg.sh b/ci/test_wheel_cugraph-pyg.sh index d3fb1cb484..d9bd1bbd86 100755 --- a/ci/test_wheel_cugraph-pyg.sh +++ b/ci/test_wheel_cugraph-pyg.sh @@ -18,7 +18,7 @@ librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapid rmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 python) libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp) pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 python) -libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp 7c6f068) +libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp e7c6f06) RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcugraph_dist From 3436e51371598aef783aff31479dadec800931d1 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 17 Apr 2024 20:56:47 +0000 Subject: [PATCH 55/63] Fix typo --- ci/build_wheel_python.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index cde60333f4..ba7fa5f742 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -21,7 +21,7 @@ PYTHON_WHEELHOUSE="${PWD}/dist/" PYTHON_PURE_WHEELHOUSE="${PWD}/pure_dist/" PYTHON_AUDITED_WHEELHOUSE="${PWD}/final_dist/" -wheelhouses=("${librmm_wheelhouse}" "${libraft_wheelhouse}" "${libcugraphops_wheelhouse}" "${pylibraft_wheelhouse}" "${CPP_WHEELHOUSE}" "${PYTHON_WHEELHOUSE}" "${PYTHON_PURE_WHEELHOUSE}") +WHEELHOUSES=("${librmm_wheelhouse}" "${libraft_wheelhouse}" "${libcugraphops_wheelhouse}" "${pylibraft_wheelhouse}" "${CPP_WHEELHOUSE}" "${PYTHON_WHEELHOUSE}" "${PYTHON_PURE_WHEELHOUSE}") mkdir -p "${PYTHON_AUDITED_WHEELHOUSE}" FIND_LINKS="" From 12c9fc0bbfcef6a15c890a6d7c2f74a52e050d70 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 17 Apr 2024 22:08:34 +0000 Subject: [PATCH 56/63] Update test scripts --- ci/test_wheel.sh | 26 +++++++++----------------- ci/test_wheel_cugraph-dgl.sh | 24 +++++++++--------------- ci/test_wheel_cugraph-equivariant.sh | 6 ++---- ci/test_wheel_cugraph-pyg.sh | 22 +++++++++------------- ci/test_wheel_cugraph.sh | 7 +------ 5 files changed, 30 insertions(+), 55 deletions(-) diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index d3a78bb1e2..e1365ccc74 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -12,27 +12,19 @@ package_dir=$2 python_package_name=$(echo ${package_name}|sed 's/-/_/g') -mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -# nx-cugraph is a pure wheel, which is part of generating the download path -if [[ "${package_name}" == "nx-cugraph" ]]; then - RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 python ./dist -else - RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist -fi - -librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) -rmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 python) -libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp) -pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 python) -libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp e7c6f06) +librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) +rmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 python) +libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp) +pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 python) +libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp e7c6f06) -RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcugraph_dist +RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist +RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist +RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 python ./dist -# use 'ls' to expand wildcard before adding `[extra]` requires for pip -# pip creates wheels using python package names -python -m pip install $(ls ./dist/${python_package_name}*.whl)[test] --find-links /tmp/libcugraph_dist --find-links ${librmm_wheelhouse} --find-links ${rmm_wheelhouse} --find-links ${libraft_wheelhouse} --find-links ${pylibraft_wheelhouse} --find-links ${libcugraphops_wheelhouse} +python -m pip install "${python_package_name}-${RAPIDS_PY_CUDA_SUFFIX}[test]" --find-links ${librmm_wheelhouse} --find-links ${rmm_wheelhouse} --find-links ${libraft_wheelhouse} --find-links ${pylibraft_wheelhouse} --find-links ${libcugraphops_wheelhouse} --find-links ./dist # Run smoke tests for aarch64 pull requests arch=$(uname -m) diff --git a/ci/test_wheel_cugraph-dgl.sh b/ci/test_wheel_cugraph-dgl.sh index dc7e2d228d..0f2d4d058e 100755 --- a/ci/test_wheel_cugraph-dgl.sh +++ b/ci/test_wheel_cugraph-dgl.sh @@ -10,23 +10,17 @@ python_package_name=$(echo ${package_name}|sed 's/-/_/g') RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -# Download wheels built during this job. -RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps -RAPIDS_PY_WHEEL_NAME="cugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps +librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) +rmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 python) +libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp) +pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 python) +libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp e7c6f06) -librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) -rmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 python) -libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp) -pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 python) -libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp e7c6f06) - -RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcugraph_dist - -# use 'ls' to expand wildcard before adding `[extra]` requires for pip -RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist -# pip creates wheels using python package names -python -m pip install $(ls ./dist/${python_package_name}*.whl)[test] --find-links /tmp/libcugraph_dist --find-links ${librmm_wheelhouse} --find-links ${rmm_wheelhouse} --find-links ${libraft_wheelhouse} --find-links ${pylibraft_wheelhouse} --find-links ${libcugraphops_wheelhouse} --find-links ./local-deps +RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist +RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist +RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 python ./dist +python -m pip install "${python_package_name}-${RAPIDS_PY_CUDA_SUFFIX}[test]" --find-links ${librmm_wheelhouse} --find-links ${rmm_wheelhouse} --find-links ${libraft_wheelhouse} --find-links ${pylibraft_wheelhouse} --find-links ${libcugraphops_wheelhouse} --find-links ./dist PKG_CUDA_VER="$(echo ${CUDA_VERSION} | cut -d '.' -f1,2 | tr -d '.')" PKG_CUDA_VER_MAJOR=${PKG_CUDA_VER:0:2} diff --git a/ci/test_wheel_cugraph-equivariant.sh b/ci/test_wheel_cugraph-equivariant.sh index cb952055f0..1085d29aa0 100755 --- a/ci/test_wheel_cugraph-equivariant.sh +++ b/ci/test_wheel_cugraph-equivariant.sh @@ -8,14 +8,12 @@ package_dir="python/cugraph-equivariant" python_package_name=$(echo ${package_name}|sed 's/-/_/g') -mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -# use 'ls' to expand wildcard before adding `[extra]` requires for pip +RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist -# pip creates wheels using python package names -python -m pip install $(ls ./dist/${python_package_name}*.whl)[test] +python -m pip install "${python_package_name}-${RAPIDS_PY_CUDA_SUFFIX}[test]" --find-links ./dist PKG_CUDA_VER="$(echo ${CUDA_VERSION} | cut -d '.' -f1,2 | tr -d '.')" PKG_CUDA_VER_MAJOR=${PKG_CUDA_VER:0:2} diff --git a/ci/test_wheel_cugraph-pyg.sh b/ci/test_wheel_cugraph-pyg.sh index d9bd1bbd86..0d9f32917d 100755 --- a/ci/test_wheel_cugraph-pyg.sh +++ b/ci/test_wheel_cugraph-pyg.sh @@ -10,22 +10,18 @@ python_package_name=$(echo ${package_name}|sed 's/-/_/g') RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -# Download wheels built during this job. -RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps -RAPIDS_PY_WHEEL_NAME="cugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps +librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) +rmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 python) +libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp) +pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 python) +libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp e7c6f06) -librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) -rmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 python) -libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp) -pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 python) -libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libcugraphops_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp e7c6f06) +RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist +RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist +RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 python ./dist -RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcugraph_dist - -# use 'ls' to expand wildcard before adding `[extra]` requires for pip -RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist # pip creates wheels using python package names -python -m pip install $(ls ./dist/${python_package_name}*.whl)[test] --find-links /tmp/libcugraph_dist --find-links ${librmm_wheelhouse} --find-links ${rmm_wheelhouse} --find-links ${libraft_wheelhouse} --find-links ${pylibraft_wheelhouse} --find-links ${libcugraphops_wheelhouse} --find-links ./local-deps +python -m pip install "${python_package_name}-${RAPIDS_PY_CUDA_SUFFIX}[test]" --find-links ${librmm_wheelhouse} --find-links ${rmm_wheelhouse} --find-links ${libraft_wheelhouse} --find-links ${pylibraft_wheelhouse} --find-links ${libcugraphops_wheelhouse} --find-links ./dist # RAPIDS_DATASET_ROOT_DIR is used by test scripts export RAPIDS_DATASET_ROOT_DIR="$(realpath datasets)" diff --git a/ci/test_wheel_cugraph.sh b/ci/test_wheel_cugraph.sh index d351ea2162..3dd3771a5f 100755 --- a/ci/test_wheel_cugraph.sh +++ b/ci/test_wheel_cugraph.sh @@ -1,11 +1,6 @@ #!/bin/bash -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. set -eoxu pipefail -# Download the pylibcugraph built in the previous step -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-pylibcugraph-dep -python -m pip install --no-deps ./local-pylibcugraph-dep/pylibcugraph*.whl - ./ci/test_wheel.sh cugraph python/cugraph From cd8c8b020ea245340ad2db78607a7a18f7010fa0 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 18 Apr 2024 00:49:15 +0000 Subject: [PATCH 57/63] Make VERSION file a symlink --- python/libcugraph/libcugraph/VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 120000 python/libcugraph/libcugraph/VERSION diff --git a/python/libcugraph/libcugraph/VERSION b/python/libcugraph/libcugraph/VERSION deleted file mode 100644 index 0bff6981a3..0000000000 --- a/python/libcugraph/libcugraph/VERSION +++ /dev/null @@ -1 +0,0 @@ -24.06.00 diff --git a/python/libcugraph/libcugraph/VERSION b/python/libcugraph/libcugraph/VERSION new file mode 120000 index 0000000000..d62dc733ef --- /dev/null +++ b/python/libcugraph/libcugraph/VERSION @@ -0,0 +1 @@ +../../../VERSION \ No newline at end of file From 57255b308ca042968788f95e9a823fd36005a259 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 18 Apr 2024 00:49:46 +0000 Subject: [PATCH 58/63] Add missing alpha specs to installs --- ci/test_wheel.sh | 2 +- ci/test_wheel_cugraph-dgl.sh | 2 +- ci/test_wheel_cugraph-equivariant.sh | 2 +- ci/test_wheel_cugraph-pyg.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index e1365ccc74..fc62588ad8 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -24,7 +24,7 @@ RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 c RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 python ./dist -python -m pip install "${python_package_name}-${RAPIDS_PY_CUDA_SUFFIX}[test]" --find-links ${librmm_wheelhouse} --find-links ${rmm_wheelhouse} --find-links ${libraft_wheelhouse} --find-links ${pylibraft_wheelhouse} --find-links ${libcugraphops_wheelhouse} --find-links ./dist +python -m pip install "${python_package_name}-${RAPIDS_PY_CUDA_SUFFIX}[test]>=0.0.0a0" --find-links ${librmm_wheelhouse} --find-links ${rmm_wheelhouse} --find-links ${libraft_wheelhouse} --find-links ${pylibraft_wheelhouse} --find-links ${libcugraphops_wheelhouse} --find-links ./dist # Run smoke tests for aarch64 pull requests arch=$(uname -m) diff --git a/ci/test_wheel_cugraph-dgl.sh b/ci/test_wheel_cugraph-dgl.sh index 0f2d4d058e..f54a4e12af 100755 --- a/ci/test_wheel_cugraph-dgl.sh +++ b/ci/test_wheel_cugraph-dgl.sh @@ -20,7 +20,7 @@ RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 c RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 python ./dist -python -m pip install "${python_package_name}-${RAPIDS_PY_CUDA_SUFFIX}[test]" --find-links ${librmm_wheelhouse} --find-links ${rmm_wheelhouse} --find-links ${libraft_wheelhouse} --find-links ${pylibraft_wheelhouse} --find-links ${libcugraphops_wheelhouse} --find-links ./dist +python -m pip install "${python_package_name}-${RAPIDS_PY_CUDA_SUFFIX}[test]>=0.0.0a0" --find-links ${librmm_wheelhouse} --find-links ${rmm_wheelhouse} --find-links ${libraft_wheelhouse} --find-links ${pylibraft_wheelhouse} --find-links ${libcugraphops_wheelhouse} --find-links ./dist PKG_CUDA_VER="$(echo ${CUDA_VERSION} | cut -d '.' -f1,2 | tr -d '.')" PKG_CUDA_VER_MAJOR=${PKG_CUDA_VER:0:2} diff --git a/ci/test_wheel_cugraph-equivariant.sh b/ci/test_wheel_cugraph-equivariant.sh index 1085d29aa0..798643088f 100755 --- a/ci/test_wheel_cugraph-equivariant.sh +++ b/ci/test_wheel_cugraph-equivariant.sh @@ -13,7 +13,7 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist -python -m pip install "${python_package_name}-${RAPIDS_PY_CUDA_SUFFIX}[test]" --find-links ./dist +python -m pip install "${python_package_name}-${RAPIDS_PY_CUDA_SUFFIX}[test]>=0.0.0a0" --find-links ./dist PKG_CUDA_VER="$(echo ${CUDA_VERSION} | cut -d '.' -f1,2 | tr -d '.')" PKG_CUDA_VER_MAJOR=${PKG_CUDA_VER:0:2} diff --git a/ci/test_wheel_cugraph-pyg.sh b/ci/test_wheel_cugraph-pyg.sh index 0d9f32917d..3a1cf4ee21 100755 --- a/ci/test_wheel_cugraph-pyg.sh +++ b/ci/test_wheel_cugraph-pyg.sh @@ -21,7 +21,7 @@ RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download- RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 python ./dist # pip creates wheels using python package names -python -m pip install "${python_package_name}-${RAPIDS_PY_CUDA_SUFFIX}[test]" --find-links ${librmm_wheelhouse} --find-links ${rmm_wheelhouse} --find-links ${libraft_wheelhouse} --find-links ${pylibraft_wheelhouse} --find-links ${libcugraphops_wheelhouse} --find-links ./dist +python -m pip install "${python_package_name}-${RAPIDS_PY_CUDA_SUFFIX}[test]>=0.0.0a0" --find-links ${librmm_wheelhouse} --find-links ${rmm_wheelhouse} --find-links ${libraft_wheelhouse} --find-links ${pylibraft_wheelhouse} --find-links ${libcugraphops_wheelhouse} --find-links ./dist # RAPIDS_DATASET_ROOT_DIR is used by test scripts export RAPIDS_DATASET_ROOT_DIR="$(realpath datasets)" From 9c169f3868e33955b6be94669cf06fd3ef21fee7 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 18 Apr 2024 00:50:00 +0000 Subject: [PATCH 59/63] Remove incorrect package names --- ci/test_wheel_cugraph-pyg.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/test_wheel_cugraph-pyg.sh b/ci/test_wheel_cugraph-pyg.sh index 3a1cf4ee21..e1c7223ab4 100755 --- a/ci/test_wheel_cugraph-pyg.sh +++ b/ci/test_wheel_cugraph-pyg.sh @@ -17,8 +17,8 @@ pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-ge libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp e7c6f06) RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist -RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist -RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 python ./dist +RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist +RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 python ./dist # pip creates wheels using python package names python -m pip install "${python_package_name}-${RAPIDS_PY_CUDA_SUFFIX}[test]>=0.0.0a0" --find-links ${librmm_wheelhouse} --find-links ${rmm_wheelhouse} --find-links ${libraft_wheelhouse} --find-links ${pylibraft_wheelhouse} --find-links ${libcugraphops_wheelhouse} --find-links ./dist From 779a0b87d0233105dc49f83e3c500f8dc9b9b848 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 18 Apr 2024 04:09:41 +0000 Subject: [PATCH 60/63] Fix cugraph-equivariant tests --- ci/test_wheel_cugraph-equivariant.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/test_wheel_cugraph-equivariant.sh b/ci/test_wheel_cugraph-equivariant.sh index 798643088f..4f716c2295 100755 --- a/ci/test_wheel_cugraph-equivariant.sh +++ b/ci/test_wheel_cugraph-equivariant.sh @@ -10,8 +10,8 @@ python_package_name=$(echo ${package_name}|sed 's/-/_/g') RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist -RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist +RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist +RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist python -m pip install "${python_package_name}-${RAPIDS_PY_CUDA_SUFFIX}[test]>=0.0.0a0" --find-links ./dist From 18ef0e0e480aa70eaac0895ccc1eb86f52ecb9fe Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 18 Apr 2024 04:26:25 +0000 Subject: [PATCH 61/63] Use a consistent raft commit --- ci/build_wheel_cpp.sh | 2 +- ci/build_wheel_python.sh | 4 ++-- ci/test_wheel.sh | 4 ++-- ci/test_wheel_cugraph-dgl.sh | 4 ++-- ci/test_wheel_cugraph-pyg.sh | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index 79cbb11bea..b16d8a2120 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -41,7 +41,7 @@ done cd "${package_dir}" librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) -libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp 0435e6b) +libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp efafdb6) libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp e7c6f06) python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check --find-links ${librmm_wheelhouse} --find-links ${libraft_wheelhouse} --find-links ${libcugraphops_wheelhouse} diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index ba7fa5f742..5150de027f 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -13,9 +13,9 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" CPP_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcugraph_dist) librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) -libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp 0435e6b) +libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp efafdb6) libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp e7c6f06) -pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 python) +pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 python efafdb6) PYTHON_WHEELHOUSE="${PWD}/dist/" PYTHON_PURE_WHEELHOUSE="${PWD}/pure_dist/" diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index fc62588ad8..ba7e59ae3d 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -16,8 +16,8 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) rmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 python) -libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp) -pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 python) +libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp efafdb6) +pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 python efafdb6) libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp e7c6f06) RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist diff --git a/ci/test_wheel_cugraph-dgl.sh b/ci/test_wheel_cugraph-dgl.sh index f54a4e12af..92bc107d8c 100755 --- a/ci/test_wheel_cugraph-dgl.sh +++ b/ci/test_wheel_cugraph-dgl.sh @@ -12,8 +12,8 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) rmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 python) -libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp) -pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 python) +libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp 0435e6b) +pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 python 0435e6b) libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp e7c6f06) RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist diff --git a/ci/test_wheel_cugraph-pyg.sh b/ci/test_wheel_cugraph-pyg.sh index e1c7223ab4..bca3a4e228 100755 --- a/ci/test_wheel_cugraph-pyg.sh +++ b/ci/test_wheel_cugraph-pyg.sh @@ -12,8 +12,8 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) rmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 python) -libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp) -pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 python) +libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp 0435e6b) +pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 python 0435e6b) libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp e7c6f06) RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist From fef50815450c3d9fbf35f4e4eaa807c961657bde Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 26 Jun 2024 16:01:46 -0500 Subject: [PATCH 62/63] update RAPIDS dependency versions to 24.8 --- dependencies.yaml | 8 ++++---- python/cugraph/pyproject.toml | 4 ++-- python/libcugraph/pyproject.toml | 12 ++++++------ python/pylibcugraph/pyproject.toml | 8 ++++---- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 91b6579aec..06867ca5a2 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -471,7 +471,7 @@ dependencies: common: - output_types: [pyproject, requirements] packages: - - libcugraph==24.6.* + - libcugraph==24.8.* libcugraph_build: common: - output_types: [pyproject, requirements] @@ -481,9 +481,9 @@ dependencies: common: - output_types: [requirements, pyproject] packages: - - librmm==24.6.* - - libraft==24.6.* - - libcugraphops==24.6.* + - librmm==24.8.* + - libraft==24.8.* + - libcugraphops==24.8.* python_run_cugraph: common: - output_types: [conda, pyproject] diff --git a/python/cugraph/pyproject.toml b/python/cugraph/pyproject.toml index 8a2b966cc5..2f0dbd5f11 100644 --- a/python/cugraph/pyproject.toml +++ b/python/cugraph/pyproject.toml @@ -5,7 +5,7 @@ requires = [ "cmake>=3.26.4", "cython>=3.0.0", - "libcugraph==24.6.*", + "libcugraph==24.8.*", "ninja", "pylibcugraph==24.8.*", "pylibraft==24.8.*", @@ -33,7 +33,7 @@ dependencies = [ "dask-cuda==24.8.*", "dask-cudf==24.8.*", "fsspec[http]>=0.6.0", - "libcugraph==24.6.*", + "libcugraph==24.8.*", "numba>=0.57", "numpy>=1.23,<2.0a0", "pylibcugraph==24.8.*", diff --git a/python/libcugraph/pyproject.toml b/python/libcugraph/pyproject.toml index 3aa048f0b6..cc266bb6fb 100644 --- a/python/libcugraph/pyproject.toml +++ b/python/libcugraph/pyproject.toml @@ -3,9 +3,9 @@ [build-system] requires = [ "cmake>=3.26.4", - "libcugraphops==24.6.*", - "libraft==24.6.*", - "librmm==24.6.*", + "libcugraphops==24.8.*", + "libraft==24.8.*", + "librmm==24.8.*", "ninja", "scikit-build-core[pyproject]>=0.7.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. @@ -22,9 +22,9 @@ authors = [ license = { text = "Apache 2.0" } requires-python = ">=3.9" dependencies = [ - "libcugraphops==24.6.*", - "libraft==24.6.*", - "librmm==24.6.*", + "libcugraphops==24.8.*", + "libraft==24.8.*", + "librmm==24.8.*", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ "Intended Audience :: Developers", diff --git a/python/pylibcugraph/pyproject.toml b/python/pylibcugraph/pyproject.toml index f5950f5414..9dd7a975ba 100644 --- a/python/pylibcugraph/pyproject.toml +++ b/python/pylibcugraph/pyproject.toml @@ -5,7 +5,7 @@ requires = [ "cmake>=3.26.4", "cython>=3.0.0", - "libcugraph==24.6.*", + "libcugraph==24.8.*", "ninja", "pylibraft==24.8.*", "rmm==24.8.*", @@ -28,9 +28,9 @@ license = { text = "Apache 2.0" } requires-python = ">=3.9" dependencies = [ "cupy-cuda11x>=12.0.0", - "libcugraph==24.6.*", - "pylibraft==24.6.*", - "rmm==24.6.*", + "libcugraph==24.8.*", + "pylibraft==24.8.*", + "rmm==24.8.*", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ "Intended Audience :: Developers", From 0da56113877631ecc9b181931e916c69f7f3e229 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Thu, 27 Jun 2024 19:09:53 -0500 Subject: [PATCH 63/63] remove rmm pr wheel artifact downloads --- ci/build_wheel_cpp.sh | 3 +-- ci/build_wheel_python.sh | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index b16d8a2120..ec6cb37a79 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -40,11 +40,10 @@ done cd "${package_dir}" -librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp efafdb6) libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp e7c6f06) -python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check --find-links ${librmm_wheelhouse} --find-links ${libraft_wheelhouse} --find-links ${libcugraphops_wheelhouse} +python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check --find-links ${libraft_wheelhouse} --find-links ${libcugraphops_wheelhouse} mkdir -p final_dist python -m auditwheel repair --exclude libraft.so --exclude libcugraph-ops++.so -w final_dist dist/* diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index 5150de027f..9fefc4d790 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -12,7 +12,6 @@ git_commit=$(git rev-parse HEAD) RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" CPP_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcugraph_dist) -librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) libraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 cpp efafdb6) libcugraphops_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cugraph-ops 629 cpp e7c6f06) pylibraft_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2264 python efafdb6) @@ -21,7 +20,7 @@ PYTHON_WHEELHOUSE="${PWD}/dist/" PYTHON_PURE_WHEELHOUSE="${PWD}/pure_dist/" PYTHON_AUDITED_WHEELHOUSE="${PWD}/final_dist/" -WHEELHOUSES=("${librmm_wheelhouse}" "${libraft_wheelhouse}" "${libcugraphops_wheelhouse}" "${pylibraft_wheelhouse}" "${CPP_WHEELHOUSE}" "${PYTHON_WHEELHOUSE}" "${PYTHON_PURE_WHEELHOUSE}") +WHEELHOUSES=("${libraft_wheelhouse}" "${libcugraphops_wheelhouse}" "${pylibraft_wheelhouse}" "${CPP_WHEELHOUSE}" "${PYTHON_WHEELHOUSE}" "${PYTHON_PURE_WHEELHOUSE}") mkdir -p "${PYTHON_AUDITED_WHEELHOUSE}" FIND_LINKS=""