From 07644ec0035abe45f9bc7cc54a90b18535c4edf2 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 30 Sep 2021 15:48:21 -0500 Subject: [PATCH 01/66] add option to build faiss shared libs --- cpp/CMakeLists.txt | 2 ++ cpp/cmake/thirdparty/get_faiss.cmake | 15 +++++++++++---- cpp/test/CMakeLists.txt | 1 - 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 7ab8a9f46e..cab85e8b9c 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -49,6 +49,8 @@ option(DISABLE_DEPRECATION_WARNINGS "Disable depreaction warnings " ON) option(DISABLE_OPENMP "Disable OpenMP" OFF) option(NVTX "Enable nvtx markers" OFF) +option(RAFT_USE_FAISS_STATIC "Build and statically link the FAISS library for nearest neighbors search on GPU" ON) + message(VERBOSE "RAFT: Build RAFT unit-tests: ${BUILD_TESTS}") message(VERBOSE "RAFT: Enable detection of conda environment for dependencies: ${DETECT_CONDA_ENV}") message(VERBOSE "RAFT: Disable depreaction warnings " ${DISABLE_DEPRECATION_WARNINGS}) diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake index a65401579c..6f4bac6a40 100644 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -15,7 +15,7 @@ #============================================================================= function(find_and_configure_faiss) - set(oneValueArgs VERSION PINNED_TAG) + set(oneValueArgs VERSION PINNED_TAG BUILD_STATIC_LIBS) cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} ) @@ -24,14 +24,20 @@ function(find_and_configure_faiss) LIBRARY_NAMES faiss ) + set(BUILD_SHARED_LIBS OFF) + if (NOT BUILD_STATIC_LIBS) + set(BUILD_SHARED_LIBS ON) + endif() + rapids_cpm_find(FAISS ${PKG_VERSION} - GLOBAL_TARGETS faiss + GLOBAL_TARGETS faiss + BUILD_EXPORT_SET raft-exports CPM_ARGS GIT_REPOSITORY https://github.com/facebookresearch/faiss.git GIT_TAG ${PKG_PINNED_TAG} OPTIONS "FAISS_ENABLE_PYTHON OFF" - "BUILD_SHARED_LIBS OFF" + "BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS}" "CUDAToolkit_ROOT ${CUDAToolkit_LIBRARY_DIR}" "FAISS_ENABLE_GPU ON" "BUILD_TESTING OFF" @@ -39,7 +45,7 @@ function(find_and_configure_faiss) ) if(FAISS_ADDED) - set(FAISS_GPU_HEADERS ${FAISS_SOURCE_DIR} PARENT_SCOPE) + target_include_directories(faiss INTERFACE $) endif() if(TARGET faiss AND NOT TARGET FAISS::FAISS) @@ -50,4 +56,5 @@ endfunction() find_and_configure_faiss(VERSION 1.7.0 PINNED_TAG bde7c0027191f29c9dadafe4f6e68ca0ee31fb30 + BUILD_STATIC_LIBS ${RAFT_USE_FAISS_STATIC} ) diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index 43e1c65695..6040aca26c 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -117,7 +117,6 @@ target_compile_options(test_raft target_include_directories(test_raft PUBLIC "$" "$" - "${FAISS_GPU_HEADERS}" ) From f3e1f0927e795d12440fa5188f8484a7e5dba27d Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 30 Sep 2021 16:25:41 -0500 Subject: [PATCH 02/66] add FAISS::FAISS to the list of global targets --- cpp/cmake/thirdparty/get_faiss.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake index 6f4bac6a40..1d0c562cfd 100644 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -30,7 +30,7 @@ function(find_and_configure_faiss) endif() rapids_cpm_find(FAISS ${PKG_VERSION} - GLOBAL_TARGETS faiss + GLOBAL_TARGETS faiss FAISS::FAISS BUILD_EXPORT_SET raft-exports CPM_ARGS GIT_REPOSITORY https://github.com/facebookresearch/faiss.git From e0386442802093398d790dd2818aeb9331d339bb Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 30 Sep 2021 17:26:37 -0500 Subject: [PATCH 03/66] update cuco --- cpp/cmake/thirdparty/get_cuco.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/cmake/thirdparty/get_cuco.cmake b/cpp/cmake/thirdparty/get_cuco.cmake index 2ee7a1c384..33e28ff622 100644 --- a/cpp/cmake/thirdparty/get_cuco.cmake +++ b/cpp/cmake/thirdparty/get_cuco.cmake @@ -22,7 +22,7 @@ function(find_and_configure_cuco VERSION) INSTALL_EXPORT_SET raft-exports CPM_ARGS GIT_REPOSITORY https://github.com/NVIDIA/cuCollections.git - GIT_TAG a3c85ee2ea25ddfdd4596c6b9d546f7c7590743f + GIT_TAG 729857a5698a0e8d8f812e0464f65f37854ae17b OPTIONS "BUILD_TESTS OFF" "BUILD_BENCHMARKS OFF" "BUILD_EXAMPLES OFF" From 3fa9c94860d1dc4a3693666c0affb5879ae9f0a4 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 30 Sep 2021 17:58:53 -0500 Subject: [PATCH 04/66] create a faissTargets export-set --- cpp/cmake/thirdparty/get_faiss.cmake | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake index 1d0c562cfd..4e04592b7e 100644 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -52,6 +52,8 @@ function(find_and_configure_faiss) add_library(FAISS::FAISS ALIAS faiss) endif() + rapids_export(BUILD faiss EXPORT_SET faissTargets) + endfunction() find_and_configure_faiss(VERSION 1.7.0 From 85572d097dfb0a4294e6d497c4e432c42c0161cf Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 30 Sep 2021 18:10:45 -0500 Subject: [PATCH 05/66] add faiss to raft-exports --- cpp/cmake/thirdparty/get_faiss.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake index 4e04592b7e..f9d286668a 100644 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -52,7 +52,7 @@ function(find_and_configure_faiss) add_library(FAISS::FAISS ALIAS faiss) endif() - rapids_export(BUILD faiss EXPORT_SET faissTargets) + rapids_export(BUILD faiss EXPORT_SET raft-exports) endfunction() From b19a4b6c5c059ac7f3c95c99897e40f00736d7f8 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 30 Sep 2021 18:14:38 -0500 Subject: [PATCH 06/66] revert --- cpp/cmake/thirdparty/get_faiss.cmake | 2 -- 1 file changed, 2 deletions(-) diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake index 4e04592b7e..1d0c562cfd 100644 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -52,8 +52,6 @@ function(find_and_configure_faiss) add_library(FAISS::FAISS ALIAS faiss) endif() - rapids_export(BUILD faiss EXPORT_SET faissTargets) - endfunction() find_and_configure_faiss(VERSION 1.7.0 From 14785ff90fae3aca8b80d7ab85f9752fc19d95ce Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 30 Sep 2021 18:28:44 -0500 Subject: [PATCH 07/66] create a faiss-exports export-set --- cpp/cmake/thirdparty/get_faiss.cmake | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake index 1d0c562cfd..ef552c121a 100644 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -46,6 +46,11 @@ function(find_and_configure_faiss) if(FAISS_ADDED) target_include_directories(faiss INTERFACE $) + rapids_export(BUILD faiss + EXPORT_SET faiss-exports + GLOBAL_TARGETS faiss + NAMESPACE raft:: + ) endif() if(TARGET faiss AND NOT TARGET FAISS::FAISS) From 25b62e6986b07e65687aaec95120cb1824e5a0db Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 30 Sep 2021 18:33:01 -0500 Subject: [PATCH 08/66] mark faiss to be found as part of resolving raft-exports dependencies --- cpp/cmake/thirdparty/get_faiss.cmake | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake index ef552c121a..2c70756b15 100644 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -57,6 +57,8 @@ function(find_and_configure_faiss) add_library(FAISS::FAISS ALIAS faiss) endif() + rapids_export_package(BUILD faiss cugraph-exports) + endfunction() find_and_configure_faiss(VERSION 1.7.0 From 76edf367c80d4cca7febce5c4a19fea0b7da5aa8 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 30 Sep 2021 18:45:03 -0500 Subject: [PATCH 09/66] create faiss-exports export set --- cpp/cmake/thirdparty/get_faiss.cmake | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake index 2c70756b15..8a8ed54859 100644 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -44,21 +44,20 @@ function(find_and_configure_faiss) "CMAKE_MESSAGE_LOG_LEVEL VERBOSE" ) + if(TARGET faiss AND NOT TARGET FAISS::FAISS) + add_library(FAISS::FAISS ALIAS faiss) + endif() + if(FAISS_ADDED) target_include_directories(faiss INTERFACE $) + install(TARGETS faiss EXPORT faiss-exports) rapids_export(BUILD faiss EXPORT_SET faiss-exports GLOBAL_TARGETS faiss NAMESPACE raft:: - ) + LANGUAGES CUDA) endif() - if(TARGET faiss AND NOT TARGET FAISS::FAISS) - add_library(FAISS::FAISS ALIAS faiss) - endif() - - rapids_export_package(BUILD faiss cugraph-exports) - endfunction() find_and_configure_faiss(VERSION 1.7.0 From 92851b3acbdd96170653da005e5477605abaac90 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 30 Sep 2021 18:45:03 -0500 Subject: [PATCH 10/66] create faiss-exports export set --- cpp/cmake/thirdparty/get_faiss.cmake | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake index 1d0c562cfd..43555cc11b 100644 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -46,6 +46,12 @@ function(find_and_configure_faiss) if(FAISS_ADDED) target_include_directories(faiss INTERFACE $) + install(TARGETS faiss EXPORT faiss-exports) + rapids_export(BUILD faiss + EXPORT_SET faiss-exports + GLOBAL_TARGETS faiss + NAMESPACE raft:: + LANGUAGES CUDA) endif() if(TARGET faiss AND NOT TARGET FAISS::FAISS) From 8e266ef3577a317b53a9b1d84c2e9ba54b12636d Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 5 Oct 2021 17:05:48 -0500 Subject: [PATCH 11/66] export faiss::faiss target to build export set --- cpp/CMakeLists.txt | 12 +++++++++++- cpp/cmake/thirdparty/get_faiss.cmake | 28 +++++++++++++++------------- cpp/test/CMakeLists.txt | 2 +- 3 files changed, 27 insertions(+), 15 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index cab85e8b9c..ce51813000 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -105,9 +105,9 @@ rapids_cpm_init() include(cmake/thirdparty/get_thrust.cmake) include(cmake/thirdparty/get_rmm.cmake) include(cmake/thirdparty/get_cuco.cmake) +include(cmake/thirdparty/get_faiss.cmake) if(BUILD_TESTS) - include(cmake/thirdparty/get_faiss.cmake) include(cmake/thirdparty/get_gtest.cmake) include(cmake/thirdparty/get_nccl.cmake) include(cmake/thirdparty/get_ucx.cmake) @@ -163,11 +163,20 @@ and common functions for the analytics components of RAPIDS. ]=]) +set(raft_code_string +[=[ +if (TARGET faiss::faiss AND (NOT TARGET faiss)) + add_library(faiss ALIAS faiss::faiss) +endif() +]=] +) + rapids_export(INSTALL raft EXPORT_SET raft-exports GLOBAL_TARGETS raft # since we can't hook into EXPORT SETS NAMESPACE raft:: DOCUMENTATION doc_string + FINAL_CODE_BLOCK raft_code_string ) ############################################################################## @@ -179,6 +188,7 @@ rapids_export(BUILD raft LANGUAGES CUDA DOCUMENTATION doc_string NAMESPACE raft:: + FINAL_CODE_BLOCK raft_code_string ) ############################################################################## diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake index 8a8ed54859..b82095bc00 100644 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -29,35 +29,37 @@ function(find_and_configure_faiss) set(BUILD_SHARED_LIBS ON) endif() - rapids_cpm_find(FAISS ${PKG_VERSION} - GLOBAL_TARGETS faiss FAISS::FAISS - BUILD_EXPORT_SET raft-exports + rapids_cpm_find(faiss ${PKG_VERSION} + GLOBAL_TARGETS faiss::faiss + INSTALL_EXPORT_SET raft-exports CPM_ARGS GIT_REPOSITORY https://github.com/facebookresearch/faiss.git GIT_TAG ${PKG_PINNED_TAG} OPTIONS "FAISS_ENABLE_PYTHON OFF" - "BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS}" "CUDAToolkit_ROOT ${CUDAToolkit_LIBRARY_DIR}" "FAISS_ENABLE_GPU ON" "BUILD_TESTING OFF" "CMAKE_MESSAGE_LOG_LEVEL VERBOSE" ) - if(TARGET faiss AND NOT TARGET FAISS::FAISS) - add_library(FAISS::FAISS ALIAS faiss) + if(TARGET faiss AND NOT TARGET faiss::faiss) + add_library(faiss::faiss ALIAS faiss) endif() - if(FAISS_ADDED) - target_include_directories(faiss INTERFACE $) - install(TARGETS faiss EXPORT faiss-exports) + if(faiss_ADDED) rapids_export(BUILD faiss - EXPORT_SET faiss-exports - GLOBAL_TARGETS faiss - NAMESPACE raft:: - LANGUAGES CUDA) + EXPORT_SET faiss-targets + GLOBAL_TARGETS faiss::faiss + NAMESPACE faiss::) endif() + # We generate the faiss-config files when we built faiss locally, so always do `find_dependency` + rapids_export_package(BUILD faiss raft-exports) + + # Tell cmake where it can find the generated faiss-config.cmake we wrote. + include("${rapids-cmake-dir}/export/find_package_root.cmake") + rapids_export_find_package_root(BUILD faiss [=[${CMAKE_CURRENT_LIST_DIR}]=] raft-exports) endfunction() find_and_configure_faiss(VERSION 1.7.0 diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index 6040aca26c..5761f21735 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -129,7 +129,7 @@ PRIVATE CUDA::cusparse rmm::rmm cuco::cuco - FAISS::FAISS + faiss::faiss GTest::gtest GTest::gtest_main Threads::Threads From 83e3aa682935ac145bb86458b833c310f73239ac Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 6 Oct 2021 10:10:46 -0500 Subject: [PATCH 12/66] fix faiss GLOBAL_TARGETS --- cpp/cmake/thirdparty/get_faiss.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake index b82095bc00..ff16809824 100644 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -50,7 +50,7 @@ function(find_and_configure_faiss) if(faiss_ADDED) rapids_export(BUILD faiss EXPORT_SET faiss-targets - GLOBAL_TARGETS faiss::faiss + GLOBAL_TARGETS faiss NAMESPACE faiss::) endif() From 2773edc97601c64b6f65633547492834c15acc66 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 6 Oct 2021 13:25:55 -0500 Subject: [PATCH 13/66] update rapids-cmake version --- cpp/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 2e4c1b6ad7..2715f3a71f 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -15,7 +15,7 @@ #============================================================================= cmake_minimum_required(VERSION 3.20.1 FATAL_ERROR) -file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-21.10/RAPIDS.cmake +file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-21.12/RAPIDS.cmake ${CMAKE_BINARY_DIR}/RAPIDS.cmake) include(${CMAKE_BINARY_DIR}/RAPIDS.cmake) include(rapids-cmake) From 3ff1c4d0366a3115871189139f1cd29d42d641c2 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 6 Oct 2021 18:58:14 -0500 Subject: [PATCH 14/66] fix naming for generated Findfaiss.cmake module so we find conda-installed faiss --- cpp/cmake/thirdparty/get_faiss.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake index ff16809824..6e0ed338d4 100644 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -19,7 +19,7 @@ function(find_and_configure_faiss) cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} ) - rapids_find_generate_module(FAISS + rapids_find_generate_module(faiss HEADER_NAMES faiss/IndexFlat.h LIBRARY_NAMES faiss ) From cbb6f2d2626a4d11782f14791845aee091b65aec Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 6 Oct 2021 21:53:09 -0500 Subject: [PATCH 15/66] ensure faiss::faiss target is available in build and install export sets --- cpp/CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 2715f3a71f..13de640c08 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -165,7 +165,9 @@ and common functions for the analytics components of RAPIDS. set(raft_code_string [=[ -if (TARGET faiss::faiss AND (NOT TARGET faiss)) +if(TARGET faiss AND (NOT TARGET faiss::faiss)) + add_library(faiss::faiss ALIAS faiss) +elseif(TARGET faiss::faiss AND (NOT TARGET faiss)) add_library(faiss ALIAS faiss::faiss) endif() ]=] From 713a872f8d2b598ed8e80700f2c7490f95848d29 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 6 Oct 2021 23:45:09 -0500 Subject: [PATCH 16/66] add gtest to raft-exports, remove dead code --- cpp/CMakeLists.txt | 5 +++-- cpp/cmake/thirdparty/get_gtest.cmake | 14 +++++++++++++- cpp/cmake/thirdparty/get_thrust.cmake | 27 --------------------------- cpp/test/CMakeLists.txt | 9 ++------- 4 files changed, 18 insertions(+), 37 deletions(-) delete mode 100644 cpp/cmake/thirdparty/get_thrust.cmake diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 13de640c08..f512df4053 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -102,13 +102,14 @@ endif() # add third party dependencies using CPM rapids_cpm_init() -include(cmake/thirdparty/get_thrust.cmake) include(cmake/thirdparty/get_rmm.cmake) include(cmake/thirdparty/get_cuco.cmake) +# CPMFindPackage(faiss) so it's available for raft consumers to use include(cmake/thirdparty/get_faiss.cmake) +# CPMFindPackage(GTest) so it's available for raft consumers to use +include(cmake/thirdparty/get_gtest.cmake) if(BUILD_TESTS) - include(cmake/thirdparty/get_gtest.cmake) include(cmake/thirdparty/get_nccl.cmake) include(cmake/thirdparty/get_ucx.cmake) endif() diff --git a/cpp/cmake/thirdparty/get_gtest.cmake b/cpp/cmake/thirdparty/get_gtest.cmake index 7c234283d5..2354d3beaa 100644 --- a/cpp/cmake/thirdparty/get_gtest.cmake +++ b/cpp/cmake/thirdparty/get_gtest.cmake @@ -17,7 +17,19 @@ function(find_and_configure_gtest ) include(${rapids-cmake-dir}/cpm/gtest.cmake) - rapids_cpm_gtest() + rapids_cpm_gtest(BUILD_EXPORT_SET raft-exports + INSTALL_EXPORT_SET raft-exports) + + if(GTest_ADDED) + rapids_export(BUILD GTest + VERSION ${GTest_VERSION} + EXPORT_SET GTestTargets + GLOBAL_TARGETS gtest gmock gtest_main gmock_main + NAMESPACE GTest::) + + include("${rapids-cmake-dir}/export/find_package_root.cmake") + rapids_export_find_package_root(BUILD GTest [=[${CMAKE_CURRENT_LIST_DIR}]=] raft-exports) + endif() endfunction() diff --git a/cpp/cmake/thirdparty/get_thrust.cmake b/cpp/cmake/thirdparty/get_thrust.cmake deleted file mode 100644 index c28ff6e66d..0000000000 --- a/cpp/cmake/thirdparty/get_thrust.cmake +++ /dev/null @@ -1,27 +0,0 @@ -# ============================================================================= -# Copyright (c) 2021, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. -# ============================================================================= - -# Use CPM to find or clone thrust -function(find_and_configure_thrust) - include(${rapids-cmake-dir}/cpm/thrust.cmake) - - rapids_cpm_thrust( - NAMESPACE raft - BUILD_EXPORT_SET raft-exports - INSTALL_EXPORT_SET raft-exports - ) - -endfunction() - -find_and_configure_thrust() diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index 5761f21735..999644361a 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -122,13 +122,8 @@ target_include_directories(test_raft target_link_libraries(test_raft PRIVATE - CUDA::cublas - CUDA::curand - CUDA::cusolver - CUDA::cudart - CUDA::cusparse - rmm::rmm - cuco::cuco + raft::raft + NCCL::NCCL faiss::faiss GTest::gtest GTest::gtest_main From ceb4d8b9eb6b06a1d7832bbb77cc7bd8799f0777 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 12 Oct 2021 08:47:06 -0500 Subject: [PATCH 17/66] update CMake faiss variable name --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index 7dff773b67..ecdf216aef 100755 --- a/build.sh +++ b/build.sh @@ -147,7 +147,7 @@ if (( ${NUMARGS} == 0 )) || hasArg cppraft; then -DNVTX=${NVTX} \ -DDISABLE_DEPRECATION_WARNING=${BUILD_DISABLE_DEPRECATION_WARNING} \ -DBUILD_GTEST=${BUILD_GTEST} \ - -DBUILD_STATIC_FAISS=${BUILD_STATIC_FAISS} + -DRAFT_USE_FAISS_STATIC=${BUILD_STATIC_FAISS} # Run all c++ targets at once From 1d3ffab3443be5116c69da59e65910899b3357cc Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 12 Oct 2021 09:05:56 -0500 Subject: [PATCH 18/66] make -v set cmake --log-level=VERBOSE --- build.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build.sh b/build.sh index ecdf216aef..5dc73ce417 100755 --- a/build.sh +++ b/build.sh @@ -42,6 +42,7 @@ PYTHON_DEPS_CLONE=${REPODIR}/python/external_repositories BUILD_DIRS="${CPP_RAFT_BUILD_DIR} ${PY_RAFT_BUILD_DIR} ${PYTHON_DEPS_CLONE}" # Set defaults for vars modified by flags to this script +CMAKE_LOG_LEVEL="" VERBOSE_FLAG="" BUILD_ALL_GPU_ARCH=0 BUILD_GTEST=OFF @@ -83,6 +84,7 @@ fi # Process flags if hasArg -v; then VERBOSE_FLAG=-v + CMAKE_LOG_LEVEL="--log-level=VERBOSE" set -x fi if hasArg -g; then @@ -141,7 +143,7 @@ if (( ${NUMARGS} == 0 )) || hasArg cppraft; then echo "Building for *ALL* supported GPU architectures..." fi - cmake -S ${REPODIR}/cpp -B ${CPP_RAFT_BUILD_DIR} \ + cmake -S ${REPODIR}/cpp -B ${CPP_RAFT_BUILD_DIR} ${CMAKE_LOG_LEVEL} \ -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ -DCMAKE_CUDA_ARCHITECTURES=${RAFT_CMAKE_CUDA_ARCHITECTURES} \ -DNVTX=${NVTX} \ From b48e39384a0b199631c665c4565e0518f3c14c8e Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 13 Oct 2021 16:17:07 -0500 Subject: [PATCH 19/66] update cuco hash to the commit that uses rapids-cmake v21.12 --- cpp/cmake/thirdparty/get_cuco.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/cmake/thirdparty/get_cuco.cmake b/cpp/cmake/thirdparty/get_cuco.cmake index 33e28ff622..623ce06466 100644 --- a/cpp/cmake/thirdparty/get_cuco.cmake +++ b/cpp/cmake/thirdparty/get_cuco.cmake @@ -22,7 +22,7 @@ function(find_and_configure_cuco VERSION) INSTALL_EXPORT_SET raft-exports CPM_ARGS GIT_REPOSITORY https://github.com/NVIDIA/cuCollections.git - GIT_TAG 729857a5698a0e8d8f812e0464f65f37854ae17b + GIT_TAG 23c9a1fdbc06c29687c54016f6bb0b503535032c OPTIONS "BUILD_TESTS OFF" "BUILD_BENCHMARKS OFF" "BUILD_EXAMPLES OFF" From aedc6d75ee6e5cb796e4061b5a0a7f6e44f4865f Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 19 Nov 2021 15:00:49 -0600 Subject: [PATCH 20/66] update cuco version --- cpp/cmake/thirdparty/get_cuco.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cpp/cmake/thirdparty/get_cuco.cmake b/cpp/cmake/thirdparty/get_cuco.cmake index a0c0faf0a9..fdac7cda11 100644 --- a/cpp/cmake/thirdparty/get_cuco.cmake +++ b/cpp/cmake/thirdparty/get_cuco.cmake @@ -30,4 +30,5 @@ function(find_and_configure_cuco VERSION) endfunction() -find_and_configure_cuco(0.0.1) +# cuCollections doesn't have a version yet +find_and_configure_cuco(0.0) From bb4b7c78a88a589ca6faa6892dd92ec6fe94f810 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 1 Dec 2021 17:42:40 -0600 Subject: [PATCH 21/66] move faiss into separate raft-faiss-exports export set, only include faiss if it's in the raft_FIND_COMPONENTS list --- cpp/CMakeLists.txt | 40 +++++++++++++++++++++++++--- cpp/cmake/thirdparty/get_faiss.cmake | 6 ++--- 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 920be0110a..4f61b7b49e 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -154,6 +154,12 @@ install(FILES include/raft.hpp DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/raft ) +include("${rapids-cmake-dir}/export/write_dependencies.cmake") +rapids_export_write_dependencies( + INSTALL raft-faiss-exports + "${PROJECT_BINARY_DIR}/rapids-cmake/raft/export/raft-faiss-dependencies.cmake" +) + ############################################################################## # - install export ----------------------------------------------------------- set(doc_string @@ -165,7 +171,7 @@ and common functions for the analytics components of RAPIDS. ]=]) -set(raft_code_string +set(common_code_string [=[ if(TARGET faiss AND (NOT TARGET faiss::faiss)) add_library(faiss::faiss ALIAS faiss) @@ -175,26 +181,52 @@ endif() ]=] ) - rapids_export(INSTALL raft +set(install_code_string +[=[ +if(faiss IN_LIST raft_FIND_COMPONENTS) + if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/raft-faiss-dependencies.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/raft-faiss-dependencies.cmake") + endif() +endif() +]=] +) + +string(APPEND install_code_string ${common_code_string}) + +rapids_export(INSTALL raft EXPORT_SET raft-exports GLOBAL_TARGETS raft # since we can't hook into EXPORT SETS NAMESPACE raft:: DOCUMENTATION doc_string - FINAL_CODE_BLOCK raft_code_string + FINAL_CODE_BLOCK install_code_string ) ############################################################################## # - build export ------------------------------------------------------------- +set(build_code_string +[=[ +if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/raft-faiss-dependencies.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/raft-faiss-dependencies.cmake") +endif() +]=] +) + +string(APPEND build_code_string ${common_code_string}) + rapids_export(BUILD raft EXPORT_SET raft-exports GLOBAL_TARGETS raft # since we can't hook into EXPORT SETS LANGUAGES CUDA DOCUMENTATION doc_string NAMESPACE raft:: - FINAL_CODE_BLOCK raft_code_string + FINAL_CODE_BLOCK build_code_string ) +rapids_export_write_dependencies( + BUILD raft-faiss-exports "${PROJECT_BINARY_DIR}/raft-faiss-dependencies.cmake" +) + ############################################################################## # - build test executable ---------------------------------------------------- diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake index 6e0ed338d4..1c1052a5c3 100644 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -31,7 +31,7 @@ function(find_and_configure_faiss) rapids_cpm_find(faiss ${PKG_VERSION} GLOBAL_TARGETS faiss::faiss - INSTALL_EXPORT_SET raft-exports + INSTALL_EXPORT_SET raft-faiss-exports CPM_ARGS GIT_REPOSITORY https://github.com/facebookresearch/faiss.git GIT_TAG ${PKG_PINNED_TAG} @@ -55,11 +55,11 @@ function(find_and_configure_faiss) endif() # We generate the faiss-config files when we built faiss locally, so always do `find_dependency` - rapids_export_package(BUILD faiss raft-exports) + rapids_export_package(BUILD faiss raft-faiss-exports) # Tell cmake where it can find the generated faiss-config.cmake we wrote. include("${rapids-cmake-dir}/export/find_package_root.cmake") - rapids_export_find_package_root(BUILD faiss [=[${CMAKE_CURRENT_LIST_DIR}]=] raft-exports) + rapids_export_find_package_root(BUILD faiss [=[${CMAKE_CURRENT_LIST_DIR}]=] raft-faiss-exports) endfunction() find_and_configure_faiss(VERSION 1.7.0 From e798ab1403711a8b9e619386ed1a2468008bd254 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 1 Dec 2021 18:07:46 -0600 Subject: [PATCH 22/66] check for faiss in raft_FIND_COMPONENTS for build side export set too --- cpp/CMakeLists.txt | 33 ++++++++------------------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 4f61b7b49e..2e6b20053c 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -171,56 +171,39 @@ and common functions for the analytics components of RAPIDS. ]=]) -set(common_code_string -[=[ -if(TARGET faiss AND (NOT TARGET faiss::faiss)) - add_library(faiss::faiss ALIAS faiss) -elseif(TARGET faiss::faiss AND (NOT TARGET faiss)) - add_library(faiss ALIAS faiss::faiss) -endif() -]=] -) - -set(install_code_string +set(code_string [=[ if(faiss IN_LIST raft_FIND_COMPONENTS) if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/raft-faiss-dependencies.cmake") include("${CMAKE_CURRENT_LIST_DIR}/raft-faiss-dependencies.cmake") endif() + if(TARGET faiss AND (NOT TARGET faiss::faiss)) + add_library(faiss::faiss ALIAS faiss) + elseif(TARGET faiss::faiss AND (NOT TARGET faiss)) + add_library(faiss ALIAS faiss::faiss) + endif() endif() ]=] ) -string(APPEND install_code_string ${common_code_string}) - rapids_export(INSTALL raft EXPORT_SET raft-exports GLOBAL_TARGETS raft # since we can't hook into EXPORT SETS NAMESPACE raft:: DOCUMENTATION doc_string - FINAL_CODE_BLOCK install_code_string + FINAL_CODE_BLOCK code_string ) ############################################################################## # - build export ------------------------------------------------------------- -set(build_code_string -[=[ -if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/raft-faiss-dependencies.cmake") - include("${CMAKE_CURRENT_LIST_DIR}/raft-faiss-dependencies.cmake") -endif() -]=] -) - -string(APPEND build_code_string ${common_code_string}) - rapids_export(BUILD raft EXPORT_SET raft-exports GLOBAL_TARGETS raft # since we can't hook into EXPORT SETS LANGUAGES CUDA DOCUMENTATION doc_string NAMESPACE raft:: - FINAL_CODE_BLOCK build_code_string + FINAL_CODE_BLOCK code_string ) rapids_export_write_dependencies( From 2d5898d9983afa3fc513b31520da6161e842c90b Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 2 Dec 2021 14:27:02 -0600 Subject: [PATCH 23/66] pass EXCLUDE_FROM_ALL in get_faiss.cmake --- cpp/cmake/thirdparty/get_faiss.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake index 1c1052a5c3..506f393c32 100644 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -35,6 +35,7 @@ function(find_and_configure_faiss) CPM_ARGS GIT_REPOSITORY https://github.com/facebookresearch/faiss.git GIT_TAG ${PKG_PINNED_TAG} + EXCLUDE_FROM_ALL OPTIONS "FAISS_ENABLE_PYTHON OFF" "CUDAToolkit_ROOT ${CUDAToolkit_LIBRARY_DIR}" From 2cb3c3a51f594051f8c0d5651e8cd5aa74e3fca4 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 2 Dec 2021 14:33:06 -0600 Subject: [PATCH 24/66] enable CUDA language in code_string --- cpp/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 2e6b20053c..27ded711e7 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -174,6 +174,7 @@ and common functions for the analytics components of RAPIDS. set(code_string [=[ if(faiss IN_LIST raft_FIND_COMPONENTS) + enable_language(CUDA) if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/raft-faiss-dependencies.cmake") include("${CMAKE_CURRENT_LIST_DIR}/raft-faiss-dependencies.cmake") endif() From 23912f5d4a8d437e579fe32bc3f413c5edd07b75 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 2 Dec 2021 14:38:01 -0600 Subject: [PATCH 25/66] pass TRUE --- cpp/cmake/thirdparty/get_faiss.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake index 506f393c32..0427523f27 100644 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -33,9 +33,9 @@ function(find_and_configure_faiss) GLOBAL_TARGETS faiss::faiss INSTALL_EXPORT_SET raft-faiss-exports CPM_ARGS - GIT_REPOSITORY https://github.com/facebookresearch/faiss.git - GIT_TAG ${PKG_PINNED_TAG} - EXCLUDE_FROM_ALL + GIT_REPOSITORY https://github.com/facebookresearch/faiss.git + GIT_TAG ${PKG_PINNED_TAG} + EXCLUDE_FROM_ALL TRUE OPTIONS "FAISS_ENABLE_PYTHON OFF" "CUDAToolkit_ROOT ${CUDAToolkit_LIBRARY_DIR}" From fc306f6dfa14da87b7edc5a7dab7a53d6494a9a7 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 2 Dec 2021 14:47:45 -0600 Subject: [PATCH 26/66] drop CUDA from the list of global languages --- cpp/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 27ded711e7..fd19706061 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -201,7 +201,6 @@ rapids_export(INSTALL raft rapids_export(BUILD raft EXPORT_SET raft-exports GLOBAL_TARGETS raft # since we can't hook into EXPORT SETS - LANGUAGES CUDA DOCUMENTATION doc_string NAMESPACE raft:: FINAL_CODE_BLOCK code_string From 61c263af1c3d96567ee30f39bf17fadf37b48955 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 2 Dec 2021 15:40:36 -0600 Subject: [PATCH 27/66] link raft to faiss if faiss component is requested --- cpp/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index fd19706061..147fab2913 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -183,6 +183,7 @@ if(faiss IN_LIST raft_FIND_COMPONENTS) elseif(TARGET faiss::faiss AND (NOT TARGET faiss)) add_library(faiss ALIAS faiss::faiss) endif() + target_link_libraries(raft INTERFACE faiss::faiss) endif() ]=] ) From 41bd51b944764d35dc13f013323a77ea557ca4f5 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 8 Dec 2021 15:35:45 -0600 Subject: [PATCH 28/66] link raft to faiss if faiss component requested --- cpp/CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 147fab2913..01d0fdc830 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -183,7 +183,9 @@ if(faiss IN_LIST raft_FIND_COMPONENTS) elseif(TARGET faiss::faiss AND (NOT TARGET faiss)) add_library(faiss ALIAS faiss::faiss) endif() - target_link_libraries(raft INTERFACE faiss::faiss) + if(TARGET raft::raft) + target_link_libraries(raft::raft INTERFACE faiss::faiss) + endif() endif() ]=] ) From ce8a00592387859005d8e0bc6c736256a08106ed Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 15 Dec 2021 18:14:52 -0600 Subject: [PATCH 29/66] fix FAISS::FAISS -> faiss::faiss --- cpp/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 0bf79f8d0f..1b6af1113b 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -153,7 +153,7 @@ set(RAFT_LINK_LIBRARIES target_link_libraries(raft INTERFACE ${RAFT_LINK_LIBRARIES}) target_link_libraries(raft_distance PUBLIC ${RAFT_LINK_LIBRARIES}) -target_link_libraries(raft_nn PUBLIC ${RAFT_LINK_LIBRARIES} FAISS::FAISS) +target_link_libraries(raft_nn PUBLIC ${RAFT_LINK_LIBRARIES} faiss::faiss) target_compile_options(raft_distance PRIVATE "$<$:${RAFT_CXX_FLAGS}>" From c39685a7aab08eb80c172ebf1612a7d898f9af55 Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Mon, 20 Dec 2021 09:33:33 -0500 Subject: [PATCH 30/66] Simplify raft component CMake logic, and allow compilation without FAISS --- cpp/CMakeLists.txt | 95 +++++++++++++++++++++++----------------------- 1 file changed, 48 insertions(+), 47 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index efebfff429..1cd3ab55d5 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -114,31 +114,12 @@ if(BUILD_TESTS) endif() ############################################################################## -# - install targets----------------------------------------------------------- -rapids_cmake_install_lib_dir( lib_dir ) - -include(CPack) - -file(GLOB_RECURSE RAFT_DISTANCE_SOURCES "src/distance/specializations/*.cu") -file(GLOB_RECURSE RAFT_NN_SOURCES "src/nn/specializations/*.cu" ) - -add_library(raft_distance SHARED ${RAFT_DISTANCE_SOURCES}) -add_library(raft::raft_distance ALIAS raft_distance) - -add_library(raft_nn SHARED ${RAFT_NN_SOURCES}) -add_library(raft::raft_nn ALIAS raft_nn) +# - raft --------------------------------------------------------------------- add_library(raft INTERFACE) add_library(raft::raft ALIAS raft) target_include_directories(raft INTERFACE "$" "$") - -target_include_directories(raft_distance PUBLIC "$" - "$") - -target_include_directories(raft_nn PUBLIC "$" - "$") - set(RAFT_LINK_LIBRARIES CUDA::cublas CUDA::curand @@ -149,47 +130,67 @@ set(RAFT_LINK_LIBRARIES rmm::rmm cuco::cuco ) - target_link_libraries(raft INTERFACE ${RAFT_LINK_LIBRARIES}) -target_link_libraries(raft_distance PUBLIC ${RAFT_LINK_LIBRARIES}) -target_link_libraries(raft_nn PUBLIC ${RAFT_LINK_LIBRARIES} FAISS::FAISS) set(RAFT_COMPILE_DEFINITIONS $<$:NVTX_ENABLED> ) - target_compile_definitions(raft INTERFACE ${RAFT_COMPILE_DEFINITIONS}) -target_compile_definitions(raft_distance PRIVATE ${RAFT_COMPILE_DEFINITIONS}) -target_compile_definitions(raft_nn PRIVATE ${RAFT_COMPILE_DEFINITIONS}) - -target_compile_options(raft_distance - PRIVATE "$<$:${RAFT_CXX_FLAGS}>" - "$<$:${RAFT_CUDA_FLAGS}>" - ) - +target_compile_features(raft INTERFACE cxx_std_17 $) -target_compile_options(raft_nn - PRIVATE "$<$:${RAFT_CXX_FLAGS}>" - "$<$:${RAFT_CUDA_FLAGS}>" - ) +############################################################################## +# - raft_distance ------------------------------------------------------------ +add_library(raft_distance SHARED + src/distance/specializations/detail + src/distance/specializations/detail/canberra.cu + src/distance/specializations/detail/chebyshev.cu + src/distance/specializations/detail/correlation.cu + src/distance/specializations/detail/cosine.cu + src/distance/specializations/detail/hamming_unexpanded.cu + src/distance/specializations/detail/hellinger_expanded.cu + src/distance/specializations/detail/jensen_shannon.cu + src/distance/specializations/detail/kl_divergence.cu + src/distance/specializations/detail/l1.cu + src/distance/specializations/detail/l2_expanded.cu + src/distance/specializations/detail/l2_sqrt_expanded.cu + src/distance/specializations/detail/l2_sqrt_unexpanded.cu + src/distance/specializations/detail/l2_unexpanded.cu + src/distance/specializations/detail/lp_unexpanded.cu +) -target_compile_features(raft_distance PUBLIC cxx_std_17 $) -target_compile_features(raft_nn PUBLIC cxx_std_17 $) -target_compile_features(raft INTERFACE cxx_std_17 $) +add_library(raft::raft_distance ALIAS raft_distance) +target_link_libraries(raft_distance PUBLIC raft::raft) -install(TARGETS raft_distance - DESTINATION ${lib_dir} - EXPORT raft-exports) +############################################################################## +# - raft_nn ------------------------------------------------------------------ + +if(TARGET FAISS::FAISS) + add_library(raft_nn SHARED + src/nn/specializations/ball_cover.cu + src/nn/specializations/detail/ball_cover_lowdim.cu + src/nn/specializations/fused_l2_knn.cu + src/nn/specializations/knn.cu + ) + add_library(raft::raft_nn ALIAS raft_nn) + + target_link_libraries(raft_nn PUBLIC raft::raft FAISS::FAISS) +endif() -install(TARGETS raft_nn - DESTINATION ${lib_dir} - EXPORT raft-exports) +############################################################################## +# - install targets----------------------------------------------------------- +rapids_cmake_install_lib_dir( lib_dir ) +include(GNUInstallDirs) +include(CPack) -install(TARGETS raft +install(TARGETS raft raft_distance DESTINATION ${lib_dir} EXPORT raft-exports) +if(TARGET raft_nn) + install(TARGETS raft_nn + DESTINATION ${lib_dir} + EXPORT raft-exports) +endif() -include(GNUInstallDirs) install(DIRECTORY include/raft/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/raft ) From af46a094c9211278004eb86d5ed4ec2e4816dc9e Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Mon, 20 Dec 2021 13:20:02 -0500 Subject: [PATCH 31/66] Add RAFT_ENABLE_NN_COMPONENT CMake build option --- cpp/CMakeLists.txt | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 1cd3ab55d5..924ae60869 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -48,6 +48,7 @@ option(DETECT_CONDA_ENV "Enable detection of conda environment for dependencies" option(DISABLE_DEPRECATION_WARNINGS "Disable depreaction warnings " ON) option(DISABLE_OPENMP "Disable OpenMP" OFF) option(NVTX "Enable nvtx markers" OFF) +option(RAFT_ENABLE_NN_COMPONENT "Enable the raft::nn component" ON) message(VERBOSE "RAFT: Build RAFT unit-tests: ${BUILD_TESTS}") message(VERBOSE "RAFT: Enable detection of conda environment for dependencies: ${DETECT_CONDA_ENV}") @@ -56,6 +57,7 @@ message(VERBOSE "RAFT: Disable OpenMP: ${DISABLE_OPENMP}") message(VERBOSE "RAFT: Enable kernel resource usage info: ${CUDA_ENABLE_KERNELINFO}") message(VERBOSE "RAFT: Enable lineinfo in nvcc: ${CUDA_ENABLE_LINEINFO}") message(VERBOSE "RAFT: Enable nvtx markers: ${NVTX}") +message(VERBOSE "RAFT: Enable raft::nn target: ${RAFT_ENABLE_NN_COMPONENT}") message(VERBOSE "RAFT: Statically link the CUDA runtime: ${CUDA_STATIC_RUNTIME}") # Set RMM logging level @@ -106,8 +108,11 @@ include(cmake/thirdparty/get_rmm.cmake) include(cmake/thirdparty/get_libcudacxx.cmake) include(cmake/thirdparty/get_cuco.cmake) -if(BUILD_TESTS) +if(BUILD_TESTS OR RAFT_ENABLE_NN_COMPONENT) include(cmake/thirdparty/get_faiss.cmake) +endif() + +if(BUILD_TESTS) include(cmake/thirdparty/get_gtest.cmake) include(cmake/thirdparty/get_nccl.cmake) include(cmake/thirdparty/get_ucx.cmake) @@ -164,7 +169,7 @@ target_link_libraries(raft_distance PUBLIC raft::raft) ############################################################################## # - raft_nn ------------------------------------------------------------------ -if(TARGET FAISS::FAISS) +if(RAFT_ENABLE_NN_COMPONENT) add_library(raft_nn SHARED src/nn/specializations/ball_cover.cu src/nn/specializations/detail/ball_cover_lowdim.cu From 1e9b9bd496477d812aa1f1b1d34ef8925d77e781 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Mon, 20 Dec 2021 14:28:42 -0500 Subject: [PATCH 32/66] Adding option to turn off buildign all shared libs --- cpp/CMakeLists.txt | 86 +++++++++++++++++++++++++++++----------------- 1 file changed, 54 insertions(+), 32 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 924ae60869..8a8235e07c 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -48,6 +48,8 @@ option(DETECT_CONDA_ENV "Enable detection of conda environment for dependencies" option(DISABLE_DEPRECATION_WARNINGS "Disable depreaction warnings " ON) option(DISABLE_OPENMP "Disable OpenMP" OFF) option(NVTX "Enable nvtx markers" OFF) + +option(RAFT_BUILD_SHARED_LIBS "Enable building raft shared library instantiations" ON) option(RAFT_ENABLE_NN_COMPONENT "Enable the raft::nn component" ON) message(VERBOSE "RAFT: Build RAFT unit-tests: ${BUILD_TESTS}") @@ -143,42 +145,53 @@ set(RAFT_COMPILE_DEFINITIONS target_compile_definitions(raft INTERFACE ${RAFT_COMPILE_DEFINITIONS}) target_compile_features(raft INTERFACE cxx_std_17 $) +if(RAFT_BUILD_SHARED_LIBS) ############################################################################## # - raft_distance ------------------------------------------------------------ -add_library(raft_distance SHARED - src/distance/specializations/detail - src/distance/specializations/detail/canberra.cu - src/distance/specializations/detail/chebyshev.cu - src/distance/specializations/detail/correlation.cu - src/distance/specializations/detail/cosine.cu - src/distance/specializations/detail/hamming_unexpanded.cu - src/distance/specializations/detail/hellinger_expanded.cu - src/distance/specializations/detail/jensen_shannon.cu - src/distance/specializations/detail/kl_divergence.cu - src/distance/specializations/detail/l1.cu - src/distance/specializations/detail/l2_expanded.cu - src/distance/specializations/detail/l2_sqrt_expanded.cu - src/distance/specializations/detail/l2_sqrt_unexpanded.cu - src/distance/specializations/detail/l2_unexpanded.cu - src/distance/specializations/detail/lp_unexpanded.cu -) - -add_library(raft::raft_distance ALIAS raft_distance) -target_link_libraries(raft_distance PUBLIC raft::raft) + add_library(raft_distance SHARED + src/distance/specializations/detail + src/distance/specializations/detail/canberra.cu + src/distance/specializations/detail/chebyshev.cu + src/distance/specializations/detail/correlation.cu + src/distance/specializations/detail/cosine.cu + src/distance/specializations/detail/hamming_unexpanded.cu + src/distance/specializations/detail/hellinger_expanded.cu + src/distance/specializations/detail/jensen_shannon.cu + src/distance/specializations/detail/kl_divergence.cu + src/distance/specializations/detail/l1.cu + src/distance/specializations/detail/l2_expanded.cu + src/distance/specializations/detail/l2_sqrt_expanded.cu + src/distance/specializations/detail/l2_sqrt_unexpanded.cu + src/distance/specializations/detail/l2_unexpanded.cu + src/distance/specializations/detail/lp_unexpanded.cu + ) + + add_library(raft::raft_distance ALIAS raft_distance) + target_link_libraries(raft_distance PUBLIC raft::raft) + target_compile_options(raft_distance + PRIVATE "$<$:${RAFT_CXX_FLAGS}>" + "$<$:${RAFT_CUDA_FLAGS}>" + ) ############################################################################## # - raft_nn ------------------------------------------------------------------ -if(RAFT_ENABLE_NN_COMPONENT) - add_library(raft_nn SHARED - src/nn/specializations/ball_cover.cu - src/nn/specializations/detail/ball_cover_lowdim.cu - src/nn/specializations/fused_l2_knn.cu - src/nn/specializations/knn.cu - ) - add_library(raft::raft_nn ALIAS raft_nn) + if(RAFT_ENABLE_NN_COMPONENT) + add_library(raft_nn SHARED + src/nn/specializations/ball_cover.cu + src/nn/specializations/detail/ball_cover_lowdim.cu + src/nn/specializations/fused_l2_knn.cu + src/nn/specializations/knn.cu + ) + add_library(raft::raft_nn ALIAS raft_nn) + + target_link_libraries(raft_nn PUBLIC raft::raft FAISS::FAISS) + target_compile_options(raft_nn + PRIVATE "$<$:${RAFT_CXX_FLAGS}>" + "$<$:${RAFT_CUDA_FLAGS}>" + ) + endif() - target_link_libraries(raft_nn PUBLIC raft::raft FAISS::FAISS) endif() ############################################################################## @@ -187,15 +200,24 @@ rapids_cmake_install_lib_dir( lib_dir ) include(GNUInstallDirs) include(CPack) -install(TARGETS raft raft_distance +install(TARGETS raft DESTINATION ${lib_dir} EXPORT raft-exports) -if(TARGET raft_nn) - install(TARGETS raft_nn + +if(RAFT_BUILD_SHARED_LIBS) + install(TARGETS raft_distance DESTINATION ${lib_dir} EXPORT raft-exports) + + if(TARGET raft_nn) + install(TARGETS raft_nn + DESTINATION ${lib_dir} + EXPORT raft-exports) + endif() endif() + + install(DIRECTORY include/raft/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/raft ) From 7fc712cc4efcaa1e58463e25d7491017a691dcae Mon Sep 17 00:00:00 2001 From: Paul Taylor Date: Mon, 20 Dec 2021 14:44:15 -0500 Subject: [PATCH 33/66] Refactor get_faiss to export faiss::faiss build target --- cpp/CMakeLists.txt | 3 +-- cpp/cmake/thirdparty/get_faiss.cmake | 24 +++++++++++++++++------- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 8a8235e07c..4385a2158e 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -185,13 +185,12 @@ if(RAFT_BUILD_SHARED_LIBS) ) add_library(raft::raft_nn ALIAS raft_nn) - target_link_libraries(raft_nn PUBLIC raft::raft FAISS::FAISS) + target_link_libraries(raft_nn PUBLIC raft::raft faiss::faiss) target_compile_options(raft_nn PRIVATE "$<$:${RAFT_CXX_FLAGS}>" "$<$:${RAFT_CUDA_FLAGS}>" ) endif() - endif() ############################################################################## diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake index a65401579c..712c3900d1 100644 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -23,12 +23,12 @@ function(find_and_configure_faiss) HEADER_NAMES faiss/IndexFlat.h LIBRARY_NAMES faiss ) - - rapids_cpm_find(FAISS ${PKG_VERSION} - GLOBAL_TARGETS faiss + rapids_cpm_find(faiss ${PKG_VERSION} + GLOBAL_TARGETS faiss::faiss CPM_ARGS GIT_REPOSITORY https://github.com/facebookresearch/faiss.git GIT_TAG ${PKG_PINNED_TAG} + EXCLUDE_FROM_ALL TRUE OPTIONS "FAISS_ENABLE_PYTHON OFF" "BUILD_SHARED_LIBS OFF" @@ -38,14 +38,24 @@ function(find_and_configure_faiss) "CMAKE_MESSAGE_LOG_LEVEL VERBOSE" ) - if(FAISS_ADDED) - set(FAISS_GPU_HEADERS ${FAISS_SOURCE_DIR} PARENT_SCOPE) + if(TARGET faiss AND NOT TARGET faiss::faiss) + add_library(faiss::faiss ALIAS faiss) endif() - if(TARGET faiss AND NOT TARGET FAISS::FAISS) - add_library(FAISS::FAISS ALIAS faiss) + if(faiss_ADDED) + rapids_export(BUILD faiss + EXPORT_SET faiss-targets + GLOBAL_TARGETS faiss + NAMESPACE faiss::) endif() + # We generate the faiss-config files when we built faiss locally, so always do `find_dependency` + rapids_export_package(BUILD faiss raft-faiss-exports) + + # Tell cmake where it can find the generated faiss-config.cmake we wrote. + include("${rapids-cmake-dir}/export/find_package_root.cmake") + rapids_export_find_package_root(BUILD faiss [=[${CMAKE_CURRENT_LIST_DIR}]=] raft-faiss-exports) + endfunction() find_and_configure_faiss(VERSION 1.7.0 From a85b085cbbefba1d71f9ae30b8efd885220b10b6 Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Mon, 20 Dec 2021 15:02:59 -0500 Subject: [PATCH 34/66] RAFT_ENABLE_NN_COMPONENT is a dependent option of RAFT_BUILD_SHARED_LIBS --- cpp/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 4385a2158e..aa0a07cf38 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -50,7 +50,8 @@ option(DISABLE_OPENMP "Disable OpenMP" OFF) option(NVTX "Enable nvtx markers" OFF) option(RAFT_BUILD_SHARED_LIBS "Enable building raft shared library instantiations" ON) -option(RAFT_ENABLE_NN_COMPONENT "Enable the raft::nn component" ON) +include(CMakeDependentOption) +cmake_dependent_option(RAFT_ENABLE_NN_COMPONENT "Enable the raft::nn component" ON RAFT_BUILD_SHARED_LIBS OFF) message(VERBOSE "RAFT: Build RAFT unit-tests: ${BUILD_TESTS}") message(VERBOSE "RAFT: Enable detection of conda environment for dependencies: ${DETECT_CONDA_ENV}") From 260a727b792ac26af15251e1ad8298603ee23336 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Mon, 20 Dec 2021 17:35:07 -0500 Subject: [PATCH 35/66] Using lowercase faiss target --- cpp/cmake/thirdparty/get_faiss.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake index 712c3900d1..b74ede9381 100644 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -19,7 +19,7 @@ function(find_and_configure_faiss) cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} ) - rapids_find_generate_module(FAISS + rapids_find_generate_module(faiss HEADER_NAMES faiss/IndexFlat.h LIBRARY_NAMES faiss ) From bb4a7f32fed41ef310910dfe299c25fca3f40f69 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Mon, 20 Dec 2021 17:52:08 -0500 Subject: [PATCH 36/66] Changing RAFT_BUILD_SHARED_LIBS -> RAFT_COMPILE_LIBRARIES --- cpp/CMakeLists.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index aa0a07cf38..0048a2398b 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -49,9 +49,9 @@ option(DISABLE_DEPRECATION_WARNINGS "Disable depreaction warnings " ON) option(DISABLE_OPENMP "Disable OpenMP" OFF) option(NVTX "Enable nvtx markers" OFF) -option(RAFT_BUILD_SHARED_LIBS "Enable building raft shared library instantiations" ON) +option(RAFT_COMPILE_LIBRARIES "Enable building raft shared library instantiations" ON) include(CMakeDependentOption) -cmake_dependent_option(RAFT_ENABLE_NN_COMPONENT "Enable the raft::nn component" ON RAFT_BUILD_SHARED_LIBS OFF) +cmake_dependent_option(RAFT_ENABLE_NN_COMPONENT "Enable the raft::nn component" ON RAFT_COMPILE_LIBRARIES OFF) message(VERBOSE "RAFT: Build RAFT unit-tests: ${BUILD_TESTS}") message(VERBOSE "RAFT: Enable detection of conda environment for dependencies: ${DETECT_CONDA_ENV}") @@ -146,7 +146,7 @@ set(RAFT_COMPILE_DEFINITIONS target_compile_definitions(raft INTERFACE ${RAFT_COMPILE_DEFINITIONS}) target_compile_features(raft INTERFACE cxx_std_17 $) -if(RAFT_BUILD_SHARED_LIBS) +if(RAFT_COMPILE_LIBRARIES) ############################################################################## # - raft_distance ------------------------------------------------------------ add_library(raft_distance SHARED @@ -204,7 +204,7 @@ install(TARGETS raft DESTINATION ${lib_dir} EXPORT raft-exports) -if(RAFT_BUILD_SHARED_LIBS) +if(RAFT_COMPILE_LIBRARIES) install(TARGETS raft_distance DESTINATION ${lib_dir} EXPORT raft-exports) From 8c02d2c395f908300481db0fae9d8d9cde866742 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Tue, 21 Dec 2021 12:14:04 -0500 Subject: [PATCH 37/66] Passing deps transitively --- cpp/CMakeLists.txt | 98 +++++++++++++++++++++++++---------------- cpp/test/CMakeLists.txt | 1 - 2 files changed, 60 insertions(+), 39 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 0048a2398b..1c2aa8cda1 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -49,9 +49,11 @@ option(DISABLE_DEPRECATION_WARNINGS "Disable depreaction warnings " ON) option(DISABLE_OPENMP "Disable OpenMP" OFF) option(NVTX "Enable nvtx markers" OFF) -option(RAFT_COMPILE_LIBRARIES "Enable building raft shared library instantiations" ON) include(CMakeDependentOption) -cmake_dependent_option(RAFT_ENABLE_NN_COMPONENT "Enable the raft::nn component" ON RAFT_COMPILE_LIBRARIES OFF) + +cmake_dependent_option(RAFT_COMPILE_LIBRARIES "Enable building raft shared library instantiations" ON BUILD_TESTS OFF) +cmake_dependent_option(RAFT_ENABLE_NN_COMPONENT "Enable the raft::nn component" ON BUILD_TESTS OFF) +cmake_dependent_option(RAFT_ENABLE_DISTANCE_COMPONENT "Enable the raft::distance component" ON BUILD_TESTS OFF) message(VERBOSE "RAFT: Build RAFT unit-tests: ${BUILD_TESTS}") message(VERBOSE "RAFT: Enable detection of conda environment for dependencies: ${DETECT_CONDA_ENV}") @@ -126,8 +128,11 @@ endif() add_library(raft INTERFACE) add_library(raft::raft ALIAS raft) -target_include_directories(raft INTERFACE "$" - "$") + +target_include_directories(raft INTERFACE + "$" + "$") + set(RAFT_LINK_LIBRARIES CUDA::cublas CUDA::curand @@ -146,52 +151,71 @@ set(RAFT_COMPILE_DEFINITIONS target_compile_definitions(raft INTERFACE ${RAFT_COMPILE_DEFINITIONS}) target_compile_features(raft INTERFACE cxx_std_17 $) -if(RAFT_COMPILE_LIBRARIES) ############################################################################## # - raft_distance ------------------------------------------------------------ - add_library(raft_distance SHARED - src/distance/specializations/detail - src/distance/specializations/detail/canberra.cu - src/distance/specializations/detail/chebyshev.cu - src/distance/specializations/detail/correlation.cu - src/distance/specializations/detail/cosine.cu - src/distance/specializations/detail/hamming_unexpanded.cu - src/distance/specializations/detail/hellinger_expanded.cu - src/distance/specializations/detail/jensen_shannon.cu - src/distance/specializations/detail/kl_divergence.cu - src/distance/specializations/detail/l1.cu - src/distance/specializations/detail/l2_expanded.cu - src/distance/specializations/detail/l2_sqrt_expanded.cu - src/distance/specializations/detail/l2_sqrt_unexpanded.cu - src/distance/specializations/detail/l2_unexpanded.cu - src/distance/specializations/detail/lp_unexpanded.cu - ) +if(RAFT_ENABLE_DISTANCE_COMPONENT) + + if(NOT RAFT_COMPILE_LIBRARIES) + add_library(raft_distance INTERFACE) + + else() + add_library(raft_distance SHARED + src/distance/specializations/detail + src/distance/specializations/detail/canberra.cu + src/distance/specializations/detail/chebyshev.cu + src/distance/specializations/detail/correlation.cu + src/distance/specializations/detail/cosine.cu + src/distance/specializations/detail/hamming_unexpanded.cu + src/distance/specializations/detail/hellinger_expanded.cu + src/distance/specializations/detail/jensen_shannon.cu + src/distance/specializations/detail/kl_divergence.cu + src/distance/specializations/detail/l1.cu + src/distance/specializations/detail/l2_expanded.cu + src/distance/specializations/detail/l2_sqrt_expanded.cu + src/distance/specializations/detail/l2_sqrt_unexpanded.cu + src/distance/specializations/detail/l2_unexpanded.cu + src/distance/specializations/detail/lp_unexpanded.cu + ) + + target_link_libraries(raft_distance PUBLIC raft::raft) + target_compile_options(raft_distance + PRIVATE "$<$:${RAFT_CXX_FLAGS}>" + "$<$:${RAFT_CUDA_FLAGS}>" + ) + endif() add_library(raft::raft_distance ALIAS raft_distance) - target_link_libraries(raft_distance PUBLIC raft::raft) - target_compile_options(raft_distance - PRIVATE "$<$:${RAFT_CXX_FLAGS}>" - "$<$:${RAFT_CUDA_FLAGS}>" - ) +endif() ############################################################################## # - raft_nn ------------------------------------------------------------------ - if(RAFT_ENABLE_NN_COMPONENT) +if(RAFT_ENABLE_NN_COMPONENT) + + set(RAFT_NN_INCLUDE_DIRECTORIES ${FAISS_GPU_INCLUDES}) + + if(NOT RAFT_COMPILE_LIBRARIES) + add_library(raft_nn INTERFACE) + target_include_directories(raft_nn INTERFACE ${RAFT_NN_INCLUDE_DIRECTORIES}) + + else() add_library(raft_nn SHARED src/nn/specializations/ball_cover.cu src/nn/specializations/detail/ball_cover_lowdim.cu src/nn/specializations/fused_l2_knn.cu src/nn/specializations/knn.cu ) - add_library(raft::raft_nn ALIAS raft_nn) target_link_libraries(raft_nn PUBLIC raft::raft faiss::faiss) target_compile_options(raft_nn PRIVATE "$<$:${RAFT_CXX_FLAGS}>" "$<$:${RAFT_CUDA_FLAGS}>" ) + target_include_directories(raft_nn PUBLIC ${RAFT_NN_INCLUDE_DIRECTORIES}) endif() + + add_library(raft::raft_nn ALIAS raft_nn) + endif() ############################################################################## @@ -204,19 +228,17 @@ install(TARGETS raft DESTINATION ${lib_dir} EXPORT raft-exports) -if(RAFT_COMPILE_LIBRARIES) +if(TARGET raft_distance) install(TARGETS raft_distance DESTINATION ${lib_dir} EXPORT raft-exports) - - if(TARGET raft_nn) - install(TARGETS raft_nn - DESTINATION ${lib_dir} - EXPORT raft-exports) - endif() endif() - +if(TARGET raft_nn) + install(TARGETS raft_nn + DESTINATION ${lib_dir} + EXPORT raft-exports) +endif() install(DIRECTORY include/raft/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/raft @@ -237,7 +259,7 @@ mathematical computational primitives, and utilities that accelerate building analytics and data science algorithms in the RAPIDS ecosystem. ]=]) - rapids_export(INSTALL raft +rapids_export(INSTALL raft EXPORT_SET raft-exports GLOBAL_TARGETS raft raft_distance# since we can't hook into EXPORT SETS NAMESPACE raft:: diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index b37c671525..ca4999c1df 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -119,7 +119,6 @@ target_compile_options(test_raft target_include_directories(test_raft PUBLIC "$" - "${FAISS_GPU_HEADERS}" ) From ebb7c1d7c3bfa1352fea6662b0f819a8e97f1aed Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Tue, 21 Dec 2021 14:47:06 -0500 Subject: [PATCH 38/66] Moving faiss back to test target. We need to figure out how to pass dependencies through transitively --- cpp/CMakeLists.txt | 19 ++++++++++++++----- cpp/test/CMakeLists.txt | 1 + 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 1c2aa8cda1..b68061dcff 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -49,11 +49,15 @@ option(DISABLE_DEPRECATION_WARNINGS "Disable depreaction warnings " ON) option(DISABLE_OPENMP "Disable OpenMP" OFF) option(NVTX "Enable nvtx markers" OFF) -include(CMakeDependentOption) +option(RAFT_COMPILE_LIBRARIES "Enable building raft shared library instantiations" OFF) +option(RAFT_ENABLE_NN_COMPONENT "Enable the raft::nn component" OFF) +option(RAFT_ENABLE_DISTANCE_COMPONENT "Enable the raft::distance component" OFF) -cmake_dependent_option(RAFT_COMPILE_LIBRARIES "Enable building raft shared library instantiations" ON BUILD_TESTS OFF) -cmake_dependent_option(RAFT_ENABLE_NN_COMPONENT "Enable the raft::nn component" ON BUILD_TESTS OFF) -cmake_dependent_option(RAFT_ENABLE_DISTANCE_COMPONENT "Enable the raft::distance component" ON BUILD_TESTS OFF) +if(BUILD_TESTS) + set(RAFT_COMPILE_LIBRARIES ON) + set(RAFT_ENABLE_NN_COMPONENT ON) + set(RAFT_ENABLE_DISTANCE_COMPONENT ON) +endif() message(VERBOSE "RAFT: Build RAFT unit-tests: ${BUILD_TESTS}") message(VERBOSE "RAFT: Enable detection of conda environment for dependencies: ${DETECT_CONDA_ENV}") @@ -155,6 +159,8 @@ target_compile_features(raft INTERFACE cxx_std_17 $ # - raft_distance ------------------------------------------------------------ if(RAFT_ENABLE_DISTANCE_COMPONENT) + message("RAFT-distance component enabled.") + if(NOT RAFT_COMPILE_LIBRARIES) add_library(raft_distance INTERFACE) @@ -192,10 +198,13 @@ endif() if(RAFT_ENABLE_NN_COMPONENT) + message("RAFT-nn component enabled.") + set(RAFT_NN_INCLUDE_DIRECTORIES ${FAISS_GPU_INCLUDES}) if(NOT RAFT_COMPILE_LIBRARIES) add_library(raft_nn INTERFACE) + target_link_libraries(raft_nn INTERFACE raft::raft) target_include_directories(raft_nn INTERFACE ${RAFT_NN_INCLUDE_DIRECTORIES}) else() @@ -206,7 +215,7 @@ if(RAFT_ENABLE_NN_COMPONENT) src/nn/specializations/knn.cu ) - target_link_libraries(raft_nn PUBLIC raft::raft faiss::faiss) + target_link_libraries(raft_nn PUBLIC raft::raft) target_compile_options(raft_nn PRIVATE "$<$:${RAFT_CXX_FLAGS}>" "$<$:${RAFT_CUDA_FLAGS}>" diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index ca4999c1df..3666bd8f2c 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -127,6 +127,7 @@ PRIVATE raft # transitively links all CUDA libs, etc raft_distance raft_nn + faiss::faiss GTest::gtest GTest::gtest_main Threads::Threads From c1e9c2d296fadca04d3a50a2fc7209740aac5b4e Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Wed, 22 Dec 2021 09:17:46 -0500 Subject: [PATCH 39/66] Another round of cmake cleanups --- cpp/CMakeLists.txt | 106 ++++++++++----------------- cpp/cmake/thirdparty/get_faiss.cmake | 1 + 2 files changed, 40 insertions(+), 67 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index b68061dcff..958666d3bb 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -49,15 +49,10 @@ option(DISABLE_DEPRECATION_WARNINGS "Disable depreaction warnings " ON) option(DISABLE_OPENMP "Disable OpenMP" OFF) option(NVTX "Enable nvtx markers" OFF) -option(RAFT_COMPILE_LIBRARIES "Enable building raft shared library instantiations" OFF) -option(RAFT_ENABLE_NN_COMPONENT "Enable the raft::nn component" OFF) -option(RAFT_ENABLE_DISTANCE_COMPONENT "Enable the raft::distance component" OFF) - -if(BUILD_TESTS) - set(RAFT_COMPILE_LIBRARIES ON) - set(RAFT_ENABLE_NN_COMPONENT ON) - set(RAFT_ENABLE_DISTANCE_COMPONENT ON) -endif() +option(RAFT_COMPILE_LIBRARIES "Enable building raft shared library instantiations" ${BUILD_TESTS}) +include(CMakeDependentOption) +cmake_dependent_option(RAFT_ENABLE_NN_COMPONENT "Enable the raft::nn component" ON RAFT_COMPILE_LIBRARIES OFF) +cmake_dependent_option(RAFT_ENABLE_DISTANCE_COMPONENT "Enable the raft::distance component" ON RAFT_COMPILE_LIBRARIES OFF) message(VERBOSE "RAFT: Build RAFT unit-tests: ${BUILD_TESTS}") message(VERBOSE "RAFT: Enable detection of conda environment for dependencies: ${DETECT_CONDA_ENV}") @@ -67,6 +62,7 @@ message(VERBOSE "RAFT: Enable kernel resource usage info: ${CUDA_ENABLE_KERNELIN message(VERBOSE "RAFT: Enable lineinfo in nvcc: ${CUDA_ENABLE_LINEINFO}") message(VERBOSE "RAFT: Enable nvtx markers: ${NVTX}") message(VERBOSE "RAFT: Enable raft::nn target: ${RAFT_ENABLE_NN_COMPONENT}") +message(VERBOSE "RAFT: Enable raft::distance target: ${RAFT_ENABLE_DISTANCE_COMPONENT}") message(VERBOSE "RAFT: Statically link the CUDA runtime: ${CUDA_STATIC_RUNTIME}") # Set RMM logging level @@ -158,73 +154,49 @@ target_compile_features(raft INTERFACE cxx_std_17 $ ############################################################################## # - raft_distance ------------------------------------------------------------ if(RAFT_ENABLE_DISTANCE_COMPONENT) - - message("RAFT-distance component enabled.") - - if(NOT RAFT_COMPILE_LIBRARIES) - add_library(raft_distance INTERFACE) - - else() - add_library(raft_distance SHARED - src/distance/specializations/detail - src/distance/specializations/detail/canberra.cu - src/distance/specializations/detail/chebyshev.cu - src/distance/specializations/detail/correlation.cu - src/distance/specializations/detail/cosine.cu - src/distance/specializations/detail/hamming_unexpanded.cu - src/distance/specializations/detail/hellinger_expanded.cu - src/distance/specializations/detail/jensen_shannon.cu - src/distance/specializations/detail/kl_divergence.cu - src/distance/specializations/detail/l1.cu - src/distance/specializations/detail/l2_expanded.cu - src/distance/specializations/detail/l2_sqrt_expanded.cu - src/distance/specializations/detail/l2_sqrt_unexpanded.cu - src/distance/specializations/detail/l2_unexpanded.cu - src/distance/specializations/detail/lp_unexpanded.cu - ) - - target_link_libraries(raft_distance PUBLIC raft::raft) - target_compile_options(raft_distance - PRIVATE "$<$:${RAFT_CXX_FLAGS}>" - "$<$:${RAFT_CUDA_FLAGS}>" - ) - endif() - + add_library(raft_distance SHARED + src/distance/specializations/detail + src/distance/specializations/detail/canberra.cu + src/distance/specializations/detail/chebyshev.cu + src/distance/specializations/detail/correlation.cu + src/distance/specializations/detail/cosine.cu + src/distance/specializations/detail/hamming_unexpanded.cu + src/distance/specializations/detail/hellinger_expanded.cu + src/distance/specializations/detail/jensen_shannon.cu + src/distance/specializations/detail/kl_divergence.cu + src/distance/specializations/detail/l1.cu + src/distance/specializations/detail/l2_expanded.cu + src/distance/specializations/detail/l2_sqrt_expanded.cu + src/distance/specializations/detail/l2_sqrt_unexpanded.cu + src/distance/specializations/detail/l2_unexpanded.cu + src/distance/specializations/detail/lp_unexpanded.cu + ) add_library(raft::raft_distance ALIAS raft_distance) + + target_link_libraries(raft_distance PUBLIC raft::raft) + target_compile_options(raft_distance + PRIVATE "$<$:${RAFT_CXX_FLAGS}>" + "$<$:${RAFT_CUDA_FLAGS}>" + ) endif() ############################################################################## # - raft_nn ------------------------------------------------------------------ if(RAFT_ENABLE_NN_COMPONENT) - - message("RAFT-nn component enabled.") - - set(RAFT_NN_INCLUDE_DIRECTORIES ${FAISS_GPU_INCLUDES}) - - if(NOT RAFT_COMPILE_LIBRARIES) - add_library(raft_nn INTERFACE) - target_link_libraries(raft_nn INTERFACE raft::raft) - target_include_directories(raft_nn INTERFACE ${RAFT_NN_INCLUDE_DIRECTORIES}) - - else() - add_library(raft_nn SHARED - src/nn/specializations/ball_cover.cu - src/nn/specializations/detail/ball_cover_lowdim.cu - src/nn/specializations/fused_l2_knn.cu - src/nn/specializations/knn.cu - ) - - target_link_libraries(raft_nn PUBLIC raft::raft) - target_compile_options(raft_nn - PRIVATE "$<$:${RAFT_CXX_FLAGS}>" - "$<$:${RAFT_CUDA_FLAGS}>" - ) - target_include_directories(raft_nn PUBLIC ${RAFT_NN_INCLUDE_DIRECTORIES}) - endif() - + add_library(raft_nn SHARED + src/nn/specializations/ball_cover.cu + src/nn/specializations/detail/ball_cover_lowdim.cu + src/nn/specializations/fused_l2_knn.cu + src/nn/specializations/knn.cu + ) add_library(raft::raft_nn ALIAS raft_nn) + target_link_libraries(raft_nn PUBLIC raft::raft faiss::faiss) + target_compile_options(raft_nn + PRIVATE "$<$:${RAFT_CXX_FLAGS}>" + "$<$:${RAFT_CUDA_FLAGS}>" + ) endif() ############################################################################## diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake index b74ede9381..09251c0a55 100644 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -51,6 +51,7 @@ function(find_and_configure_faiss) # We generate the faiss-config files when we built faiss locally, so always do `find_dependency` rapids_export_package(BUILD faiss raft-faiss-exports) + rapids_export_package(INSTALL faiss raft-faiss-exports) # Tell cmake where it can find the generated faiss-config.cmake we wrote. include("${rapids-cmake-dir}/export/find_package_root.cmake") From 52e863559430b473eaad22f7aa9ce0a5e7846dce Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Wed, 22 Dec 2021 11:28:19 -0500 Subject: [PATCH 40/66] Refactor again now that I understand the libs are an optimization --- cpp/CMakeLists.txt | 70 +++++++++++++++-------------- cpp/test/CMakeLists.txt | 2 +- cpp/test/distance/distance_base.cuh | 2 + cpp/test/sparse/knn_graph.cu | 2 + cpp/test/spatial/ball_cover.cu | 4 +- cpp/test/spatial/knn.cu | 2 + cpp/test/spatial/selection.cu | 2 + 7 files changed, 48 insertions(+), 36 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 31d50a1b74..f112f5eace 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -51,10 +51,7 @@ option(NVTX "Enable nvtx markers" OFF) option(RAFT_COMPILE_LIBRARIES "Enable building raft shared library instantiations" ON) include(CMakeDependentOption) -cmake_dependent_option(RAFT_ENABLE_NN_COMPONENT "Enable the raft::nn component" ON RAFT_COMPILE_LIBRARIES OFF) -cmake_dependent_option(RAFT_ENABLE_DISTANCE_COMPONENT "Enable the raft::distance component" ON RAFT_COMPILE_LIBRARIES OFF) - -cmake_dependent_option(RAFT_USE_FAISS_STATIC "Build and statically link the FAISS library for nearest neighbors search on GPU" ON RAFT_ENABLE_NN_COMPONENT OFF) +cmake_dependent_option(RAFT_USE_FAISS_STATIC "Build and statically link the FAISS library for nearest neighbors search on GPU" ON RAFT_COMPILE_LIBRARIES OFF) message(VERBOSE "RAFT: Build RAFT unit-tests: ${BUILD_TESTS}") message(VERBOSE "RAFT: Enable detection of conda environment for dependencies: ${DETECT_CONDA_ENV}") @@ -112,16 +109,14 @@ rapids_cpm_init() include(cmake/thirdparty/get_rmm.cmake) include(cmake/thirdparty/get_libcudacxx.cmake) include(cmake/thirdparty/get_cuco.cmake) +# CPMFindPackage(faiss) so it's available for raft consumers to use +include(cmake/thirdparty/get_faiss.cmake) # CPMFindPackage(GTest) so it's available for raft consumers to use include(cmake/thirdparty/get_gtest.cmake) -if(BUILD_TESTS OR RAFT_ENABLE_NN_COMPONENT) - include(cmake/thirdparty/get_faiss.cmake) -endif() - if(BUILD_TESTS) - include(cmake/thirdparty/get_nccl.cmake) - include(cmake/thirdparty/get_ucx.cmake) + # include(cmake/thirdparty/get_nccl.cmake) + # include(cmake/thirdparty/get_ucx.cmake) endif() ############################################################################## @@ -149,8 +144,12 @@ target_compile_features(raft INTERFACE cxx_std_17 $ ############################################################################## # - raft_distance ------------------------------------------------------------ -if(RAFT_ENABLE_DISTANCE_COMPONENT) - add_library(raft_distance SHARED +add_library(raft_distance INTERFACE) +add_library(raft::raft_distance ALIAS raft_distance) +target_link_libraries(raft_distance INTERFACE raft::raft $) + +if(RAFT_COMPILE_LIBRARIES) + add_library(raft_distance_lib SHARED src/distance/specializations/detail src/distance/specializations/detail/canberra.cu src/distance/specializations/detail/chebyshev.cu @@ -167,32 +166,47 @@ if(RAFT_ENABLE_DISTANCE_COMPONENT) src/distance/specializations/detail/l2_unexpanded.cu src/distance/specializations/detail/lp_unexpanded.cu ) - add_library(raft::raft_distance ALIAS raft_distance) + set_target_properties(raft_distance_lib PROPERTIES OUTPUT_NAME raft_distance) - target_link_libraries(raft_distance PUBLIC raft::raft) - target_compile_options(raft_distance + target_link_libraries(raft_distance_lib PRIVATE raft::raft) + target_compile_options(raft_distance_lib PRIVATE "$<$:${RAFT_CXX_FLAGS}>" "$<$:${RAFT_CUDA_FLAGS}>" ) + target_compile_definitions(raft_distance_lib + INTERFACE "RAFT_DISTANCE_COMPILED") + + install(TARGETS raft_distance_lib + DESTINATION ${lib_dir} + EXPORT raft-exports) endif() ############################################################################## # - raft_nn ------------------------------------------------------------------ +add_library(raft_nn INTERFACE) +add_library(raft::raft_nn ALIAS raft_nn) +target_link_libraries(raft_nn INTERFACE raft::raft faiss::faiss $) -if(RAFT_ENABLE_NN_COMPONENT) - add_library(raft_nn SHARED +if(RAFT_COMPILE_LIBRARIES) + add_library(raft_nn_lib SHARED src/nn/specializations/ball_cover.cu src/nn/specializations/detail/ball_cover_lowdim.cu src/nn/specializations/fused_l2_knn.cu src/nn/specializations/knn.cu ) - add_library(raft::raft_nn ALIAS raft_nn) + set_target_properties(raft_nn_lib PROPERTIES OUTPUT_NAME raft_nn) - target_link_libraries(raft_nn PUBLIC raft::raft faiss::faiss) - target_compile_options(raft_nn + target_link_libraries(raft_nn_lib PRIVATE raft::raft faiss::faiss) + target_compile_options(raft_nn_lib PRIVATE "$<$:${RAFT_CXX_FLAGS}>" "$<$:${RAFT_CUDA_FLAGS}>" ) + target_compile_definitions(raft_distance_lib + INTERFACE "RAFT_NN_COMPILED") + + install(TARGETS raft_nn_lib + DESTINATION ${lib_dir} + EXPORT raft-exports) endif() ############################################################################## @@ -201,22 +215,10 @@ rapids_cmake_install_lib_dir( lib_dir ) include(GNUInstallDirs) include(CPack) -install(TARGETS raft +install(TARGETS raft raft_nn raft_distance DESTINATION ${lib_dir} EXPORT raft-exports) -if(TARGET raft_distance) - install(TARGETS raft_distance - DESTINATION ${lib_dir} - EXPORT raft-exports) -endif() - -if(TARGET raft_nn) - install(TARGETS raft_nn - DESTINATION ${lib_dir} - EXPORT raft-exports) -endif() - install(DIRECTORY include/raft/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/raft ) @@ -263,7 +265,7 @@ endif() rapids_export(INSTALL raft EXPORT_SET raft-exports - GLOBAL_TARGETS raft raft_distance# since we can't hook into EXPORT SETS + GLOBAL_TARGETS raft raft_distance raft_nn# since we can't hook into EXPORT SETS NAMESPACE raft:: DOCUMENTATION doc_string FINAL_CODE_BLOCK code_string diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index 5db9c55703..77c394527d 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -127,7 +127,7 @@ PRIVATE raft::raft raft::raft_distance raft::raft_nn - NCCL::NCCL + # NCCL::NCCL faiss::faiss GTest::gtest GTest::gtest_main diff --git a/cpp/test/distance/distance_base.cuh b/cpp/test/distance/distance_base.cuh index 475202137b..8f0de29eed 100644 --- a/cpp/test/distance/distance_base.cuh +++ b/cpp/test/distance/distance_base.cuh @@ -20,7 +20,9 @@ #include #include #include +#if defined RAFT_DISTANCE_COMPILED #include +#endif #include namespace raft { diff --git a/cpp/test/sparse/knn_graph.cu b/cpp/test/sparse/knn_graph.cu index df9bb4e3e4..88a3f24df6 100644 --- a/cpp/test/sparse/knn_graph.cu +++ b/cpp/test/sparse/knn_graph.cu @@ -23,7 +23,9 @@ #include #include +#if defined RAFT_NN_COMPILED #include +#endif #include diff --git a/cpp/test/spatial/ball_cover.cu b/cpp/test/spatial/ball_cover.cu index 7b44c477aa..257950e4d7 100644 --- a/cpp/test/spatial/ball_cover.cu +++ b/cpp/test/spatial/ball_cover.cu @@ -20,9 +20,11 @@ #include #include #include +#if defined RAFT_NN_COMPILED #include -#include +#endif +#include #include #include diff --git a/cpp/test/spatial/knn.cu b/cpp/test/spatial/knn.cu index 5681f66e25..8af1505bcd 100644 --- a/cpp/test/spatial/knn.cu +++ b/cpp/test/spatial/knn.cu @@ -19,7 +19,9 @@ #include #include +#if defined RAFT_NN_COMPILED #include +#endif #include diff --git a/cpp/test/spatial/selection.cu b/cpp/test/spatial/selection.cu index 4409f893a8..8ccf3b6b73 100644 --- a/cpp/test/spatial/selection.cu +++ b/cpp/test/spatial/selection.cu @@ -21,7 +21,9 @@ #include #include +#if defined RAFT_NN_COMPILED #include +#endif namespace raft { namespace spatial { From 293302f3a19c4ff551c6d912482438175ea74435 Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Tue, 4 Jan 2022 14:42:35 -0500 Subject: [PATCH 41/66] Move `nn` and `distance` to CMake components --- cpp/CMakeLists.txt | 100 +++++++++++++++++++-------- cpp/cmake/thirdparty/get_faiss.cmake | 7 +- cpp/cmake/thirdparty/get_gtest.cmake | 1 - cpp/test/CMakeLists.txt | 6 +- 4 files changed, 79 insertions(+), 35 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index f112f5eace..69f918e88b 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -111,12 +111,11 @@ include(cmake/thirdparty/get_libcudacxx.cmake) include(cmake/thirdparty/get_cuco.cmake) # CPMFindPackage(faiss) so it's available for raft consumers to use include(cmake/thirdparty/get_faiss.cmake) -# CPMFindPackage(GTest) so it's available for raft consumers to use -include(cmake/thirdparty/get_gtest.cmake) if(BUILD_TESTS) - # include(cmake/thirdparty/get_nccl.cmake) - # include(cmake/thirdparty/get_ucx.cmake) + include(cmake/thirdparty/get_gtest.cmake) + include(cmake/thirdparty/get_nccl.cmake) + include(cmake/thirdparty/get_ucx.cmake) endif() ############################################################################## @@ -145,7 +144,9 @@ target_compile_features(raft INTERFACE cxx_std_17 $ ############################################################################## # - raft_distance ------------------------------------------------------------ add_library(raft_distance INTERFACE) -add_library(raft::raft_distance ALIAS raft_distance) +add_library(raft::distance ALIAS raft_distance) +set_target_properties(raft_distance PROPERTIES EXPORT_NAME distance) + target_link_libraries(raft_distance INTERFACE raft::raft $) if(RAFT_COMPILE_LIBRARIES) @@ -178,13 +179,15 @@ if(RAFT_COMPILE_LIBRARIES) install(TARGETS raft_distance_lib DESTINATION ${lib_dir} - EXPORT raft-exports) + EXPORT raft-distance-exports) endif() ############################################################################## # - raft_nn ------------------------------------------------------------------ add_library(raft_nn INTERFACE) -add_library(raft::raft_nn ALIAS raft_nn) +add_library(raft::nn ALIAS raft_nn) +set_target_properties(raft_nn PROPERTIES EXPORT_NAME nn) + target_link_libraries(raft_nn INTERFACE raft::raft faiss::faiss $) if(RAFT_COMPILE_LIBRARIES) @@ -206,7 +209,7 @@ if(RAFT_COMPILE_LIBRARIES) install(TARGETS raft_nn_lib DESTINATION ${lib_dir} - EXPORT raft-exports) + EXPORT raft-nn-exports) endif() ############################################################################## @@ -215,9 +218,16 @@ rapids_cmake_install_lib_dir( lib_dir ) include(GNUInstallDirs) include(CPack) -install(TARGETS raft raft_nn raft_distance +install(TARGETS raft DESTINATION ${lib_dir} EXPORT raft-exports) +install(TARGETS raft_distance + DESTINATION ${lib_dir} + EXPORT raft-distance-exports) +install(TARGETS raft_nn + DESTINATION ${lib_dir} + EXPORT raft-nn-exports) + install(DIRECTORY include/raft/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/raft @@ -227,12 +237,6 @@ install(DIRECTORY include/raft/ install(FILES include/raft.hpp DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/raft) -include("${rapids-cmake-dir}/export/write_dependencies.cmake") -rapids_export_write_dependencies( - INSTALL raft-faiss-exports - "${PROJECT_BINARY_DIR}/rapids-cmake/raft/export/raft-faiss-dependencies.cmake" -) - ############################################################################## # - install export ----------------------------------------------------------- set(doc_string @@ -242,30 +246,49 @@ Provide targets for the RAFT: RAPIDS Analytics Framework Toolkit. RAPIDS Analytics Framework Toolkit contains shared representations, mathematical computational primitives, and utilities that accelerate building analytics and data science algorithms in the RAPIDS ecosystem. + +Optional Components: + - nn + - distance + +Imported Targets: + - raft::raft + - raft::nn brought in by the `nn` optional component + - raft::distance brought in by the `distance` optional component + ]=]) set(code_string [=[ -if(faiss IN_LIST raft_FIND_COMPONENTS) - enable_language(CUDA) - if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/raft-faiss-dependencies.cmake") - include("${CMAKE_CURRENT_LIST_DIR}/raft-faiss-dependencies.cmake") +set(raft_allowed_components distance nn) +foreach(comp IN LISTS raft_allowed_components) + if(${comp} IN_LIST raft_FIND_COMPONENTS) + include("${CMAKE_CURRENT_LIST_DIR}/raft-distance-targets.cmake") + + if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/raft-${comp}-dependencies.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/raft-${comp}-dependencies.cmake") + endif() endif() + + enable_language(CUDA) + +endforeach() + +if(nn IN_LIST raft_FIND_COMPONENTS) if(TARGET faiss AND (NOT TARGET faiss::faiss)) add_library(faiss::faiss ALIAS faiss) elseif(TARGET faiss::faiss AND (NOT TARGET faiss)) add_library(faiss ALIAS faiss::faiss) endif() - if(TARGET raft::raft) - target_link_libraries(raft::raft INTERFACE faiss::faiss) - endif() endif() ]=] ) +# enable_cuda + rapids_export(INSTALL raft EXPORT_SET raft-exports - GLOBAL_TARGETS raft raft_distance raft_nn# since we can't hook into EXPORT SETS + GLOBAL_TARGETS raft raft_distance raft_nn NAMESPACE raft:: DOCUMENTATION doc_string FINAL_CODE_BLOCK code_string @@ -276,15 +299,38 @@ rapids_export(INSTALL raft rapids_export(BUILD raft EXPORT_SET raft-exports - GLOBAL_TARGETS raft raft_distance raft_nn# since we can't hook into EXPORT SETS + GLOBAL_TARGETS raft raft_distance raft_nn DOCUMENTATION doc_string NAMESPACE raft:: FINAL_CODE_BLOCK code_string ) -rapids_export_write_dependencies( - BUILD raft-faiss-exports "${PROJECT_BINARY_DIR}/raft-faiss-dependencies.cmake" -) +############################################################################## +# - export/install optional components -------------------------------------- + +include("${rapids-cmake-dir}/export/write_dependencies.cmake") + +set(raft_components distance nn) +foreach(comp IN LISTS raft_components) + install( + EXPORT raft-${comp}-exports + FILE raft-${comp}-targets.cmake + NAMESPACE raft:: + DESTINATION "${lib_dir}/cmake/raft" + ) + export( + EXPORT raft-${comp}-exports + FILE ${RAFT_BINARY_DIR}/raft-${comp}-targets.cmake + NAMESPACE raft:: + ) + rapids_export_write_dependencies( + BUILD raft-${comp}-exports "${PROJECT_BINARY_DIR}/raft-${comp}-dependencies.cmake" + ) + rapids_export_write_dependencies( + INSTALL raft-${comp}-exports "${PROJECT_BINARY_DIR}/rapids-cmake/raft/export/raft-${comp}-dependencies.cmake" + ) + +endforeach() ############################################################################## # - build test executable ---------------------------------------------------- diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake index 666619b2db..bb61a9f4e7 100644 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -31,7 +31,7 @@ function(find_and_configure_faiss) rapids_cpm_find(faiss ${PKG_VERSION} GLOBAL_TARGETS faiss::faiss - INSTALL_EXPORT_SET raft-faiss-exports + INSTALL_EXPORT_SET raft-nn-exports CPM_ARGS GIT_REPOSITORY https://github.com/facebookresearch/faiss.git GIT_TAG ${PKG_PINNED_TAG} @@ -56,12 +56,11 @@ function(find_and_configure_faiss) endif() # We generate the faiss-config files when we built faiss locally, so always do `find_dependency` - rapids_export_package(BUILD faiss raft-faiss-exports) - rapids_export_package(INSTALL faiss raft-faiss-exports) + rapids_export_package(BUILD faiss raft-nn-exports) # Tell cmake where it can find the generated faiss-config.cmake we wrote. include("${rapids-cmake-dir}/export/find_package_root.cmake") - rapids_export_find_package_root(BUILD faiss [=[${CMAKE_CURRENT_LIST_DIR}]=] raft-faiss-exports) + rapids_export_find_package_root(BUILD faiss [=[${CMAKE_CURRENT_LIST_DIR}]=] raft-nn-exports) endfunction() find_and_configure_faiss(VERSION 1.7.0 diff --git a/cpp/cmake/thirdparty/get_gtest.cmake b/cpp/cmake/thirdparty/get_gtest.cmake index 885c0f6bed..72fb0e18c6 100644 --- a/cpp/cmake/thirdparty/get_gtest.cmake +++ b/cpp/cmake/thirdparty/get_gtest.cmake @@ -18,7 +18,6 @@ function(find_and_configure_gtest ) include(${rapids-cmake-dir}/cpm/gtest.cmake) rapids_cpm_gtest(BUILD_EXPORT_SET raft-exports - INSTALL_EXPORT_SET raft-exports EXCLUDE_FROM_ALL TRUE) if(GTest_ADDED) diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index 77c394527d..59d34e7a2c 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -125,9 +125,9 @@ target_include_directories(test_raft target_link_libraries(test_raft PRIVATE raft::raft - raft::raft_distance - raft::raft_nn - # NCCL::NCCL + raft::distance + raft::nn + NCCL::NCCL faiss::faiss GTest::gtest GTest::gtest_main From 61e5b3cece0aa4c9625eeac131fe326c13114a97 Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Thu, 6 Jan 2022 10:34:54 -0500 Subject: [PATCH 42/66] Make searching for faiss a controllable option --- cpp/CMakeLists.txt | 4 +- cpp/cmake/thirdparty/get_faiss.cmake | 63 +++++++++++++++------------- 2 files changed, 34 insertions(+), 33 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 69f918e88b..c050872e18 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -50,6 +50,7 @@ option(DISABLE_OPENMP "Disable OpenMP" OFF) option(NVTX "Enable nvtx markers" OFF) option(RAFT_COMPILE_LIBRARIES "Enable building raft shared library instantiations" ON) +option(RAFT_ENABLE_NN_DEPENDENCIES "Search for raft::nn dependencies like faiss" ${RAFT_COMPILE_LIBRARIES}) include(CMakeDependentOption) cmake_dependent_option(RAFT_USE_FAISS_STATIC "Build and statically link the FAISS library for nearest neighbors search on GPU" ON RAFT_COMPILE_LIBRARIES OFF) @@ -60,8 +61,6 @@ message(VERBOSE "RAFT: Disable OpenMP: ${DISABLE_OPENMP}") message(VERBOSE "RAFT: Enable kernel resource usage info: ${CUDA_ENABLE_KERNELINFO}") message(VERBOSE "RAFT: Enable lineinfo in nvcc: ${CUDA_ENABLE_LINEINFO}") message(VERBOSE "RAFT: Enable nvtx markers: ${NVTX}") -message(VERBOSE "RAFT: Enable raft::nn target: ${RAFT_ENABLE_NN_COMPONENT}") -message(VERBOSE "RAFT: Enable raft::distance target: ${RAFT_ENABLE_DISTANCE_COMPONENT}") message(VERBOSE "RAFT: Statically link the CUDA runtime: ${CUDA_STATIC_RUNTIME}") # Set RMM logging level @@ -109,7 +108,6 @@ rapids_cpm_init() include(cmake/thirdparty/get_rmm.cmake) include(cmake/thirdparty/get_libcudacxx.cmake) include(cmake/thirdparty/get_cuco.cmake) -# CPMFindPackage(faiss) so it's available for raft consumers to use include(cmake/thirdparty/get_faiss.cmake) if(BUILD_TESTS) diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake index bb61a9f4e7..59c1d4ba2b 100644 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -19,44 +19,47 @@ function(find_and_configure_faiss) cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} ) - rapids_find_generate_module(faiss - HEADER_NAMES faiss/IndexFlat.h - LIBRARY_NAMES faiss - ) + if(RAFT_ENABLE_NN_DEPENDENCIES OR RAFT_COMPILE_LIBRARIES) + rapids_find_generate_module(faiss + HEADER_NAMES faiss/IndexFlat.h + LIBRARY_NAMES faiss + ) - set(BUILD_SHARED_LIBS OFF) - if (NOT BUILD_STATIC_LIBS) - set(BUILD_SHARED_LIBS ON) - endif() + set(BUILD_SHARED_LIBS OFF) + if (NOT BUILD_STATIC_LIBS) + set(BUILD_SHARED_LIBS ON) + endif() - rapids_cpm_find(faiss ${PKG_VERSION} - GLOBAL_TARGETS faiss::faiss - INSTALL_EXPORT_SET raft-nn-exports - CPM_ARGS - GIT_REPOSITORY https://github.com/facebookresearch/faiss.git - GIT_TAG ${PKG_PINNED_TAG} - EXCLUDE_FROM_ALL TRUE - OPTIONS - "FAISS_ENABLE_PYTHON OFF" - "CUDAToolkit_ROOT ${CUDAToolkit_LIBRARY_DIR}" - "FAISS_ENABLE_GPU ON" - "BUILD_TESTING OFF" - "CMAKE_MESSAGE_LOG_LEVEL VERBOSE" - ) + rapids_cpm_find(faiss ${PKG_VERSION} + GLOBAL_TARGETS faiss::faiss + INSTALL_EXPORT_SET raft-nn-exports + CPM_ARGS + GIT_REPOSITORY https://github.com/facebookresearch/faiss.git + GIT_TAG ${PKG_PINNED_TAG} + EXCLUDE_FROM_ALL TRUE + OPTIONS + "FAISS_ENABLE_PYTHON OFF" + "CUDAToolkit_ROOT ${CUDAToolkit_LIBRARY_DIR}" + "FAISS_ENABLE_GPU ON" + "BUILD_TESTING OFF" + "CMAKE_MESSAGE_LOG_LEVEL VERBOSE" + ) - if(TARGET faiss AND NOT TARGET faiss::faiss) - add_library(faiss::faiss ALIAS faiss) - endif() + if(TARGET faiss AND NOT TARGET faiss::faiss) + add_library(faiss::faiss ALIAS faiss) + endif() - if(faiss_ADDED) - rapids_export(BUILD faiss - EXPORT_SET faiss-targets - GLOBAL_TARGETS faiss - NAMESPACE faiss::) + if(faiss_ADDED) + rapids_export(BUILD faiss + EXPORT_SET faiss-targets + GLOBAL_TARGETS faiss + NAMESPACE faiss::) + endif() endif() # We generate the faiss-config files when we built faiss locally, so always do `find_dependency` rapids_export_package(BUILD faiss raft-nn-exports) + rapids_export_package(INSTALL faiss raft-nn-exports) # Tell cmake where it can find the generated faiss-config.cmake we wrote. include("${rapids-cmake-dir}/export/find_package_root.cmake") From 60c81a72982bb14e9d877f090e90472045b2d254 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 13 Jan 2022 10:47:28 -0800 Subject: [PATCH 43/66] fix typo --- cpp/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index c050872e18..f9f021b697 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -261,7 +261,7 @@ set(code_string set(raft_allowed_components distance nn) foreach(comp IN LISTS raft_allowed_components) if(${comp} IN_LIST raft_FIND_COMPONENTS) - include("${CMAKE_CURRENT_LIST_DIR}/raft-distance-targets.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/raft-${comp}-targets.cmake") if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/raft-${comp}-dependencies.cmake") include("${CMAKE_CURRENT_LIST_DIR}/raft-${comp}-dependencies.cmake") From d20e5efaf29d554eae972b0e0bd8aac8456436c5 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 13 Jan 2022 16:41:27 -0800 Subject: [PATCH 44/66] add GTest targets to install side of raft-exports export set --- cpp/cmake/thirdparty/get_gtest.cmake | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cpp/cmake/thirdparty/get_gtest.cmake b/cpp/cmake/thirdparty/get_gtest.cmake index 72fb0e18c6..d81e5e8606 100644 --- a/cpp/cmake/thirdparty/get_gtest.cmake +++ b/cpp/cmake/thirdparty/get_gtest.cmake @@ -27,10 +27,16 @@ function(find_and_configure_gtest ) GLOBAL_TARGETS gtest gmock gtest_main gmock_main NAMESPACE GTest::) + # We generate the GTest-config files when we built GTest locally, so always do `find_dependency` + rapids_export_package(BUILD GTest raft-exports) + + # Tell cmake where it can find the generated GTest-config.cmake we wrote. include("${rapids-cmake-dir}/export/find_package_root.cmake") rapids_export_find_package_root(BUILD GTest [=[${CMAKE_CURRENT_LIST_DIR}]=] raft-exports) endif() + rapids_export_package(INSTALL GTest raft-exports) + endfunction() find_and_configure_gtest() From 1594f7f91d62c597020b2eea264042999274a48e Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 13 Jan 2022 17:12:15 -0800 Subject: [PATCH 45/66] install gtest if not already installed --- cpp/cmake/thirdparty/get_gtest.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/cmake/thirdparty/get_gtest.cmake b/cpp/cmake/thirdparty/get_gtest.cmake index d81e5e8606..c28fd5d3fa 100644 --- a/cpp/cmake/thirdparty/get_gtest.cmake +++ b/cpp/cmake/thirdparty/get_gtest.cmake @@ -18,6 +18,7 @@ function(find_and_configure_gtest ) include(${rapids-cmake-dir}/cpm/gtest.cmake) rapids_cpm_gtest(BUILD_EXPORT_SET raft-exports + INSTALL_EXPORT_SET raft-exports EXCLUDE_FROM_ALL TRUE) if(GTest_ADDED) From 1c74204f9c4ad008e65c17a5989fd08d455b2634 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 14 Jan 2022 12:48:28 -0800 Subject: [PATCH 46/66] revert get_gtest.cmake changes --- cpp/cmake/thirdparty/get_gtest.cmake | 7 ------- 1 file changed, 7 deletions(-) diff --git a/cpp/cmake/thirdparty/get_gtest.cmake b/cpp/cmake/thirdparty/get_gtest.cmake index c28fd5d3fa..72fb0e18c6 100644 --- a/cpp/cmake/thirdparty/get_gtest.cmake +++ b/cpp/cmake/thirdparty/get_gtest.cmake @@ -18,7 +18,6 @@ function(find_and_configure_gtest ) include(${rapids-cmake-dir}/cpm/gtest.cmake) rapids_cpm_gtest(BUILD_EXPORT_SET raft-exports - INSTALL_EXPORT_SET raft-exports EXCLUDE_FROM_ALL TRUE) if(GTest_ADDED) @@ -28,16 +27,10 @@ function(find_and_configure_gtest ) GLOBAL_TARGETS gtest gmock gtest_main gmock_main NAMESPACE GTest::) - # We generate the GTest-config files when we built GTest locally, so always do `find_dependency` - rapids_export_package(BUILD GTest raft-exports) - - # Tell cmake where it can find the generated GTest-config.cmake we wrote. include("${rapids-cmake-dir}/export/find_package_root.cmake") rapids_export_find_package_root(BUILD GTest [=[${CMAKE_CURRENT_LIST_DIR}]=] raft-exports) endif() - rapids_export_package(INSTALL GTest raft-exports) - endfunction() find_and_configure_gtest() From 0bc750b14a220c4a9faea9f80d6f6cc86f3ca1ad Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 17 Jan 2022 14:08:20 -0800 Subject: [PATCH 47/66] remove raft_ prefix from global targets list --- cpp/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index f9f021b697..7c47de2ec3 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -286,7 +286,7 @@ endif() rapids_export(INSTALL raft EXPORT_SET raft-exports - GLOBAL_TARGETS raft raft_distance raft_nn + GLOBAL_TARGETS raft distance nn NAMESPACE raft:: DOCUMENTATION doc_string FINAL_CODE_BLOCK code_string @@ -297,7 +297,7 @@ rapids_export(INSTALL raft rapids_export(BUILD raft EXPORT_SET raft-exports - GLOBAL_TARGETS raft raft_distance raft_nn + GLOBAL_TARGETS raft distance nn DOCUMENTATION doc_string NAMESPACE raft:: FINAL_CODE_BLOCK code_string From a3c0370b65b5ff4f88161335cdffe543808a8feb Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 17 Jan 2022 15:45:30 -0800 Subject: [PATCH 48/66] revert previous change, move target_link_libraries below add_library calls that reference the target, fix typo raft_distance_lib -> raft_nn_lib --- cpp/CMakeLists.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 7c47de2ec3..8b0e0d7193 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -145,8 +145,6 @@ add_library(raft_distance INTERFACE) add_library(raft::distance ALIAS raft_distance) set_target_properties(raft_distance PROPERTIES EXPORT_NAME distance) -target_link_libraries(raft_distance INTERFACE raft::raft $) - if(RAFT_COMPILE_LIBRARIES) add_library(raft_distance_lib SHARED src/distance/specializations/detail @@ -180,14 +178,14 @@ if(RAFT_COMPILE_LIBRARIES) EXPORT raft-distance-exports) endif() +target_link_libraries(raft_distance INTERFACE raft::raft $) + ############################################################################## # - raft_nn ------------------------------------------------------------------ add_library(raft_nn INTERFACE) add_library(raft::nn ALIAS raft_nn) set_target_properties(raft_nn PROPERTIES EXPORT_NAME nn) -target_link_libraries(raft_nn INTERFACE raft::raft faiss::faiss $) - if(RAFT_COMPILE_LIBRARIES) add_library(raft_nn_lib SHARED src/nn/specializations/ball_cover.cu @@ -202,7 +200,7 @@ if(RAFT_COMPILE_LIBRARIES) PRIVATE "$<$:${RAFT_CXX_FLAGS}>" "$<$:${RAFT_CUDA_FLAGS}>" ) - target_compile_definitions(raft_distance_lib + target_compile_definitions(raft_nn_lib INTERFACE "RAFT_NN_COMPILED") install(TARGETS raft_nn_lib @@ -210,6 +208,8 @@ if(RAFT_COMPILE_LIBRARIES) EXPORT raft-nn-exports) endif() +target_link_libraries(raft_nn INTERFACE raft::raft faiss::faiss $) + ############################################################################## # - install targets----------------------------------------------------------- rapids_cmake_install_lib_dir( lib_dir ) @@ -286,7 +286,7 @@ endif() rapids_export(INSTALL raft EXPORT_SET raft-exports - GLOBAL_TARGETS raft distance nn + GLOBAL_TARGETS raft raft_distance raft_nn NAMESPACE raft:: DOCUMENTATION doc_string FINAL_CODE_BLOCK code_string @@ -297,7 +297,7 @@ rapids_export(INSTALL raft rapids_export(BUILD raft EXPORT_SET raft-exports - GLOBAL_TARGETS raft distance nn + GLOBAL_TARGETS raft raft_distance raft_nn DOCUMENTATION doc_string NAMESPACE raft:: FINAL_CODE_BLOCK code_string From ea209c14f57ddb80ead1a5b9ab0495856f2dc59e Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 17 Jan 2022 17:29:18 -0800 Subject: [PATCH 49/66] fix _lib name to add prefix --- cpp/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 8b0e0d7193..8870b1b1e4 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -178,7 +178,7 @@ if(RAFT_COMPILE_LIBRARIES) EXPORT raft-distance-exports) endif() -target_link_libraries(raft_distance INTERFACE raft::raft $) +target_link_libraries(raft_distance INTERFACE raft::raft $) ############################################################################## # - raft_nn ------------------------------------------------------------------ @@ -208,7 +208,7 @@ if(RAFT_COMPILE_LIBRARIES) EXPORT raft-nn-exports) endif() -target_link_libraries(raft_nn INTERFACE raft::raft faiss::faiss $) +target_link_libraries(raft_nn INTERFACE raft::raft faiss::faiss $) ############################################################################## # - install targets----------------------------------------------------------- From 7093ad50f6ac37a0f3b9d3bb7ab0c2163b282540 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 17 Jan 2022 20:50:37 -0800 Subject: [PATCH 50/66] add missing PKG_ prefix --- cpp/cmake/thirdparty/get_faiss.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake index 59c1d4ba2b..46ce522f8c 100644 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -26,7 +26,7 @@ function(find_and_configure_faiss) ) set(BUILD_SHARED_LIBS OFF) - if (NOT BUILD_STATIC_LIBS) + if (NOT PKG_BUILD_STATIC_LIBS) set(BUILD_SHARED_LIBS ON) endif() From f613e39cb8c99e8a0b0da71800dbbfa2a4b5ca8c Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 17 Jan 2022 20:53:29 -0800 Subject: [PATCH 51/66] guard creating alias targets when added as a submodule via CPM --- cpp/CMakeLists.txt | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 8870b1b1e4..257adf76b5 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -142,7 +142,11 @@ target_compile_features(raft INTERFACE cxx_std_17 $ ############################################################################## # - raft_distance ------------------------------------------------------------ add_library(raft_distance INTERFACE) -add_library(raft::distance ALIAS raft_distance) + +if(TARGET raft_distance AND (NOT TARGET raft::distance)) + add_library(raft::distance ALIAS raft_distance) +endif() + set_target_properties(raft_distance PROPERTIES EXPORT_NAME distance) if(RAFT_COMPILE_LIBRARIES) @@ -183,7 +187,11 @@ target_link_libraries(raft_distance INTERFACE raft::raft $ Date: Wed, 19 Jan 2022 11:00:20 -0800 Subject: [PATCH 52/66] update cuco hash --- cpp/cmake/thirdparty/get_cuco.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/cmake/thirdparty/get_cuco.cmake b/cpp/cmake/thirdparty/get_cuco.cmake index fdac7cda11..92d20a27fa 100644 --- a/cpp/cmake/thirdparty/get_cuco.cmake +++ b/cpp/cmake/thirdparty/get_cuco.cmake @@ -22,7 +22,7 @@ function(find_and_configure_cuco VERSION) INSTALL_EXPORT_SET raft-exports CPM_ARGS GIT_REPOSITORY https://github.com/NVIDIA/cuCollections.git - GIT_TAG f0eecb203590f1f4ac4a9f1700229f4434ac64dc + GIT_TAG 0ca860b824f5dc22cf8a41f09912e62e11f07d82 OPTIONS "BUILD_TESTS OFF" "BUILD_BENCHMARKS OFF" "BUILD_EXAMPLES OFF" From 902657ceb030f4f41f06048c5dc9b220dc1b83e8 Mon Sep 17 00:00:00 2001 From: Ashwin Srinath Date: Wed, 19 Jan 2022 15:34:34 -0500 Subject: [PATCH 53/66] Replace the use of RMM's CUDA Python bindings with those from CUDA-Python --- python/raft/common/cuda.pxd | 24 +++++----------------- python/raft/common/cuda.pyx | 38 ++++++++++++++++++++--------------- python/raft/common/handle.pxd | 3 +-- python/raft/common/handle.pyx | 9 +++++---- 4 files changed, 33 insertions(+), 41 deletions(-) diff --git a/python/raft/common/cuda.pxd b/python/raft/common/cuda.pxd index e407213f44..0459cb96af 100644 --- a/python/raft/common/cuda.pxd +++ b/python/raft/common/cuda.pxd @@ -1,5 +1,5 @@ # -# Copyright (c) 2019, NVIDIA CORPORATION. +# Copyright (c) 2019-2022, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,23 +14,9 @@ # limitations under the License. # -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 +from cuda.ccudart cimport cudaStream_t +cdef class Stream: + cdef cudaStream_t s -# Populate this with more typedef's (eg: events) as and when needed -cdef extern from * nogil: - ctypedef void* _Stream "cudaStream_t" - ctypedef int _Error "cudaError_t" - - -# Populate this with more runtime api method declarations as and when needed -cdef extern from "cuda_runtime_api.h" nogil: - _Error cudaStreamCreate(_Stream* s) - _Error cudaStreamDestroy(_Stream s) - _Error cudaStreamSynchronize(_Stream s) - _Error cudaGetLastError() - const char* cudaGetErrorString(_Error e) - const char* cudaGetErrorName(_Error e) + cdef cudaStream_t getStream(self) diff --git a/python/raft/common/cuda.pyx b/python/raft/common/cuda.pyx index 0b97eeba67..d00fd88af0 100644 --- a/python/raft/common/cuda.pyx +++ b/python/raft/common/cuda.pyx @@ -1,5 +1,5 @@ # -# Copyright (c) 2020-2021, NVIDIA CORPORATION. +# Copyright (c) 2020-2022, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,10 +19,21 @@ # cython: embedsignature = True # cython: language_level = 3 +from cuda.ccudart cimport ( + cudaStream_t, + cudaError_t, + cudaSuccess, + cudaStreamCreate, + cudaStreamDestroy, + cudaStreamSynchronize, + cudaGetLastError, + cudaGetErrorString, + cudaGetErrorName +) class CudaRuntimeError(RuntimeError): def __init__(self, extraMsg=None): - cdef _Error e = cudaGetLastError() + cdef cudaError_t e = cudaGetLastError() cdef bytes errMsg = cudaGetErrorString(e) cdef bytes errName = cudaGetErrorName(e) msg = "Error! %s reason='%s'" % (errName.decode(), errMsg.decode()) @@ -47,26 +58,22 @@ cdef class Stream: """ # NOTE: - # If we store _Stream directly, this always leads to the following error: - # "Cannot convert Python object to '_Stream'" + # If we store cudaStream_t directly, this always leads to the following error: + # "Cannot convert Python object to 'cudaStream_t'" # I was unable to find a good solution to this in reasonable time. Also, # since cudaStream_t is a pointer anyways, storing it as an integer should # be just fine (although, that certainly is ugly and hacky!). - cdef size_t s def __cinit__(self): - if self.s != 0: - return - cdef _Stream stream - cdef _Error e = cudaStreamCreate(&stream) - if e != 0: + cdef cudaStream_t stream + cdef cudaError_t e = cudaStreamCreate(&stream) + if e != cudaSuccess: raise CudaRuntimeError("Stream create") - self.s = stream + self.s = stream def __dealloc__(self): self.sync() - cdef _Stream stream = <_Stream>self.s - cdef _Error e = cudaStreamDestroy(stream) + cdef cudaError_t e = cudaStreamDestroy(self.s) if e != 0: raise CudaRuntimeError("Stream destroy") @@ -76,10 +83,9 @@ cdef class Stream: could raise exception due to issues with previous asynchronous launches """ - cdef _Stream stream = <_Stream>self.s - cdef _Error e = cudaStreamSynchronize(stream) + cdef cudaError_t e = cudaStreamSynchronize(self.s) if e != 0: raise CudaRuntimeError("Stream sync") - def getStream(self): + cdef cudaStream_t getStream(self): return self.s diff --git a/python/raft/common/handle.pxd b/python/raft/common/handle.pxd index d2ae0a401d..8415b7e3d7 100644 --- a/python/raft/common/handle.pxd +++ b/python/raft/common/handle.pxd @@ -1,5 +1,5 @@ # -# Copyright (c) 2020, NVIDIA CORPORATION. +# Copyright (c) 2020-2022, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -21,7 +21,6 @@ from libcpp.memory cimport shared_ptr -from .cuda cimport _Stream from rmm._lib.cuda_stream_view cimport cuda_stream_view from rmm._lib.cuda_stream_pool cimport cuda_stream_pool from libcpp.memory cimport shared_ptr diff --git a/python/raft/common/handle.pyx b/python/raft/common/handle.pyx index 1accf9e679..c4faabb610 100644 --- a/python/raft/common/handle.pyx +++ b/python/raft/common/handle.pyx @@ -1,5 +1,5 @@ # -# Copyright (c) 2020, NVIDIA CORPORATION. +# Copyright (c) 2020-2022, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -24,9 +24,10 @@ from libcpp.memory cimport shared_ptr from rmm._lib.cuda_stream_view cimport cuda_stream_per_thread from rmm._lib.cuda_stream_view cimport cuda_stream_view -from .cuda cimport _Stream, _Error, cudaStreamSynchronize +from .cuda cimport Stream from .cuda import CudaRuntimeError + cdef class Handle: """ Handle is a lightweight python wrapper around the corresponding C++ class @@ -51,7 +52,7 @@ cdef class Handle: del handle # optional! """ - def __cinit__(self, stream=None, n_streams=0): + def __cinit__(self, stream: Stream=None, n_streams=0): self.n_streams = n_streams if n_streams > 0: self.stream_pool.reset(new cuda_stream_pool(n_streams)) @@ -64,7 +65,7 @@ cdef class Handle: self.stream_pool)) else: # this constructor constructs a handle on user stream - c_stream = cuda_stream_view(<_Stream> stream.getStream()) + c_stream = cuda_stream_view(stream.getStream()) self.c_obj.reset(new handle_t(c_stream, self.stream_pool)) From d7f00745fcc34bd15819998683e7d703450abd9d Mon Sep 17 00:00:00 2001 From: Ashwin Srinath Date: Wed, 19 Jan 2022 15:44:07 -0500 Subject: [PATCH 54/66] Add cuda-python to dev env --- conda/environments/raft_dev_cuda11.5.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/conda/environments/raft_dev_cuda11.5.yml b/conda/environments/raft_dev_cuda11.5.yml index 152f3a8db5..c6d9f3fbf5 100644 --- a/conda/environments/raft_dev_cuda11.5.yml +++ b/conda/environments/raft_dev_cuda11.5.yml @@ -6,6 +6,7 @@ channels: - conda-forge dependencies: - cudatoolkit=11.5 +- cuda-python >=11.5,<12.0 - clang=11.1.0 - clang-tools=11.1.0 - rapids-build-env=22.02.* From c7666ad76d8968a7d4c7213554b79dbfdc561ed8 Mon Sep 17 00:00:00 2001 From: Ashwin Srinath Date: Wed, 19 Jan 2022 15:51:16 -0500 Subject: [PATCH 55/66] Remove older comment about cudaStream_t attribute --- python/raft/common/cuda.pyx | 8 -------- 1 file changed, 8 deletions(-) diff --git a/python/raft/common/cuda.pyx b/python/raft/common/cuda.pyx index d00fd88af0..8d2f6d8c69 100644 --- a/python/raft/common/cuda.pyx +++ b/python/raft/common/cuda.pyx @@ -56,14 +56,6 @@ cdef class Stream: stream.sync() del stream # optional! """ - - # NOTE: - # If we store cudaStream_t directly, this always leads to the following error: - # "Cannot convert Python object to 'cudaStream_t'" - # I was unable to find a good solution to this in reasonable time. Also, - # since cudaStream_t is a pointer anyways, storing it as an integer should - # be just fine (although, that certainly is ugly and hacky!). - def __cinit__(self): cdef cudaStream_t stream cdef cudaError_t e = cudaStreamCreate(&stream) From 8446e2f022b22235f15d524c53e73011d4486b06 Mon Sep 17 00:00:00 2001 From: Ashwin Srinath Date: Wed, 19 Jan 2022 16:31:58 -0500 Subject: [PATCH 56/66] Style --- python/raft/common/cuda.pyx | 3 ++- python/raft/common/handle.pyx | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/python/raft/common/cuda.pyx b/python/raft/common/cuda.pyx index 8d2f6d8c69..b5be977bcc 100644 --- a/python/raft/common/cuda.pyx +++ b/python/raft/common/cuda.pyx @@ -19,7 +19,7 @@ # cython: embedsignature = True # cython: language_level = 3 -from cuda.ccudart cimport ( +from cuda.ccudart cimport( cudaStream_t, cudaError_t, cudaSuccess, @@ -31,6 +31,7 @@ from cuda.ccudart cimport ( cudaGetErrorName ) + class CudaRuntimeError(RuntimeError): def __init__(self, extraMsg=None): cdef cudaError_t e = cudaGetLastError() diff --git a/python/raft/common/handle.pyx b/python/raft/common/handle.pyx index c4faabb610..661c5b5f23 100644 --- a/python/raft/common/handle.pyx +++ b/python/raft/common/handle.pyx @@ -52,7 +52,7 @@ cdef class Handle: del handle # optional! """ - def __cinit__(self, stream: Stream=None, n_streams=0): + def __cinit__(self, stream: Stream = None, n_streams=0): self.n_streams = n_streams if n_streams > 0: self.stream_pool.reset(new cuda_stream_pool(n_streams)) From 4de51ed3b0e37c6ca868d68399d3e0301447d838 Mon Sep 17 00:00:00 2001 From: Ashwin Srinath Date: Wed, 19 Jan 2022 16:33:01 -0500 Subject: [PATCH 57/66] 0 -> cudaSuccess --- python/raft/common/cuda.pyx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/raft/common/cuda.pyx b/python/raft/common/cuda.pyx index b5be977bcc..c3c90936aa 100644 --- a/python/raft/common/cuda.pyx +++ b/python/raft/common/cuda.pyx @@ -67,7 +67,7 @@ cdef class Stream: def __dealloc__(self): self.sync() cdef cudaError_t e = cudaStreamDestroy(self.s) - if e != 0: + if e != cudaSuccess: raise CudaRuntimeError("Stream destroy") def sync(self): @@ -77,7 +77,7 @@ cdef class Stream: launches """ cdef cudaError_t e = cudaStreamSynchronize(self.s) - if e != 0: + if e != cudaSuccess: raise CudaRuntimeError("Stream sync") cdef cudaStream_t getStream(self): From 9d1bdc54ea98f040c1e50954a8d1e95f7ab61c1c Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Wed, 19 Jan 2022 17:27:40 -0500 Subject: [PATCH 58/66] Add raft_export while rapids-cmake adds a export(COMPONENT) feature --- cpp/CMakeLists.txt | 28 ++-- cpp/cmake/modules/config.cmake.in | 122 ++++++++++++++ cpp/cmake/modules/raft_export.cmake | 238 +++++++++++++++++++++++++++ cpp/cmake/thirdparty/get_faiss.cmake | 1 + 4 files changed, 372 insertions(+), 17 deletions(-) create mode 100644 cpp/cmake/modules/config.cmake.in create mode 100644 cpp/cmake/modules/raft_export.cmake diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 257adf76b5..7e01f19190 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -266,21 +266,13 @@ Imported Targets: set(code_string [=[ -set(raft_allowed_components distance nn) -foreach(comp IN LISTS raft_allowed_components) - if(${comp} IN_LIST raft_FIND_COMPONENTS) - include("${CMAKE_CURRENT_LIST_DIR}/raft-${comp}-targets.cmake") - - if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/raft-${comp}-dependencies.cmake") - include("${CMAKE_CURRENT_LIST_DIR}/raft-${comp}-dependencies.cmake") - endif() - endif() - +if(distance IN_LIST raft_FIND_COMPONENTS) enable_language(CUDA) - -endforeach() +endif() if(nn IN_LIST raft_FIND_COMPONENTS) + enable_language(CUDA) + if(TARGET faiss AND (NOT TARGET faiss::faiss)) add_library(faiss::faiss ALIAS faiss) elseif(TARGET faiss::faiss AND (NOT TARGET faiss)) @@ -290,11 +282,12 @@ endif() ]=] ) -# enable_cuda - -rapids_export(INSTALL raft +# Use `rapids_export` for 22.04 as it will have COMPONENT support +include(cmake/modules/raft_export.cmake) +raft_export(INSTALL raft EXPORT_SET raft-exports - GLOBAL_TARGETS raft raft_distance raft_nn + COMPONENTS nn distance + GLOBAL_TARGETS raft nn distance NAMESPACE raft:: DOCUMENTATION doc_string FINAL_CODE_BLOCK code_string @@ -303,8 +296,9 @@ rapids_export(INSTALL raft ############################################################################## # - build export ------------------------------------------------------------- -rapids_export(BUILD raft +raft_export(BUILD raft EXPORT_SET raft-exports + COMPONENTS nn distance GLOBAL_TARGETS raft raft_distance raft_nn DOCUMENTATION doc_string NAMESPACE raft:: diff --git a/cpp/cmake/modules/config.cmake.in b/cpp/cmake/modules/config.cmake.in new file mode 100644 index 0000000000..cde26e1da8 --- /dev/null +++ b/cpp/cmake/modules/config.cmake.in @@ -0,0 +1,122 @@ +#============================================================================= +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +#[=======================================================================[ + +@RAPIDS_PROJECT_DOCUMENTATION@ + +Result Variables +^^^^^^^^^^^^^^^^ + +This module will set the following variables:: + + @project_name_uppercase@_FOUND + @project_name_uppercase@_VERSION + @project_name_uppercase@_VERSION_MAJOR + @project_name_uppercase@_VERSION_MINOR + +#]=======================================================================] + +@PACKAGE_INIT@ + +cmake_minimum_required(VERSION @CMAKE_MINIMUM_REQUIRED_VERSION@) + +set(rapids_global_languages @RAPIDS_LANGUAGES@) +foreach(lang IN LISTS rapids_global_languages) + include("${CMAKE_CURRENT_LIST_DIR}/@project_name@-${lang}-language.cmake") +endforeach() +unset(rapids_global_languages) + +set(rapids_allowed_components @RAPIDS_COMPONENTS@) + +if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/@project_name@-dependencies.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/@project_name@-dependencies.cmake") +endif() +foreach(comp IN LISTS rapids_allowed_components) + # find dependencies before creating targets that use them + # this way if a dependency can't be found we fail + if(${comp} IN_LIST @project_name@_FIND_COMPONENTS AND + EXISTS "${CMAKE_CURRENT_LIST_DIR}/@project_name@-${comp}-dependencies.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/@project_name@-${comp}-dependencies.cmake") + endif() +endforeach() + +if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/@project_name@-targets.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/@project_name@-targets.cmake") +endif() +foreach(comp IN LISTS rapids_allowed_components) + if(${comp} IN_LIST @project_name@_FIND_COMPONENTS AND + EXISTS "${CMAKE_CURRENT_LIST_DIR}/@project_name@-${comp}-targets.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/@project_name@-${comp}-targets.cmake") + endif() +endforeach() + +unset(rapids_allowed_components) + +if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/@project_name@-config-version.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/@project_name@-config-version.cmake") +endif() + +# Set our version variables +set(@project_name_uppercase@_VERSION_MAJOR @rapids_orig_major_version@) +set(@project_name_uppercase@_VERSION_MINOR @rapids_orig_minor_version@) +set(@project_name_uppercase@_VERSION_PATCH @rapids_orig_patch_version@) +set(@project_name_uppercase@_VERSION @rapids_orig_version@) + + +set(rapids_global_targets @RAPIDS_GLOBAL_TARGETS@) +set(rapids_namespaced_global_targets @RAPIDS_GLOBAL_TARGETS@) +if(rapids_namespaced_global_targets) + list(TRANSFORM rapids_namespaced_global_targets PREPEND @RAPIDS_NAMESPACE@ ) +endif() + +foreach(target IN LISTS rapids_namespaced_global_targets) + if(TARGET ${target}) + get_target_property(_is_imported ${target} IMPORTED) + get_target_property(_already_global ${target} IMPORTED_GLOBAL) + if(_is_imported AND NOT _already_global) + set_target_properties(${target} PROPERTIES IMPORTED_GLOBAL TRUE) + endif() + endif() +endforeach() + +# For backwards compat +if("rapids_config_@type@" STREQUAL "rapids_config_build") + foreach(target IN LISTS rapids_global_targets) + if(TARGET ${target}) + get_target_property(_is_imported ${target} IMPORTED) + get_target_property(_already_global ${target} IMPORTED_GLOBAL) + if(_is_imported AND NOT _already_global) + set_target_properties(${target} PROPERTIES IMPORTED_GLOBAL TRUE) + endif() + if(NOT TARGET @RAPIDS_NAMESPACE@${target}) + add_library(@RAPIDS_NAMESPACE@${target} ALIAS ${target}) + endif() + endif() + endforeach() +endif() + +unset(rapids_global_targets) +unset(rapids_namespaced_global_targets) + +check_required_components(@project_name@) + +set(${CMAKE_FIND_PACKAGE_NAME}_CONFIG "${CMAKE_CURRENT_LIST_FILE}") + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(${CMAKE_FIND_PACKAGE_NAME} CONFIG_MODE) + +@RAPIDS_PROJECT_FINAL_CODE_BLOCK@ diff --git a/cpp/cmake/modules/raft_export.cmake b/cpp/cmake/modules/raft_export.cmake new file mode 100644 index 0000000000..4411433336 --- /dev/null +++ b/cpp/cmake/modules/raft_export.cmake @@ -0,0 +1,238 @@ +#============================================================================= +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= +include_guard(GLOBAL) + +#[=======================================================================[.rst: +raft_export +--------------------- + +Generate a projects -Config.cmake module and all related information + +.. code-block:: cmake + + raft_export( (BUILD|INSTALL) + EXPORT_SET + [ COMPONENTS ] + [ GLOBAL_TARGETS ] + [ VERSION ] + [ NAMESPACE ] + [ DOCUMENTATION ] + [ FINAL_CODE_BLOCK ] + [ LANGUAGES ] + ) + +The :cmake:command:`raft_export` function allow projects to easily generate a fully +correct build and install tree `Project-Config.cmake` module including any necessary +calls to :cmake:command:`find_dependency`, or :cmake:command:`CPMFindPackage`. + +.. note:: + :cmake:command:`raft_export` always installs to `lib` and doesn't use GNUInstallDirs + + The files generated by :cmake:command:`raft_export` are completly standalone + and don't require the consuming package to use `rapids-cmake` + +``project_name`` + Name of the project, to be used by consumers when using `find_package` + +``GLOBAL_TARGETS`` + Explicitly list what targets should be made globally visibile to + the consuming project. + +``VERSION`` + Explicitly list the version of the package being exported. By + default :cmake:command:`raft_export` uses the version specified by the + root level :cmake:command:`project` call. If no version has been specified + either way or `OFF` is provided as the `VERSION` value, no version compatibility + checks will be generated. + + Depending on the version string different compatibility modes will be used. + + +------------------+---------------------+ + | Version String | Compatiblity Type | + +==================+=====================+ + | None | No checks perfomed | + +------------------+---------------------+ + | X | SameMajorVersion | + +------------------+---------------------+ + | X.Y | SameMinorVersion | + +------------------+---------------------+ + | X.Y.Z | SameMinorVersion | + +------------------+---------------------+ + +.. note:: + It can be useful to explicitly specify a version string when generating + export rules for a sub-component of alarger project, or an external + project that doesn't have export rules. + +``NAMESPACE`` + Optional value to specify what namespace all targets from the + EXPORT_SET will be placed into. When provided must match the pattern + of `::`. + If not provided all targets will be placed in the `::` + namespace + + Note: When exporting with `BUILD` type, only `GLOBAL_TARGETS` will + be placed in the namespace. + +``DOCUMENTATION`` + Optional value of the variable that holds the documentation + for this config file. + + Note: This requires the documentation variable instead of the contents + so we can handle having CMake code inside the documentation + +``FINAL_CODE_BLOCK`` + Optional value of the variable that holds a string of code that will + be executed at the last step of this config file. + + Note: This requires the code block variable instead of the contents + so that we can properly insert CMake code + +``LANGUAGES`` + Non default languages, such as CUDA that are required by consumers + of your package. This makes sure all consumers properly setup these + languages correctly. + + This is required as CMake's :cmake:command:`enable_language` only supports + enabling languages for the current directory scope, and doesn't support + being called from within functions. Marking languages here overcomes + these limitations and makes it possible for packages included via + `CPM` to enable languages. + + +#]=======================================================================] +# cmake-lint: disable=R0912,R0915,W0105 +function(raft_export type project_name) + include(CMakePackageConfigHelpers) + + list(APPEND CMAKE_MESSAGE_CONTEXT "raft.export") + string(TOLOWER ${type} type) + + set(options "") + set(one_value EXPORT_SET VERSION NAMESPACE DOCUMENTATION FINAL_CODE_BLOCK) + set(multi_value COMPONENTS GLOBAL_TARGETS LANGUAGES) + cmake_parse_arguments(RAPIDS "${options}" "${one_value}" "${multi_value}" ${ARGN}) + + set(rapids_version_set ON) + if(DEFINED RAPIDS_VERSION AND NOT RAPIDS_VERSION) + # We need to capture `VERSION OFF` so we need to make sure it has an off value, and not just + # undefined + set(rapids_version_set OFF) + unset(RAPIDS_VERSION) # unset this so we don't export a version value of `OFF` + elseif(NOT DEFINED RAPIDS_VERSION AND NOT DEFINED PROJECT_VERSION) + set(rapids_version_set OFF) + elseif(DEFINED PROJECT_VERSION AND NOT DEFINED RAPIDS_VERSION) + # Choose the project version when an explicit version isn't provided + set(RAPIDS_VERSION "${PROJECT_VERSION}") + endif() + + if(rapids_version_set) + include("${rapids-cmake-dir}/export/detail/parse_version.cmake") + rapids_export_parse_version(${RAPIDS_VERSION} rapids_orig rapids_project_version) + endif() + + set(RAPIDS_PROJECT_VERSION "${project_name}::") + if(DEFINED RAPIDS_NAMESPACE) + set(RAPIDS_PROJECT_VERSION ${RAPIDS_NAMESPACE}) + endif() + + set(RAPIDS_PROJECT_DOCUMENTATION "Generated ${project_name}-config module") + if(DEFINED RAPIDS_DOCUMENTATION) + if(NOT DEFINED ${RAPIDS_DOCUMENTATION}) + message(FATAL_ERROR "DOCUMENTATION variable `${RAPIDS_DOCUMENTATION}` doesn't exist") + endif() + set(RAPIDS_PROJECT_DOCUMENTATION "${${RAPIDS_DOCUMENTATION}}") + endif() + + if(DEFINED RAPIDS_FINAL_CODE_BLOCK) + if(NOT DEFINED ${RAPIDS_FINAL_CODE_BLOCK}) + message(FATAL_ERROR "FINAL_CODE_BLOCK variable `${RAPIDS_FINAL_CODE_BLOCK}` doesn't exist") + endif() + set(RAPIDS_PROJECT_FINAL_CODE_BLOCK "${${RAPIDS_FINAL_CODE_BLOCK}}") + endif() + + # Write configuration and version files + string(TOLOWER ${project_name} project_name) + string(TOUPPER ${project_name} project_name_uppercase) + if(type STREQUAL "install") + include("${rapids-cmake-dir}/cmake/install_lib_dir.cmake") + rapids_cmake_install_lib_dir(install_location) + set(install_location "${install_location}/cmake/${project_name}") + + set(scratch_dir "${PROJECT_BINARY_DIR}/rapids-cmake/${project_name}/export") + + configure_package_config_file("${CMAKE_CURRENT_FUNCTION_LIST_DIR}/config.cmake.in" + "${scratch_dir}/${project_name}-config.cmake" + INSTALL_DESTINATION "${install_location}") + + if(rapids_version_set) + write_basic_package_version_file( + "${scratch_dir}/${project_name}-config-version.cmake" VERSION ${rapids_project_version} + COMPATIBILITY ${rapids_project_version_compat}) + endif() + + install(EXPORT ${RAPIDS_EXPORT_SET} FILE ${project_name}-targets.cmake + NAMESPACE ${RAPIDS_PROJECT_VERSION} DESTINATION "${install_location}") + + if(TARGET rapids_export_install_${RAPIDS_EXPORT_SET}) + include("${rapids-cmake-dir}/export/write_dependencies.cmake") + set(destination "${scratch_dir}/${project_name}-dependencies.cmake") + rapids_export_write_dependencies(INSTALL ${RAPIDS_EXPORT_SET} "${destination}") + endif() + + if(DEFINED RAPIDS_LANGUAGES) + include("${rapids-cmake-dir}/export/write_language.cmake") + foreach(lang IN LISTS RAPIDS_LANGUAGES) + set(destination "${scratch_dir}/${project_name}-${lang}-language.cmake") + rapids_export_write_language(INSTALL ${lang} "${destination}") + endforeach() + endif() + + # Install everything we have generated + install(DIRECTORY "${scratch_dir}/" DESTINATION "${install_location}") + + else() + set(install_location "${PROJECT_BINARY_DIR}") + configure_package_config_file("${CMAKE_CURRENT_FUNCTION_LIST_DIR}/config.cmake.in" + "${install_location}/${project_name}-config.cmake" + INSTALL_DESTINATION "${install_location}") + + if(rapids_version_set) + write_basic_package_version_file( + "${install_location}/${project_name}-config-version.cmake" VERSION ${rapids_project_version} + COMPATIBILITY ${rapids_project_version_compat}) + endif() + + export(EXPORT ${RAPIDS_EXPORT_SET} NAMESPACE ${RAPIDS_PROJECT_VERSION} + FILE "${install_location}/${project_name}-targets.cmake") + + if(TARGET rapids_export_build_${RAPIDS_EXPORT_SET}) + include("${rapids-cmake-dir}/export/write_dependencies.cmake") + rapids_export_write_dependencies(BUILD ${RAPIDS_EXPORT_SET} + "${install_location}/${project_name}-dependencies.cmake") + endif() + + if(DEFINED RAPIDS_LANGUAGES) + include("${rapids-cmake-dir}/export/write_language.cmake") + foreach(lang IN LISTS RAPIDS_LANGUAGES) + rapids_export_write_language(BUILD ${lang} + "${install_location}/${project_name}-${lang}-language.cmake") + endforeach() + endif() + + endif() + +endfunction() diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake index 46ce522f8c..1079db3294 100644 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -58,6 +58,7 @@ function(find_and_configure_faiss) endif() # We generate the faiss-config files when we built faiss locally, so always do `find_dependency` + rapids_export_package(BUILD OpenMP raft-nn-exports) # faiss uses openMP but doesn't export a need for it rapids_export_package(BUILD faiss raft-nn-exports) rapids_export_package(INSTALL faiss raft-nn-exports) From 1a2c611fb6b7857d4ea81aa4264554b148cafcda Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Wed, 19 Jan 2022 17:48:56 -0500 Subject: [PATCH 59/66] raft_export marks components as found --- cpp/cmake/modules/config.cmake.in | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/cpp/cmake/modules/config.cmake.in b/cpp/cmake/modules/config.cmake.in index cde26e1da8..bfafe3555a 100644 --- a/cpp/cmake/modules/config.cmake.in +++ b/cpp/cmake/modules/config.cmake.in @@ -48,19 +48,17 @@ endif() foreach(comp IN LISTS rapids_allowed_components) # find dependencies before creating targets that use them # this way if a dependency can't be found we fail - if(${comp} IN_LIST @project_name@_FIND_COMPONENTS AND - EXISTS "${CMAKE_CURRENT_LIST_DIR}/@project_name@-${comp}-dependencies.cmake") - include("${CMAKE_CURRENT_LIST_DIR}/@project_name@-${comp}-dependencies.cmake") + if(${comp} IN_LIST @project_name@_FIND_COMPONENTS) + include("${CMAKE_CURRENT_LIST_DIR}/@project_name@-${comp}-dependencies.cmake" OPTIONAL) endif() endforeach() -if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/@project_name@-targets.cmake") - include("${CMAKE_CURRENT_LIST_DIR}/@project_name@-targets.cmake") -endif() +include("${CMAKE_CURRENT_LIST_DIR}/@project_name@-targets.cmake" OPTIONAL) + foreach(comp IN LISTS rapids_allowed_components) - if(${comp} IN_LIST @project_name@_FIND_COMPONENTS AND - EXISTS "${CMAKE_CURRENT_LIST_DIR}/@project_name@-${comp}-targets.cmake") - include("${CMAKE_CURRENT_LIST_DIR}/@project_name@-${comp}-targets.cmake") + if(${comp} IN_LIST @project_name@_FIND_COMPONENTS) + include("${CMAKE_CURRENT_LIST_DIR}/@project_name@-${comp}-targets.cmake" OPTIONAL) + set(@project_name@_${comp}_FOUND TRUE) endif() endforeach() @@ -117,6 +115,6 @@ check_required_components(@project_name@) set(${CMAKE_FIND_PACKAGE_NAME}_CONFIG "${CMAKE_CURRENT_LIST_FILE}") include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(${CMAKE_FIND_PACKAGE_NAME} CONFIG_MODE) +find_package_handle_standard_args(${CMAKE_FIND_PACKAGE_NAME} CONFIG_MODE HANDLE_COMPONENTS) @RAPIDS_PROJECT_FINAL_CODE_BLOCK@ From 14f19377cc51eabe7674fe3279cdb5dae25365c3 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 20 Jan 2022 09:56:49 -0800 Subject: [PATCH 60/66] add unprefixed raft_distance_lib and raft_nn_lib to handle both BUILD export set and CPM downloaded modes --- cpp/CMakeLists.txt | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 7e01f19190..29cdccce1d 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -182,7 +182,9 @@ if(RAFT_COMPILE_LIBRARIES) EXPORT raft-distance-exports) endif() -target_link_libraries(raft_distance INTERFACE raft::raft $) +target_link_libraries(raft_distance INTERFACE raft::raft + $ + $) ############################################################################## # - raft_nn ------------------------------------------------------------------ @@ -216,7 +218,9 @@ if(RAFT_COMPILE_LIBRARIES) EXPORT raft-nn-exports) endif() -target_link_libraries(raft_nn INTERFACE raft::raft faiss::faiss $) +target_link_libraries(raft_nn INTERFACE raft::raft faiss::faiss + $ + $) ############################################################################## # - install targets----------------------------------------------------------- From cb69067bbc3b71e4623d8851d17d77b5260f0a30 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 20 Jan 2022 17:30:52 -0800 Subject: [PATCH 61/66] add versions.json to force Thrust v1.15.0 --- cpp/cmake/versions.json | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 cpp/cmake/versions.json diff --git a/cpp/cmake/versions.json b/cpp/cmake/versions.json new file mode 100644 index 0000000000..cca2dd8859 --- /dev/null +++ b/cpp/cmake/versions.json @@ -0,0 +1,9 @@ +{ + "packages" : { + "Thrust" : { + "version" : "1.15.0", + "git_url" : "https://github.com/NVIDIA/thrust.git", + "git_tag" : "${version}" + } + } +} From 47bbfd03845a77612fc8f760ea8be38075bd0f2f Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Thu, 20 Jan 2022 22:34:32 -0500 Subject: [PATCH 62/66] Using cuco branch w/ thrust 1.15 --- cpp/cmake/thirdparty/get_cuco.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/cmake/thirdparty/get_cuco.cmake b/cpp/cmake/thirdparty/get_cuco.cmake index 381addb03c..73997c4477 100644 --- a/cpp/cmake/thirdparty/get_cuco.cmake +++ b/cpp/cmake/thirdparty/get_cuco.cmake @@ -21,8 +21,8 @@ function(find_and_configure_cuco VERSION) BUILD_EXPORT_SET raft-exports INSTALL_EXPORT_SET raft-exports CPM_ARGS - GIT_REPOSITORY https://github.com/NVIDIA/cuCollections.git - GIT_TAG 0ca860b824f5dc22cf8a41f09912e62e11f07d82 + GIT_REPOSITORY https://github.com/trxcllnt/cuCollections.git + GIT_TAG patch-2 OPTIONS "BUILD_TESTS OFF" "BUILD_BENCHMARKS OFF" "BUILD_EXAMPLES OFF" From 2685783a82c23fb0c6188ae76489285ff9b4a394 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Fri, 21 Jan 2022 10:18:57 -0500 Subject: [PATCH 63/66] Reordering cuco and rmm cmake includes --- cpp/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 29cdccce1d..d59fbb22d4 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -105,9 +105,10 @@ endif() # add third party dependencies using CPM rapids_cpm_init() +# cuco before rmm to properly configure thrust/cub +include(cmake/thirdparty/get_cuco.cmake) include(cmake/thirdparty/get_rmm.cmake) include(cmake/thirdparty/get_libcudacxx.cmake) -include(cmake/thirdparty/get_cuco.cmake) include(cmake/thirdparty/get_faiss.cmake) if(BUILD_TESTS) From 2aa4ea9c32d85f5e080de97875c7cb4dd3024381 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Fri, 21 Jan 2022 15:02:37 -0500 Subject: [PATCH 64/66] Reverting previous changes --- cpp/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index d59fbb22d4..862541d054 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -106,8 +106,8 @@ endif() rapids_cpm_init() # cuco before rmm to properly configure thrust/cub -include(cmake/thirdparty/get_cuco.cmake) include(cmake/thirdparty/get_rmm.cmake) +include(cmake/thirdparty/get_cuco.cmake) include(cmake/thirdparty/get_libcudacxx.cmake) include(cmake/thirdparty/get_faiss.cmake) From c51fc3e4c347710a6154c2eaccd2adb69c0835f9 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 21 Jan 2022 13:28:38 -0800 Subject: [PATCH 65/66] add get_thrust.cmake back in to ensure we get Thrust v1.15.0 --- cpp/CMakeLists.txt | 8 ++++++-- cpp/cmake/thirdparty/get_thrust.cmake | 27 +++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 cpp/cmake/thirdparty/get_thrust.cmake diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index d59fbb22d4..529a3a3c00 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -105,9 +105,10 @@ endif() # add third party dependencies using CPM rapids_cpm_init() -# cuco before rmm to properly configure thrust/cub -include(cmake/thirdparty/get_cuco.cmake) +# thrust before rmm/cuco so we get the right version of thrust/cub +include(cmake/thirdparty/get_thrust.cmake) include(cmake/thirdparty/get_rmm.cmake) +include(cmake/thirdparty/get_cuco.cmake) include(cmake/thirdparty/get_libcudacxx.cmake) include(cmake/thirdparty/get_faiss.cmake) @@ -128,6 +129,7 @@ target_include_directories(raft INTERFACE "$") target_link_libraries(raft INTERFACE + raft::Thrust CUDA::cublas CUDA::curand CUDA::cusolver @@ -271,6 +273,8 @@ Imported Targets: set(code_string [=[ +thrust_create_target(raft::Thrust FROM_OPTIONS) + if(distance IN_LIST raft_FIND_COMPONENTS) enable_language(CUDA) endif() diff --git a/cpp/cmake/thirdparty/get_thrust.cmake b/cpp/cmake/thirdparty/get_thrust.cmake new file mode 100644 index 0000000000..c28ff6e66d --- /dev/null +++ b/cpp/cmake/thirdparty/get_thrust.cmake @@ -0,0 +1,27 @@ +# ============================================================================= +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + +# Use CPM to find or clone thrust +function(find_and_configure_thrust) + include(${rapids-cmake-dir}/cpm/thrust.cmake) + + rapids_cpm_thrust( + NAMESPACE raft + BUILD_EXPORT_SET raft-exports + INSTALL_EXPORT_SET raft-exports + ) + +endfunction() + +find_and_configure_thrust() From 56703082b149e628346df6b60b9e5edc430185fa Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 21 Jan 2022 13:34:04 -0800 Subject: [PATCH 66/66] revert cuco dependency hash --- cpp/cmake/thirdparty/get_cuco.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/cmake/thirdparty/get_cuco.cmake b/cpp/cmake/thirdparty/get_cuco.cmake index 73997c4477..381addb03c 100644 --- a/cpp/cmake/thirdparty/get_cuco.cmake +++ b/cpp/cmake/thirdparty/get_cuco.cmake @@ -21,8 +21,8 @@ function(find_and_configure_cuco VERSION) BUILD_EXPORT_SET raft-exports INSTALL_EXPORT_SET raft-exports CPM_ARGS - GIT_REPOSITORY https://github.com/trxcllnt/cuCollections.git - GIT_TAG patch-2 + GIT_REPOSITORY https://github.com/NVIDIA/cuCollections.git + GIT_TAG 0ca860b824f5dc22cf8a41f09912e62e11f07d82 OPTIONS "BUILD_TESTS OFF" "BUILD_BENCHMARKS OFF" "BUILD_EXAMPLES OFF"