From ddaa0251fbfd3c45a57c6fc2d10c1b6a89a312e6 Mon Sep 17 00:00:00 2001 From: Jeremy Dyer Date: Tue, 23 Mar 2021 15:16:39 -0400 Subject: [PATCH 1/8] libcudf_kafka now uses CPM to build cudf --- build.sh | 10 +- ci/release/update-version.sh | 3 + cpp/cmake/cudf-config.cmake.in | 4 +- cpp/libcudf_kafka/CMakeLists.txt | 142 +++++------------- .../cmake/Modules/ConfigureGoogleTest.cmake | 46 ------ .../Templates/GoogleTest.CMakeLists.txt.cmake | 12 -- .../cmake/thirdparty/CUDF_KAFKA_GetCUDF.cmake | 53 +++++++ .../thirdparty/CUDF_KAFKA_GetRDKafka.cmake | 25 +++ cpp/libcudf_kafka/tests/CMakeLists.txt | 99 ++---------- 9 files changed, 138 insertions(+), 256 deletions(-) delete mode 100644 cpp/libcudf_kafka/cmake/Modules/ConfigureGoogleTest.cmake delete mode 100644 cpp/libcudf_kafka/cmake/Templates/GoogleTest.CMakeLists.txt.cmake create mode 100644 cpp/libcudf_kafka/cmake/thirdparty/CUDF_KAFKA_GetCUDF.cmake create mode 100644 cpp/libcudf_kafka/cmake/thirdparty/CUDF_KAFKA_GetRDKafka.cmake diff --git a/build.sh b/build.sh index bc49b76d44e..fbaee68c3a9 100755 --- a/build.sh +++ b/build.sh @@ -194,17 +194,15 @@ if hasArg libcudf_kafka; then cmake -S $REPODIR/cpp/libcudf_kafka -B ${KAFKA_LIB_BUILD_DIR} \ ${CUDF_CMAKE_CUDA_ARCHITECTURES} \ -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ + -DBUILD_TESTS=${BUILD_TESTS} \ -DCMAKE_BUILD_TYPE=${BUILD_TYPE} + cd ${KAFKA_LIB_BUILD_DIR} + cmake --build . -j${PARALLEL_LEVEL} ${VERBOSE_FLAG} + if [[ ${INSTALL_TARGET} != "" ]]; then cmake --build . -j${PARALLEL_LEVEL} --target install ${VERBOSE_FLAG} - else - cmake --build . -j${PARALLEL_LEVEL} --target libcudf_kafka ${VERBOSE_FLAG} - fi - - if [[ ${BUILD_TESTS} == "ON" ]]; then - cmake --build . -j${PARALLEL_LEVEL} --target build_tests_libcudf_kafka ${VERBOSE_FLAG} fi fi diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 61f551f4b6d..819a0dcf6bf 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -49,6 +49,9 @@ function sed_runner() { # cpp update sed_runner 's/'"CUDA_DATAFRAME VERSION .* LANGUAGES"'/'"CUDA_DATAFRAME VERSION ${NEXT_FULL_TAG} LANGUAGES"'/g' cpp/CMakeLists.txt +# cpp libcudf_kafka update +sed_runner 's/'"CUDA_KAFKA VERSION .* LANGUAGES"'/'"CUDA_KAFKA VERSION ${NEXT_FULL_TAG} LANGUAGES"'/g' cpp/libcudf_kafka/CMakeLists.txt + # doxyfile update sed_runner 's/PROJECT_NUMBER = .*/PROJECT_NUMBER = '${NEXT_FULL_TAG}'/g' cpp/doxygen/Doxyfile diff --git a/cpp/cmake/cudf-config.cmake.in b/cpp/cmake/cudf-config.cmake.in index 14f8a661c2f..6a280264d3c 100644 --- a/cpp/cmake/cudf-config.cmake.in +++ b/cpp/cmake/cudf-config.cmake.in @@ -23,7 +23,7 @@ targets: cudf::cudf - The main cudf library. This module offers an optional testing component which defines the -following IMPORTED GLOBAL targets: +following IMPORTED GLOBAL targets: cudf::cudftestutil - The main cudf testing library cudf::gmock @@ -37,7 +37,7 @@ Result Variables This module will set the following variables in your project:: - CUDF_FOUND + cudf_FOUND CUDF_VERSION CUDF_VERSION_MAJOR CUDF_VERSION_MINOR diff --git a/cpp/libcudf_kafka/CMakeLists.txt b/cpp/libcudf_kafka/CMakeLists.txt index a307bf9d3f0..afd64203a3d 100644 --- a/cpp/libcudf_kafka/CMakeLists.txt +++ b/cpp/libcudf_kafka/CMakeLists.txt @@ -13,112 +13,73 @@ # See the License for the specific language governing permissions and # limitations under the License. #============================================================================= -cmake_minimum_required(VERSION 3.14 FATAL_ERROR) - -project(CUDA_KAFKA VERSION 0.15.0 LANGUAGES C CXX CUDA) +cmake_minimum_required(VERSION 3.18 FATAL_ERROR) + +# If `CMAKE_CUDA_ARCHITECTURES` is not defined, build for all supported architectures. If +# `CMAKE_CUDA_ARCHITECTURES` is set to an empty string (""), build for only the current +# architecture. If `CMAKE_CUDA_ARCHITECTURES` is specified by the user, use user setting. + +# This needs to be run before enabling the CUDA language due to the default initialization behavior +# of `CMAKE_CUDA_ARCHITECTURES`, https://gitlab.kitware.com/cmake/cmake/-/issues/21302 +if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES OR CMAKE_CUDA_ARCHITECTURES STREQUAL "ALL") + set(CUDF_BUILD_FOR_ALL_ARCHS TRUE) +elseif(CMAKE_CUDA_ARCHITECTURES STREQUAL "") + set(CUDF_BUILD_FOR_DETECTED_ARCHS TRUE) +endif() -# TODO: Since we have no actual CUDA code in cudf_kafka this should be removed in the future -# in favor of using FindCUDAToolkit to get the needed CUDA include headers -if(NOT CMAKE_CUDA_COMPILER) - message(SEND_ERROR "CMake cannot locate a CUDA compiler") -endif(NOT CMAKE_CUDA_COMPILER) +project(CUDA_KAFKA VERSION 0.19.0 LANGUAGES C CXX) ################################################################################################### -# - build type ------------------------------------------------------------------------------------ +# - Build options + +option(BUILD_TESTS "Build tests for libcudf_kafka" ON) -# Set a default build type if none was specified -set(DEFAULT_BUILD_TYPE "Release") +message(VERBOSE "CUDF_KAFKA: Build gtests: ${BUILD_TESTS}") -if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) - message(STATUS "Setting build type to '${DEFAULT_BUILD_TYPE}' since none specified.") - set(CMAKE_BUILD_TYPE "${DEFAULT_BUILD_TYPE}" CACHE - STRING "Choose the type of build." FORCE) - # Set the possible values of build type for cmake-gui - set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS - "Debug" "Release" "MinSizeRel" "RelWithDebInfo") -endif(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) +# Set CUDF_SOURCE_DIR to ensure that cmake Modules from parent cudf have the correct context +set(CUDF_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../") ################################################################################################### # - compiler options ------------------------------------------------------------------------------ -set(CMAKE_CXX_STANDARD 14) -set(CMAKE_CXX_STANDARD_REQUIRED ON) - -# To apply RUNPATH to transitive dependencies (this is a temporary solution) -set(CMAKE_SHARED_LINKER_FLAGS "-Wl,--disable-new-dtags") -set(CMAKE_EXE_LINKER_FLAGS "-Wl,--disable-new-dtags") - -# Build options -option(BUILD_TESTS "Configure CMake to build tests" ON) +# * find CUDAToolkit package +# * determine GPU architectures +# * enable the CMake CUDA language +# * set other CUDA compilation flags +include(../cmake/Modules/ConfigureCUDA.cmake) ################################################################################################### -# - cmake modules --------------------------------------------------------------------------------- - -message(VERBOSE "CMAKE_CURRENT_SOURCE_DIR: ${CMAKE_CURRENT_SOURCE_DIR}") -set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/" ${CMAKE_MODULE_PATH}) +# - Dependencies -include(FeatureSummary) -include(CheckIncludeFiles) -include(CheckLibraryExists) +# CPM +include(../cmake/thirdparty/CUDF_GetCPM.cmake) -################################################################################################### -# - conda environment ----------------------------------------------------------------------------- - -if("$ENV{CONDA_BUILD}" STREQUAL "1") - set(CMAKE_SYSTEM_PREFIX_PATH "$ENV{BUILD_PREFIX};$ENV{PREFIX};${CMAKE_SYSTEM_PREFIX_PATH}") - set(CONDA_INCLUDE_DIRS "$ENV{BUILD_PREFIX}/include" "$ENV{PREFIX}/include") - set(CONDA_LINK_DIRS "$ENV{BUILD_PREFIX}/lib" "$ENV{PREFIX}/lib") - message(VERBOSE "Conda build detected, CMAKE_SYSTEM_PREFIX_PATH set to: ${CMAKE_SYSTEM_PREFIX_PATH}") -endif() +# libcudf +include(cmake/thirdparty/CUDF_KAFKA_GetCUDF.cmake) -################################################################################################### -# - add gtest ------------------------------------------------------------------------------------- +# librdkafka +include(cmake/thirdparty/CUDF_KAFKA_GetRDKafka.cmake) -# TODO: This is currently using a nearly duplicate Google Test Module due to CMake source limitations. -# this should be standardized in the future to use the same Google Test Module as cudf -if(BUILD_TESTS) +# # GTests if enabled +if (BUILD_TESTS) + # include CTest module -- automatically calls enable_testing() include(CTest) - include(ConfigureGoogleTest) - - if(GTEST_FOUND) - message(VERBOSE "Google C++ Testing Framework (Google Test) found in ${GTEST_ROOT}") - include_directories(${GTEST_INCLUDE_DIR}) - add_subdirectory(${CMAKE_SOURCE_DIR}/tests) - else() - message(AUTHOR_WARNING "Google C++ Testing Framework (Google Test) not found: automated tests are disabled.") - endif(GTEST_FOUND) -endif(BUILD_TESTS) - -message(VERBOSE "CUDF_KAFKA_TEST_LIST set to: ${CUDF_KAFKA_TEST_LIST}") + add_subdirectory(tests) +endif() ################################################################################################### # - include paths --------------------------------------------------------------------------------- -if(CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES) - include_directories("${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}") -endif(CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES) - include_directories("${CMAKE_BINARY_DIR}/include" - "${CMAKE_BINARY_DIR}/include/jit" "${CMAKE_SOURCE_DIR}/include" "${CMAKE_SOURCE_DIR}/src") -if(CONDA_INCLUDE_DIRS) - include_directories("${CONDA_INCLUDE_DIRS}") -endif(CONDA_INCLUDE_DIRS) - ################################################################################################### # - library paths --------------------------------------------------------------------------------- link_directories("${CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES}" # CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES is an undocumented/unsupported variable containing the link directories for nvcc "${CMAKE_BINARY_DIR}/lib" - "${CMAKE_BINARY_DIR}" - "${GTEST_LIBRARY_DIR}" - "${RMM_LIBRARY}") - -if(CONDA_LINK_DIRS) - link_directories("${CONDA_LINK_DIRS}") -endif(CONDA_LINK_DIRS) + "${CMAKE_BINARY_DIR}") ################################################################################################### # - library target -------------------------------------------------------------------------------- @@ -127,37 +88,12 @@ add_library(cudf_kafka SHARED src/kafka_consumer.cpp ) -set_target_properties(cudf_kafka PROPERTIES BUILD_RPATH "\$ORIGIN") - -# Include paths -include_directories("${CMAKE_SOURCE_DIR}/include" - "${CMAKE_CURRENT_SOURCE_DIR}/include/cudf") - -################################################################################################### -# cudf_kafka - librdkafka ------------------------------------------------------------------------- - -find_path(RDKAFKA_INCLUDE "librdkafka" HINTS "$ENV{RDKAFKA_ROOT}/include") -find_library(RDKAFKA++_LIBRARY "rdkafka++" HINTS "$ENV{RDKAFKA_ROOT}/lib" "$ENV{RDKAFKA_ROOT}/build") - -message(VERBOSE "RDKAFKA: RDKAFKA++_LIBRARY set to ${RDKAFKA++_LIBRARY}") -message(VERBOSE "RDKAFKA: RDKAFKA_INCLUDE set to ${RDKAFKA_INCLUDE}") - -target_link_libraries(cudf_kafka ${RDKAFKA++_LIBRARY}) -include_directories("${RDKAFKA_INCLUDE}") - ################################################################################################### # - cudf_kafka Install ---------------------------------------------------------------------------- -target_link_libraries(cudf_kafka cudf) +target_link_libraries(cudf_kafka cudf::cudf RDKAFKA::RDKAFKA) install(TARGETS cudf_kafka DESTINATION lib) install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include DESTINATION include) - -add_custom_target(build_tests_libcudf_kafka - DEPENDS ${CUDF_KAFKA_TEST_LIST}) - -add_custom_target(test_libcudf_kafka - COMMAND ctest - DEPENDS build_tests_libcudf_kafka) diff --git a/cpp/libcudf_kafka/cmake/Modules/ConfigureGoogleTest.cmake b/cpp/libcudf_kafka/cmake/Modules/ConfigureGoogleTest.cmake deleted file mode 100644 index e2b3aff8546..00000000000 --- a/cpp/libcudf_kafka/cmake/Modules/ConfigureGoogleTest.cmake +++ /dev/null @@ -1,46 +0,0 @@ -set(GTEST_ROOT "${CMAKE_BINARY_DIR}/googletest") - -set(GTEST_CMAKE_ARGS "") - -configure_file("${CMAKE_SOURCE_DIR}/cmake/Templates/GoogleTest.CMakeLists.txt.cmake" - "${GTEST_ROOT}/CMakeLists.txt") - -file(MAKE_DIRECTORY "${GTEST_ROOT}/build") -file(MAKE_DIRECTORY "${GTEST_ROOT}/install") - -execute_process(COMMAND ${CMAKE_COMMAND} -G ${CMAKE_GENERATOR} . - RESULT_VARIABLE GTEST_CONFIG - WORKING_DIRECTORY ${GTEST_ROOT}) - -if(GTEST_CONFIG) - message(FATAL_ERROR "Configuring GoogleTest failed: " ${GTEST_CONFIG}) -endif(GTEST_CONFIG) - -set(PARALLEL_BUILD -j) -if($ENV{PARALLEL_LEVEL}) - set(NUM_JOBS $ENV{PARALLEL_LEVEL}) - set(PARALLEL_BUILD "${PARALLEL_BUILD}${NUM_JOBS}") -endif($ENV{PARALLEL_LEVEL}) - -if(${NUM_JOBS}) - if(${NUM_JOBS} EQUAL 1) - message(VERBOSE "GTEST BUILD: Enabling Sequential CMake build") - elseif(${NUM_JOBS} GREATER 1) - message(VERBOSE "GTEST BUILD: Enabling Parallel CMake build with ${NUM_JOBS} jobs") - endif(${NUM_JOBS} EQUAL 1) -else() - message(VERBOSE "GTEST BUILD: Enabling Parallel CMake build with all threads") -endif(${NUM_JOBS}) - -execute_process(COMMAND ${CMAKE_COMMAND} --build .. -- ${PARALLEL_BUILD} - RESULT_VARIABLE GTEST_BUILD - WORKING_DIRECTORY ${GTEST_ROOT}/build) - -if(GTEST_BUILD) - message(FATAL_ERROR "Building GoogleTest failed: " ${GTEST_BUILD}) -endif(GTEST_BUILD) - -message(VERBOSE "GoogleTest installed here: " ${GTEST_ROOT}/install) -set(GTEST_INCLUDE_DIR "${GTEST_ROOT}/install/include") -set(GTEST_LIBRARY_DIR "${GTEST_ROOT}/install/lib") -set(GTEST_FOUND TRUE) diff --git a/cpp/libcudf_kafka/cmake/Templates/GoogleTest.CMakeLists.txt.cmake b/cpp/libcudf_kafka/cmake/Templates/GoogleTest.CMakeLists.txt.cmake deleted file mode 100644 index 07692cd3d32..00000000000 --- a/cpp/libcudf_kafka/cmake/Templates/GoogleTest.CMakeLists.txt.cmake +++ /dev/null @@ -1,12 +0,0 @@ -cmake_minimum_required(VERSION 3.12) - -include(ExternalProject) - -ExternalProject_Add(GoogleTest - GIT_REPOSITORY https://github.com/google/googletest.git - GIT_TAG release-1.8.0 - GIT_SHALLOW true - SOURCE_DIR "${GTEST_ROOT}/googletest" - BINARY_DIR "${GTEST_ROOT}/build" - INSTALL_DIR "${GTEST_ROOT}/install" - CMAKE_ARGS ${GTEST_CMAKE_ARGS} -DCMAKE_INSTALL_PREFIX=${GTEST_ROOT}/install) diff --git a/cpp/libcudf_kafka/cmake/thirdparty/CUDF_KAFKA_GetCUDF.cmake b/cpp/libcudf_kafka/cmake/thirdparty/CUDF_KAFKA_GetCUDF.cmake new file mode 100644 index 00000000000..b99ed2bd27b --- /dev/null +++ b/cpp/libcudf_kafka/cmake/thirdparty/CUDF_KAFKA_GetCUDF.cmake @@ -0,0 +1,53 @@ +#============================================================================= +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +function(cudfkafka_save_if_enabled var) + if(CUDF_KAFKA_${var}) + unset(${var} PARENT_SCOPE) + unset(${var} CACHE) + endif() +endfunction() + +function(cudfkafka_restore_if_enabled var) + if(CUDF_KAFKA_${var}) + set(${var} ON CACHE INTERNAL "" FORCE) + endif() +endfunction() + +function(find_and_configure_cudf VERSION) + cudfkafka_save_if_enabled(BUILD_TESTS) + cudfkafka_save_if_enabled(BUILD_BENCHMARKS) + CPMFindPackage(NAME cudf + VERSION ${VERSION} + # GIT_REPOSITORY https://github.com/rapidsai/cudf.git + GIT_REPOSITORY https://github.com/robertmaynard/cudf.git + GIT_TAG fix/link_against_cpm_exports + GIT_SHALLOW TRUE + SOURCE_SUBDIR cpp + OPTIONS "BUILD_TESTS OFF" + "BUILD_BENCHMARKS OFF") + cudfkafka_restore_if_enabled(BUILD_TESTS) + cudfkafka_restore_if_enabled(BUILD_BENCHMARKS) + + if(NOT cudf_BINARY_DIR IN_LIST CMAKE_PREFIX_PATH) + list(APPEND CMAKE_PREFIX_PATH "${cudf_BINARY_DIR}") + set(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} PARENT_SCOPE) + endif() + +endfunction() + +set(CUDF_KAFKA_MIN_VERSION_cudf 0.19) +find_and_configure_cudf(${CUDF_KAFKA_MIN_VERSION_cudf}) diff --git a/cpp/libcudf_kafka/cmake/thirdparty/CUDF_KAFKA_GetRDKafka.cmake b/cpp/libcudf_kafka/cmake/thirdparty/CUDF_KAFKA_GetRDKafka.cmake new file mode 100644 index 00000000000..5c07db66668 --- /dev/null +++ b/cpp/libcudf_kafka/cmake/thirdparty/CUDF_KAFKA_GetRDKafka.cmake @@ -0,0 +1,25 @@ +#============================================================================= +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +find_path(RDKAFKA_INCLUDE "librdkafka" HINTS "$ENV{RDKAFKA_ROOT}/include") +find_library(RDKAFKA++_LIBRARY "rdkafka++" HINTS "$ENV{RDKAFKA_ROOT}/lib" "$ENV{RDKAFKA_ROOT}/build") + +if(RDKAFKA_INCLUDE AND RDKAFKA++_LIBRARY) + add_library(rdkafka INTERFACE) + target_link_libraries(rdkafka INTERFACE "${RDKAFKA++_LIBRARY}") + target_include_directories(rdkafka INTERFACE "${RDKAFKA_INCLUDE}") + add_library(RDKAFKA::RDKAFKA ALIAS rdkafka) +endif() \ No newline at end of file diff --git a/cpp/libcudf_kafka/tests/CMakeLists.txt b/cpp/libcudf_kafka/tests/CMakeLists.txt index af0ea1c8239..52653ed7908 100644 --- a/cpp/libcudf_kafka/tests/CMakeLists.txt +++ b/cpp/libcudf_kafka/tests/CMakeLists.txt @@ -1,5 +1,5 @@ #============================================================================= -# Copyright (c) 2018-2020, NVIDIA CORPORATION. +# Copyright (c) 2018-2021, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,100 +14,25 @@ # limitations under the License. #============================================================================= -cmake_minimum_required(VERSION 3.14 FATAL_ERROR) - -project(KAFKA_TESTS VERSION 0.15.0 LANGUAGES C CXX CUDA) - -# TODO: Since we have no actual CUDA code in cudf_kafka this should be removed in the future -# in favor of using FindCUDAToolkit to get the needed CUDA include headers -if(NOT CMAKE_CUDA_COMPILER) - message(SEND_ERROR "CMake cannot locate a CUDA compiler") -endif(NOT CMAKE_CUDA_COMPILER) - -################################################################################################### -# - build type ------------------------------------------------------------------------------------ - -# Set a default build type if none was specified -set(DEFAULT_BUILD_TYPE "Release") - -if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) - message(STATUS "Setting build type to '${DEFAULT_BUILD_TYPE}' since none specified.") - set(CMAKE_BUILD_TYPE "${DEFAULT_BUILD_TYPE}" CACHE - STRING "Choose the type of build." FORCE) - # Set the possible values of build type for cmake-gui - set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS - "Debug" "Release" "MinSizeRel" "RelWithDebInfo") -endif(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) - -################################################################################################### -# - compiler options ------------------------------------------------------------------------------ - -set(CMAKE_CXX_STANDARD 14) -set(CMAKE_CXX_STANDARD_REQUIRED ON) - -# To apply RUNPATH to transitive dependencies (this is a temporary solution) -set(CMAKE_SHARED_LINKER_FLAGS "-Wl,--disable-new-dtags") -set(CMAKE_EXE_LINKER_FLAGS "-Wl,--disable-new-dtags") - -################################################################################################### -# - conda environment ----------------------------------------------------------------------------- - -if("$ENV{CONDA_BUILD}" STREQUAL "1") - set(CMAKE_SYSTEM_PREFIX_PATH "$ENV{BUILD_PREFIX};$ENV{PREFIX};${CMAKE_SYSTEM_PREFIX_PATH}") - set(CONDA_INCLUDE_DIRS "$ENV{BUILD_PREFIX}/include" "$ENV{PREFIX}/include") - set(CONDA_LINK_DIRS "$ENV{BUILD_PREFIX}/lib" "$ENV{PREFIX}/lib") - message(STATUS "Conda build detected, CMAKE_SYSTEM_PREFIX_PATH set to: ${CMAKE_SYSTEM_PREFIX_PATH}") -endif() - ################################################################################################### # - compiler function ----------------------------------------------------------------------------- -set(CUDF_KAFKA_TEST_LIST CACHE INTERNAL "CUDF_KAFKA_TEST_LIST") - -function(ConfigureTest CMAKE_TEST_NAME CMAKE_TEST_SRC) - add_executable(${CMAKE_TEST_NAME} - ${CMAKE_TEST_SRC}) - set_target_properties(${CMAKE_TEST_NAME} PROPERTIES POSITION_INDEPENDENT_CODE ON) - target_link_libraries(${CMAKE_TEST_NAME} gmock gtest gtest_main pthread cuda cudf_kafka) - set_target_properties(${CMAKE_TEST_NAME} PROPERTIES - RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/gtests") +function(ConfigureTest CMAKE_TEST_NAME ) + add_executable(${CMAKE_TEST_NAME} ${ARGN}) + set_target_properties(${CMAKE_TEST_NAME} + PROPERTIES RUNTIME_OUTPUT_DIRECTORY "$") + target_link_libraries(${CMAKE_TEST_NAME} PRIVATE GTest::gmock_main GTest::gtest_main cudf_kafka) + target_include_directories(${CMAKE_TEST_NAME} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../include) add_test(NAME ${CMAKE_TEST_NAME} COMMAND ${CMAKE_TEST_NAME}) - set(CUDF_KAFKA_TEST_LIST ${CUDF_KAFKA_TEST_LIST} ${CMAKE_TEST_NAME} CACHE INTERNAL "CUDF_KAFKA_TEST_LIST") -endfunction(ConfigureTest) +endfunction() ################################################################################################### -# - include paths --------------------------------------------------------------------------------- - -if(CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES) - include_directories("${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}") -endif(CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES) - -include_directories("${CMAKE_BINARY_DIR}/include" - "${CMAKE_SOURCE_DIR}/include" - "${CMAKE_SOURCE_DIR}../../../tests" - "${CMAKE_SOURCE_DIR}" - "${CMAKE_SOURCE_DIR}/src" - "${GTEST_INCLUDE_DIR}") - -if(CONDA_INCLUDE_DIRS) - include_directories("${CONDA_INCLUDE_DIRS}") -endif(CONDA_INCLUDE_DIRS) +# - Kafka host tests ---------------------------------------------------------------------------------- +ConfigureTest(KAFKA_HOST_TEST + kafka_consumer_tests.cpp) ################################################################################################### -# - library paths --------------------------------------------------------------------------------- - -link_directories("${CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES}" # CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES is an undocumented/unsupported variable containing the link directories for nvcc - "${CMAKE_BINARY_DIR}/lib" - "${CMAKE_BINARY_DIR}" - "${GTEST_LIBRARY_DIR}" - "${RMM_LIBRARY}") - -if(CONDA_LINK_DIRS) - link_directories("${CONDA_LINK_DIRS}") -endif(CONDA_LINK_DIRS) - +### enable testing ################################################################################ ################################################################################################### -# - create tests ---------------------------------------------------------------------------------- -ConfigureTest(CUDF_KAFKA_HOST_READ kafka_consumer_tests.cpp) enable_testing() From b9dc62084ae97526ccdd60f917018ede0e036cf4 Mon Sep 17 00:00:00 2001 From: Jeremy Dyer Date: Tue, 23 Mar 2021 15:16:39 -0400 Subject: [PATCH 2/8] libcudf_kafka now uses CPM to build cudf --- cpp/libcudf_kafka/CMakeLists.txt | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/cpp/libcudf_kafka/CMakeLists.txt b/cpp/libcudf_kafka/CMakeLists.txt index afd64203a3d..b9cf6b0ff32 100644 --- a/cpp/libcudf_kafka/CMakeLists.txt +++ b/cpp/libcudf_kafka/CMakeLists.txt @@ -22,9 +22,9 @@ cmake_minimum_required(VERSION 3.18 FATAL_ERROR) # This needs to be run before enabling the CUDA language due to the default initialization behavior # of `CMAKE_CUDA_ARCHITECTURES`, https://gitlab.kitware.com/cmake/cmake/-/issues/21302 if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES OR CMAKE_CUDA_ARCHITECTURES STREQUAL "ALL") - set(CUDF_BUILD_FOR_ALL_ARCHS TRUE) + set(CUDA_KAFKA_BUILD_FOR_ALL_ARCHS TRUE) elseif(CMAKE_CUDA_ARCHITECTURES STREQUAL "") - set(CUDF_BUILD_FOR_DETECTED_ARCHS TRUE) + set(CUDA_KAFKA_BUILD_FOR_DETECTED_ARCHS TRUE) endif() project(CUDA_KAFKA VERSION 0.19.0 LANGUAGES C CXX) @@ -36,9 +36,6 @@ option(BUILD_TESTS "Build tests for libcudf_kafka" ON) message(VERBOSE "CUDF_KAFKA: Build gtests: ${BUILD_TESTS}") -# Set CUDF_SOURCE_DIR to ensure that cmake Modules from parent cudf have the correct context -set(CUDF_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../") - ################################################################################################### # - compiler options ------------------------------------------------------------------------------ From 7a967def0a93062fb74c6d572462b2a29165073c Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Tue, 23 Mar 2021 18:49:53 -0400 Subject: [PATCH 3/8] Use cudf's gmock/gtest targets when they exist --- cpp/libcudf_kafka/tests/CMakeLists.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cpp/libcudf_kafka/tests/CMakeLists.txt b/cpp/libcudf_kafka/tests/CMakeLists.txt index 52653ed7908..845f245a764 100644 --- a/cpp/libcudf_kafka/tests/CMakeLists.txt +++ b/cpp/libcudf_kafka/tests/CMakeLists.txt @@ -21,7 +21,11 @@ function(ConfigureTest CMAKE_TEST_NAME ) add_executable(${CMAKE_TEST_NAME} ${ARGN}) set_target_properties(${CMAKE_TEST_NAME} PROPERTIES RUNTIME_OUTPUT_DIRECTORY "$") - target_link_libraries(${CMAKE_TEST_NAME} PRIVATE GTest::gmock_main GTest::gtest_main cudf_kafka) + if(TARGET cudf::gmock_main) + target_link_libraries(${CMAKE_TEST_NAME} PRIVATE cudf::gmock_main cudf::gtest_main cudf_kafka) + else() + target_link_libraries(${CMAKE_TEST_NAME} PRIVATE GTest::gmock_main GTest::gtest_main cudf_kafka) + endif() target_include_directories(${CMAKE_TEST_NAME} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../include) add_test(NAME ${CMAKE_TEST_NAME} COMMAND ${CMAKE_TEST_NAME}) endfunction() From 1a4c73ff682029b4bd6ce88bc1b2987bb2524518 Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Tue, 23 Mar 2021 20:16:38 -0400 Subject: [PATCH 4/8] cudf_kafka get GTest via cpm --- cpp/libcudf_kafka/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cpp/libcudf_kafka/CMakeLists.txt b/cpp/libcudf_kafka/CMakeLists.txt index b9cf6b0ff32..d5aad338adb 100644 --- a/cpp/libcudf_kafka/CMakeLists.txt +++ b/cpp/libcudf_kafka/CMakeLists.txt @@ -59,6 +59,9 @@ include(cmake/thirdparty/CUDF_KAFKA_GetRDKafka.cmake) # # GTests if enabled if (BUILD_TESTS) + # GoogleTest + include(cmake/thirdparty/CUDF_GetGTest.cmake) + # include CTest module -- automatically calls enable_testing() include(CTest) add_subdirectory(tests) From 47aff63e39c28f86d95f3e6b4b3cf31e5613b4f4 Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Tue, 23 Mar 2021 20:20:33 -0400 Subject: [PATCH 5/8] we can use cudf branch-0.19 again --- cpp/libcudf_kafka/cmake/thirdparty/CUDF_KAFKA_GetCUDF.cmake | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cpp/libcudf_kafka/cmake/thirdparty/CUDF_KAFKA_GetCUDF.cmake b/cpp/libcudf_kafka/cmake/thirdparty/CUDF_KAFKA_GetCUDF.cmake index b99ed2bd27b..4796495413e 100644 --- a/cpp/libcudf_kafka/cmake/thirdparty/CUDF_KAFKA_GetCUDF.cmake +++ b/cpp/libcudf_kafka/cmake/thirdparty/CUDF_KAFKA_GetCUDF.cmake @@ -32,9 +32,8 @@ function(find_and_configure_cudf VERSION) cudfkafka_save_if_enabled(BUILD_BENCHMARKS) CPMFindPackage(NAME cudf VERSION ${VERSION} - # GIT_REPOSITORY https://github.com/rapidsai/cudf.git - GIT_REPOSITORY https://github.com/robertmaynard/cudf.git - GIT_TAG fix/link_against_cpm_exports + GIT_REPOSITORY https://github.com/rapidsai/cudf.git + GIT_TAG branch-${VERSION} GIT_SHALLOW TRUE SOURCE_SUBDIR cpp OPTIONS "BUILD_TESTS OFF" From 48b7d15c0b562b2e4891869d213f79ab37c1cb1d Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Tue, 23 Mar 2021 21:41:17 -0400 Subject: [PATCH 6/8] cudf_kafka get GTest via cpm --- cpp/libcudf_kafka/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/libcudf_kafka/CMakeLists.txt b/cpp/libcudf_kafka/CMakeLists.txt index d5aad338adb..7a6750bf21f 100644 --- a/cpp/libcudf_kafka/CMakeLists.txt +++ b/cpp/libcudf_kafka/CMakeLists.txt @@ -60,7 +60,7 @@ include(cmake/thirdparty/CUDF_KAFKA_GetRDKafka.cmake) # # GTests if enabled if (BUILD_TESTS) # GoogleTest - include(cmake/thirdparty/CUDF_GetGTest.cmake) + include(../cmake/thirdparty/CUDF_GetGTest.cmake) # include CTest module -- automatically calls enable_testing() include(CTest) From d6c599e014c3622f2ff9eb29627b21b80bae2799 Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Wed, 24 Mar 2021 09:35:40 -0400 Subject: [PATCH 7/8] cudf_kafka correctly computes tests runtime output directory --- cpp/libcudf_kafka/tests/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/libcudf_kafka/tests/CMakeLists.txt b/cpp/libcudf_kafka/tests/CMakeLists.txt index 845f245a764..f556d36d9d2 100644 --- a/cpp/libcudf_kafka/tests/CMakeLists.txt +++ b/cpp/libcudf_kafka/tests/CMakeLists.txt @@ -20,7 +20,7 @@ function(ConfigureTest CMAKE_TEST_NAME ) add_executable(${CMAKE_TEST_NAME} ${ARGN}) set_target_properties(${CMAKE_TEST_NAME} - PROPERTIES RUNTIME_OUTPUT_DIRECTORY "$") + PROPERTIES RUNTIME_OUTPUT_DIRECTORY "$") if(TARGET cudf::gmock_main) target_link_libraries(${CMAKE_TEST_NAME} PRIVATE cudf::gmock_main cudf::gtest_main cudf_kafka) else() From 70dace2e9b6d816354a8879af1a0a6dead25f21b Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Wed, 24 Mar 2021 16:02:23 -0400 Subject: [PATCH 8/8] remove cudf_kafka need to enable to CUDA language --- build.sh | 17 +++++++++-------- cpp/libcudf_kafka/CMakeLists.txt | 24 +----------------------- 2 files changed, 10 insertions(+), 31 deletions(-) diff --git a/build.sh b/build.sh index fbaee68c3a9..70b93427d5c 100755 --- a/build.sh +++ b/build.sh @@ -134,18 +134,20 @@ if hasArg clean; then done fi -if (( ${BUILD_ALL_GPU_ARCH} == 0 )); then - CUDF_CMAKE_CUDA_ARCHITECTURES="-DCMAKE_CUDA_ARCHITECTURES=" - echo "Building for the architecture of the GPU in the system..." -else - CUDF_CMAKE_CUDA_ARCHITECTURES="" - echo "Building for *ALL* supported GPU architectures..." -fi ################################################################################ # Configure, build, and install libcudf if buildAll || hasArg libcudf; then + + if (( ${BUILD_ALL_GPU_ARCH} == 0 )); then + CUDF_CMAKE_CUDA_ARCHITECTURES="-DCMAKE_CUDA_ARCHITECTURES=" + echo "Building for the architecture of the GPU in the system..." + else + CUDF_CMAKE_CUDA_ARCHITECTURES="" + echo "Building for *ALL* supported GPU architectures..." + fi + cmake -S $REPODIR/cpp -B ${LIB_BUILD_DIR} \ -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ ${CUDF_CMAKE_CUDA_ARCHITECTURES} \ @@ -192,7 +194,6 @@ fi # Build libcudf_kafka library if hasArg libcudf_kafka; then cmake -S $REPODIR/cpp/libcudf_kafka -B ${KAFKA_LIB_BUILD_DIR} \ - ${CUDF_CMAKE_CUDA_ARCHITECTURES} \ -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ -DBUILD_TESTS=${BUILD_TESTS} \ -DCMAKE_BUILD_TYPE=${BUILD_TYPE} diff --git a/cpp/libcudf_kafka/CMakeLists.txt b/cpp/libcudf_kafka/CMakeLists.txt index 7a6750bf21f..e178f5a6280 100644 --- a/cpp/libcudf_kafka/CMakeLists.txt +++ b/cpp/libcudf_kafka/CMakeLists.txt @@ -15,36 +15,14 @@ #============================================================================= cmake_minimum_required(VERSION 3.18 FATAL_ERROR) -# If `CMAKE_CUDA_ARCHITECTURES` is not defined, build for all supported architectures. If -# `CMAKE_CUDA_ARCHITECTURES` is set to an empty string (""), build for only the current -# architecture. If `CMAKE_CUDA_ARCHITECTURES` is specified by the user, use user setting. - -# This needs to be run before enabling the CUDA language due to the default initialization behavior -# of `CMAKE_CUDA_ARCHITECTURES`, https://gitlab.kitware.com/cmake/cmake/-/issues/21302 -if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES OR CMAKE_CUDA_ARCHITECTURES STREQUAL "ALL") - set(CUDA_KAFKA_BUILD_FOR_ALL_ARCHS TRUE) -elseif(CMAKE_CUDA_ARCHITECTURES STREQUAL "") - set(CUDA_KAFKA_BUILD_FOR_DETECTED_ARCHS TRUE) -endif() - -project(CUDA_KAFKA VERSION 0.19.0 LANGUAGES C CXX) +project(CUDA_KAFKA VERSION 0.19.0 LANGUAGES CXX) ################################################################################################### # - Build options - option(BUILD_TESTS "Build tests for libcudf_kafka" ON) message(VERBOSE "CUDF_KAFKA: Build gtests: ${BUILD_TESTS}") -################################################################################################### -# - compiler options ------------------------------------------------------------------------------ - -# * find CUDAToolkit package -# * determine GPU architectures -# * enable the CMake CUDA language -# * set other CUDA compilation flags -include(../cmake/Modules/ConfigureCUDA.cmake) - ################################################################################################### # - Dependencies