Consolidate C++ conda recipes and add libcugraph-tests package

Signed-off-by: Jordan Jacobelli <[email protected]>

ci/cpu/build.sh

@@ -36,10 +36,6 @@ export CONDA_BLD_DIR="${WORKSPACE}/.conda-bld"
 # ucx-py version
 export UCX_PY_VERSION='0.25.*'
 
-export CMAKE_CUDA_COMPILER_LAUNCHER="sccache"
-export CMAKE_CXX_COMPILER_LAUNCHER="sccache"
-export CMAKE_C_COMPILER_LAUNCHER="sccache"
-
 ################################################################################
 # SETUP - Check environment
 ################################################################################
@@ -91,10 +87,8 @@ if [ "$BUILD_LIBCUGRAPH" == '1' ]; then
   gpuci_logger "Building conda package for libcugraph and libcugraph_etl"
   if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
     gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libcugraph
-    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libcugraph_etl
   else
     gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libcugraph
-    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libcugraph_etl
     mkdir -p ${CONDA_BLD_DIR}/libcugraph
     mv ${CONDA_BLD_DIR}/work ${CONDA_BLD_DIR}/libcugraph/work
   fi

ci/cpu/upload.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright (c) 2018-2021, NVIDIA CORPORATION.
+# Copyright (c) 2018-2022, NVIDIA CORPORATION.
 #
 # Adopted from https://github.com/tmcdonell/travis-scripts/blob/dfaac280ac2082cd6bcaba3217428347899f2975/update-accelerate-buildbot.sh
 
@@ -24,41 +24,28 @@ if [ -z "$MY_UPLOAD_KEY" ]; then
   return 0
 fi
 
-################################################################################
-# SETUP - Get conda file output locations
-################################################################################
-
-gpuci_logger "Get conda file output locations"
-
-export LIBCUGRAPH_FILE=`conda build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libcugraph --output`
-export LIBCUGRAPH_ETL_FILE=`conda build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libcugraph_etl --output`
-export PYLIBCUGRAPH_FILE=`conda build --croot ${CONDA_BLD_DIR} conda/recipes/pylibcugraph --python=$PYTHON --output`
-export CUGRAPH_FILE=`conda build --croot ${CONDA_BLD_DIR} conda/recipes/cugraph --python=$PYTHON --output`
-
 ################################################################################
 # UPLOAD - Conda packages
 ################################################################################
 
 gpuci_logger "Starting conda uploads"
 
 if [[ "$BUILD_LIBCUGRAPH" == "1" && "$UPLOAD_LIBCUGRAPH" == "1" ]]; then
-  test -e ${LIBCUGRAPH_FILE}
-  echo "Upload libcugraph"
-  echo ${LIBCUGRAPH_FILE}
-  gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBCUGRAPH_FILE} --no-progress
-  test -e ${LIBCUGRAPH_ETL_FILE}
-  echo "Upload libcugraph_etl"
-  echo ${LIBCUGRAPH_ETL_FILE}
-  gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBCUGRAPH_ETL_FILE} --no-progress
+  LIBCUGRAPH_FILES=$(conda build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libcugraph --output)
+  while read -r LIBCUGRAPH_FILE; do
+    echo "Upload libcugraph file: ${LIBCUGRAPH_FILE}"
+    test -e ${LIBCUGRAPH_FILE}
+    gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBCUGRAPH_FILE} --no-progress
+  done <<< "${LIBCUGRAPH_FILES}"
 fi
 
 if [[ "$BUILD_CUGRAPH" == "1" ]]; then
+  PYLIBCUGRAPH_FILE=$(conda build --croot ${CONDA_BLD_DIR} conda/recipes/pylibcugraph --python=$PYTHON --output)
   test -e ${PYLIBCUGRAPH_FILE}
-  echo "Upload pylibcugraph"
-  echo ${PYLIBCUGRAPH_FILE}
+  echo "Upload pylibcugraph file: ${PYLIBCUGRAPH_FILE}"
   gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${PYLIBCUGRAPH_FILE} --no-progress
+  CUGRAPH_FILE=$(conda build --croot ${CONDA_BLD_DIR} conda/recipes/cugraph --python=$PYTHON --output)
   test -e ${CUGRAPH_FILE}
-  echo "Upload cugraph"
-  echo ${CUGRAPH_FILE}
+  echo "Upload cugraph file: ${CUGRAPH_FILE}"
   gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${CUGRAPH_FILE} --no-progress
 fi

ci/gpu/build.sh

@@ -42,10 +42,6 @@ export MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'`
 # ucx-py version
 export UCX_PY_VERSION='0.25.*'
 
-export CMAKE_CUDA_COMPILER_LAUNCHER="sccache"
-export CMAKE_CXX_COMPILER_LAUNCHER="sccache"
-export CMAKE_C_COMPILER_LAUNCHER="sccache"
-
 ################################################################################
 # SETUP - Check environment
 ################################################################################
@@ -106,11 +102,8 @@ if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
     gpuci_logger "Build from source"
     $WORKSPACE/build.sh -v clean libcugraph pylibcugraph cugraph
 else
-    CONDA_FILE=`find ${CONDA_ARTIFACT_PATH} -name "libcugraph*.tar.bz2"`
-    CONDA_FILE=`basename "$CONDA_FILE" .tar.bz2` #get filename without extension
-    CONDA_FILE=${CONDA_FILE//-/=} #convert to conda install
-    echo "Installing $CONDA_FILE"
-    gpuci_mamba_retry install -c ${CONDA_ARTIFACT_PATH} "$CONDA_FILE"
+    echo "Installing libcugraph-tests"
+    gpuci_mamba_retry install -c ${CONDA_ARTIFACT_PATH} libcugraph-tests
 
     gpuci_logger "Install the master version of dask and distributed"
     pip install "git+https://github.com/dask/distributed.git" --upgrade --no-deps

ci/test.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright (c) 2019-2021, NVIDIA CORPORATION.
+# Copyright (c) 2019-2022, NVIDIA CORPORATION.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -61,7 +61,6 @@ else
     # present. For multiple libs (ie. libcugraph and libcugraph_etl), the
     # "_work" dir is prefixed with the lib name.
     export LIBCUGRAPH_BUILD_DIR="$WORKSPACE/ci/artifacts/cugraph/cpu/libcugraph_work/cpp/build"
-    export LD_LIBRARY_PATH="$LIBCUGRAPH_BUILD_DIR:$CONDA_PREFIX/lib:$LD_LIBRARY_PATH"
     cd $LIBCUGRAPH_BUILD_DIR
 fi
 
@@ -74,7 +73,7 @@ if (python ${CUGRAPH_ROOT}/ci/utils/is_pascal.py); then
     echo "WARNING: skipping C++ tests on Pascal GPU arch."
 else
     echo "C++ gtests for cuGraph (single-GPU only)..."
-    for gt in $(find ./tests -name "*_TEST" | grep -v "MG_\|CAPI_" || true); do
+    for gt in $(find "${CONDA_PREFIX}/bin/tests" -name "*_TEST" | grep -v "MG_\|CAPI_" || true); do
         test_name=$(basename $gt)
         echo "Running gtest $test_name"
         ${gt} ${GTEST_FILTER} ${GTEST_ARGS}
@@ -83,7 +82,7 @@ else
     # FIXME: the C API tests do not generate XML, so CI systems will not show
     # them in the GUI. Failing C API tests will still fail CI though, and the
     # output will appear in logs.
-    for ct in $(find ./tests -name "CAPI_*_TEST"); do
+    for ct in $(find "${CONDA_PREFIX}/bin/tests" -name "CAPI_*_TEST"); do
         test_name=$(basename $ct)
         echo "Running C API test $test_name"
         ${ct}

conda/recipes/libcugraph/build.sh

@@ -1,6 +1,5 @@
 #!/usr/bin/env bash
-
-# Copyright (c) 2019-2021, NVIDIA CORPORATION.
+# Copyright (c) 2019-2022, NVIDIA CORPORATION.
 
 # This assumes the script is executed from the root of the repo directory
 
@@ -9,4 +8,4 @@
 # openmpi build dependencies). The conda package does NOT include these test
 # binaries or extra dependencies, but these are built here for use in CI runs.
 
-./build.sh libcugraph cpp-mgtests -v --allgpuarch
+./build.sh libcugraph libcugraph_etl cpp-mgtests -n -v --allgpuarch

conda/recipes/libcugraph/conda_build_config.yaml

@@ -0,0 +1,17 @@
+cmake_version:
+  - ">=3.20.1"
+
+doxygen_version:
+  - ">=1.8.11"
+
+boost_cpp_version:
+  - ">=1.66"
+
+nccl_version:
+  - ">=2.9.9"
+
+gtest_version:
+  - "=1.10.0"
+
+libcusolver_version:
+  - ">=11.2.1"

conda/recipes/libcugraph/install_libcugraph.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+# Copyright (c) 2022, NVIDIA CORPORATION.
+
+cmake --install cpp/build

conda/recipes/libcugraph/install_libcugraph_etl.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+# Copyright (c) 2022, NVIDIA CORPORATION.
+
+cmake --install cpp/libcugraph_etl/build

conda/recipes/libcugraph/install_libcugraph_tests.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+# Copyright (c) 2022, NVIDIA CORPORATION.
+
+cmake --install cpp/build --component testing
+cmake --install cpp/libcugraph_etl/build --component testing

conda/recipes/libcugraph/meta.yaml

@@ -3,19 +3,18 @@
 # Usage:
 #   conda build -c nvidia -c rapidsai -c conda-forge .
 {% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %}
-{% set minor_version =  version.split('.')[0] + '.' + version.split('.')[1] %}
-{% set cuda_version='.'.join(environ.get('CUDA', '9.2').split('.')[:2]) %}
-{% set cuda_major=cuda_version.split('.')[0] %}
+{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %}
+{% set cuda_version = '.'.join(environ.get('CUDA', '11.5').split('.')[:2]) %}
+{% set cuda_major = cuda_version.split('.')[0] %}
+{% set cuda_spec = ">=" + cuda_major ~ ",<" + (cuda_major | int + 1) ~ ".0a0" %} # i.e. >=11,<12.0a0
+
 package:
-  name: libcugraph
-  version: {{ version }}
+  name: libcugraph-split
 
 source:
   git_url: ../../..
 
 build:
-  number: {{ GIT_DESCRIBE_NUMBER }}
-  string: cuda{{ cuda_major }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}
   script_env:
     - CC
     - CXX
@@ -34,29 +33,82 @@ build:
 
 requirements:
   build:
-    - cmake>=3.20.1
-    - doxygen>=1.8.11
+    - cmake {{ cmake_version }}
+  host:
+    - doxygen {{ doxygen_version }}
     - cudatoolkit {{ cuda_version }}.*
-    - libraft-headers {{ minor_version }}
+    - libraft-headers {{ minor_version }}.*
     - libcugraphops {{ minor_version }}.*
     - librmm {{ minor_version }}.*
     - cudf {{ minor_version }}.*
-    - boost-cpp>=1.66
-    - nccl>=2.9.9
-    - ucx-proc=*=gpu
-    - gtest=1.10.0  # FIXME: pinned to version in https://github.com/rapidsai/integration/blob/branch-22.04/conda/recipes/versions.yaml
-    - gmock=1.10.0  # FIXME: pinned to version in https://github.com/rapidsai/integration/blob/branch-22.04/conda/recipes/versions.yaml
-  run:
-    - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }}
-    - libraft-headers {{ minor_version }}
-    - librmm {{ minor_version }}
-    - nccl>=2.9.9
+    - boost-cpp {{ boost_cpp_version }}
+    - nccl {{ nccl_version }}
     - ucx-proc=*=gpu
-    - libcugraphops {{ minor_version }}.*
-    - libcusolver>=11.2.1
+    - gtest {{ gtest_version }}  # FIXME: pinned to version in https://github.com/rapidsai/integration/blob/branch-22.04/conda/recipes/versions.yaml
+    - gmock {{ gtest_version }}  # FIXME: pinned to version in https://github.com/rapidsai/integration/blob/branch-22.04/conda/recipes/versions.yaml
 
-about:
-  home: http://rapids.ai/
-  license: Apache-2.0
-  license_file: ../../../LICENSE
-  summary: libcugraph library
+outputs:
+  - name: libcugraph
+    version: {{ version }}
+    script: install_libcugraph.sh
+    build:
+      number: {{ GIT_DESCRIBE_NUMBER }}
+      string: cuda{{ cuda_major }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}
+    requirements:
+      build:
+        - cmake {{ cmake_version }}
+      run:
+        - cudatoolkit {{ cuda_spec }}
+        - libraft-headers {{ minor_version }}
+        - librmm {{ minor_version }}
+        - nccl {{ nccl_version }}
+        - ucx-proc=*=gpu
+        - libcugraphops {{ minor_version }}
+        - libcusolver {{ libcusolver_version }}
+    about:
+      home: http://rapids.ai/
+      license: Apache-2.0
+      license_file: ../../../LICENSE
+      summary: libcugraph library
+  - name: libcugraph_etl
+    version: {{ version }}
+    script: install_libcugraph_etl.sh
+    build:
+      number: {{ GIT_DESCRIBE_NUMBER }}
+      string: cuda{{ cuda_major }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}
+    requirements:
+      build:
+        - cmake {{ cmake_version }}
+      host:
+        - {{ pin_subpackage('libcugraph', exact=True) }}
+      run:
+        - {{ pin_subpackage('libcugraph', exact=True) }}
+        - cudatoolkit {{ cuda_spec }}
+        - libcudf {{ minor_version }}
+        - libraft-headers {{ minor_version }}
+        - librmm {{ minor_version }}
+    about:
+      home: http://rapids.ai/
+      license: Apache-2.0
+      license_file: ../../../LICENSE
+      summary: libcugraph_etl library
+  - name: libcugraph-tests
+    version: {{ version }}
+    script: install_libcugraph_tests.sh
+    build:
+      number: {{ GIT_DESCRIBE_NUMBER }}
+      string: cuda{{ cuda_major }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}
+    requirements:
+      build:
+        - cmake {{ cmake_version }}
+      run:
+        - {{ pin_subpackage('libcugraph', exact=True) }}
+        - {{ pin_subpackage('libcugraph_etl', exact=True) }}
+        - cudatoolkit {{ cuda_spec }}
+        - gtest {{ gtest_version }}
+        - gmock {{ gtest_version }}
+    about:
+      home: http://rapids.ai/
+      license: Apache-2.0
+      license_file: ../../../LICENSE
+      summary: libcugraph test & benchmark executables