Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[REVIEW] FIX Remove random build directory generation for ccache #7508

Merged
merged 9 commits into from
Mar 8, 2021
Merged

[REVIEW] FIX Remove random build directory generation for ccache #7508

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
3b5587e
FIX Remove random build directory generation for ccache
dillon-cullinan Mar 3, 2021
1d5c333
FIX Get correct file output location
dillon-cullinan Mar 3, 2021
8af337f
FIX Separate build dirs for libs
dillon-cullinan Mar 4, 2021
431d6ac
TST Test moving work dir
dillon-cullinan Mar 5, 2021
a1010dc
TST Fix copy command
dillon-cullinan Mar 5, 2021
c823707
FIX Keep conda build dir consistent across all packages
dillon-cullinan Mar 6, 2021
44439a9
FIX Handle artifact directory changing based off conda build dir
dillon-cullinan Mar 6, 2021
3369730
FIX Define artifact directory in GPU build
dillon-cullinan Mar 6, 2021
d13cbaf
FIX Remove unnecessary if block
dillon-cullinan Mar 6, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 14 additions & 8 deletions ci/cpu/build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,9 @@ export CUDA_REL=${CUDA_VERSION%.*}
export GPUCI_CONDA_RETRY_MAX=1
export GPUCI_CONDA_RETRY_SLEEP=30

# Use Ninja to build
# Use Ninja to build, setup Conda Build Dir
export CMAKE_GENERATOR="Ninja"
export CONDA_BLD_DIR="${WORKSPACE}/.conda-bld"

# Switch to project root; also root of repo checkout
cd $WORKSPACE
Expand Down Expand Up @@ -63,29 +64,34 @@ if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
CONDA_CHANNEL=""
else
CONDA_BUILD_ARGS="--dirty --no-remove-work-dir"
CONDA_CHANNEL="-c $WORKSPACE/ci/artifacts/cudf/cpu/conda-bld/"
CONDA_CHANNEL="-c $WORKSPACE/ci/artifacts/cudf/cpu/.conda-bld/"
fi

if [ "$BUILD_LIBCUDF" == '1' ]; then
gpuci_logger "Build conda pkg for libcudf"
gpuci_conda_retry build conda/recipes/libcudf $CONDA_BUILD_ARGS
gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libcudf $CONDA_BUILD_ARGS
mkdir -p ${CONDA_BLD_DIR}/libcudf/work
cp -r ${CONDA_BLD_DIR}/work/* ${CONDA_BLD_DIR}/libcudf/work


gpuci_logger "Build conda pkg for libcudf_kafka"
gpuci_conda_retry build conda/recipes/libcudf_kafka $CONDA_BUILD_ARGS
gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libcudf_kafka $CONDA_BUILD_ARGS
mkdir -p ${CONDA_BLD_DIR}/libcudf_kafka/work
cp -r ${CONDA_BLD_DIR}/work/* ${CONDA_BLD_DIR}/libcudf_kafka/work
fi

if [ "$BUILD_CUDF" == '1' ]; then
gpuci_logger "Build conda pkg for cudf"
gpuci_conda_retry build conda/recipes/cudf --python=$PYTHON $CONDA_BUILD_ARGS $CONDA_CHANNEL
gpuci_conda_retry build --croot ${CONDA_BLD_DIR} conda/recipes/cudf --python=$PYTHON $CONDA_BUILD_ARGS $CONDA_CHANNEL

gpuci_logger "Build conda pkg for dask-cudf"
gpuci_conda_retry build conda/recipes/dask-cudf --python=$PYTHON $CONDA_BUILD_ARGS $CONDA_CHANNEL
gpuci_conda_retry build --croot ${CONDA_BLD_DIR} conda/recipes/dask-cudf --python=$PYTHON $CONDA_BUILD_ARGS $CONDA_CHANNEL

gpuci_logger "Build conda pkg for cudf_kafka"
gpuci_conda_retry build conda/recipes/cudf_kafka --python=$PYTHON $CONDA_BUILD_ARGS $CONDA_CHANNEL
gpuci_conda_retry build --croot ${CONDA_BLD_DIR} conda/recipes/cudf_kafka --python=$PYTHON $CONDA_BUILD_ARGS $CONDA_CHANNEL

gpuci_logger "Build conda pkg for custreamz"
gpuci_conda_retry build conda/recipes/custreamz --python=$PYTHON $CONDA_BUILD_ARGS $CONDA_CHANNEL
gpuci_conda_retry build --croot ${CONDA_BLD_DIR} conda/recipes/custreamz --python=$PYTHON $CONDA_BUILD_ARGS $CONDA_CHANNEL
fi
################################################################################
# UPLOAD - Conda packages
Expand Down
12 changes: 6 additions & 6 deletions ci/cpu/upload.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,12 +28,12 @@ fi
################################################################################

gpuci_logger "Get conda file output locations"
export LIBCUDF_FILE=`conda build conda/recipes/libcudf --output`
export LIBCUDF_KAFKA_FILE=`conda build conda/recipes/libcudf_kafka --output`
export CUDF_FILE=`conda build conda/recipes/cudf --python=$PYTHON --output`
export DASK_CUDF_FILE=`conda build conda/recipes/dask-cudf --python=$PYTHON --output`
export CUDF_KAFKA_FILE=`conda build conda/recipes/cudf_kafka --python=$PYTHON --output`
export CUSTREAMZ_FILE=`conda build conda/recipes/custreamz --python=$PYTHON --output`
export LIBCUDF_FILE=`conda build --no-build-id --croot ${WORKSPACE}/.conda-bld conda/recipes/libcudf --output`
export LIBCUDF_KAFKA_FILE=`conda build --no-build-id --croot ${WORKSPACE}/.conda-bld conda/recipes/libcudf_kafka --output`
export CUDF_FILE=`conda build --croot ${CONDA_BLD_DIR} conda/recipes/cudf --python=$PYTHON --output`
export DASK_CUDF_FILE=`conda build --croot ${CONDA_BLD_DIR} conda/recipes/dask-cudf --python=$PYTHON --output`
export CUDF_KAFKA_FILE=`conda build --croot ${CONDA_BLD_DIR} conda/recipes/cudf_kafka --python=$PYTHON --output`
export CUSTREAMZ_FILE=`conda build --croot ${CONDA_BLD_DIR} conda/recipes/custreamz --python=$PYTHON --output`

################################################################################
# UPLOAD - Conda packages
Expand Down
7 changes: 4 additions & 3 deletions ci/gpu/build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ cd $WORKSPACE

# Determine CUDA release version
export CUDA_REL=${CUDA_VERSION%.*}
export CONDA_ARTIFACT_PATH=${WORKSPACE}/ci/artifacts/cudf/cpu/.conda-bld/

# Parse git describe
export GIT_DESCRIBE_TAG=`git describe --tags`
Expand Down Expand Up @@ -170,15 +171,15 @@ else
${gt} --gtest_output=xml:${WORKSPACE}/test-results/
done

CUDF_CONDA_FILE=`find $WORKSPACE/ci/artifacts/cudf/cpu/conda-bld/ -name "libcudf-*.tar.bz2"`
CUDF_CONDA_FILE=`find ${CONDA_ARTIFACT_PATH} -name "libcudf-*.tar.bz2"`
CUDF_CONDA_FILE=`basename "$CUDF_CONDA_FILE" .tar.bz2` #get filename without extension
CUDF_CONDA_FILE=${CUDF_CONDA_FILE//-/=} #convert to conda install
KAFKA_CONDA_FILE=`find $WORKSPACE/ci/artifacts/cudf/cpu/conda-bld/ -name "libcudf_kafka-*.tar.bz2"`
KAFKA_CONDA_FILE=`find ${CONDA_ARTIFACT_PATH} -name "libcudf_kafka-*.tar.bz2"`
KAFKA_CONDA_FILE=`basename "$KAFKA_CONDA_FILE" .tar.bz2` #get filename without extension
KAFKA_CONDA_FILE=${KAFKA_CONDA_FILE//-/=} #convert to conda install

gpuci_logger "Installing $CUDF_CONDA_FILE & $KAFKA_CONDA_FILE"
conda install -c $WORKSPACE/ci/artifacts/cudf/cpu/conda-bld/ "$CUDF_CONDA_FILE" "$KAFKA_CONDA_FILE"
conda install -c ${CONDA_ARTIFACT_PATH} "$CUDF_CONDA_FILE" "$KAFKA_CONDA_FILE"

install_dask

Expand Down