From 85ffea19cf4222f2e9290c266dd1c65438a32fca Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Thu, 18 May 2023 00:22:53 +1100 Subject: [PATCH 1/9] update arrow ci support files --- .../linux_64_cuda_compiler_version10.2.yaml | 15 +++++++----- .../linux_64_cuda_compiler_versionNone.yaml | 21 +++++++++-------- ...nux_aarch64_cuda_compiler_versionNone.yaml | 23 ++++++++++++------- ...nux_ppc64le_cuda_compiler_versionNone.yaml | 23 ++++++++++++------- .../conda-recipes/.ci_support/osx_64_.yaml | 18 ++++++++------- .../conda-recipes/.ci_support/osx_arm64_.yaml | 18 ++++++++------- .../win_64_cuda_compiler_version10.2.yaml | 18 ++++++++++----- .../win_64_cuda_compiler_versionNone.yaml | 20 ++++++++++------ 8 files changed, 96 insertions(+), 60 deletions(-) diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2.yaml index 5d80a17c4dfd7..e135017bbaa61 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2.yaml @@ -1,5 +1,5 @@ aws_crt_cpp: -- 0.19.8 +- 0.20.1 aws_sdk_cpp: - 1.10.57 bzip2: @@ -33,11 +33,13 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.8.0 +- 2.10.1 libabseil: - '20230125' +libevent: +- 2.1.12 libgrpc: -- '1.52' +- '1.54' libprotobuf: - '3.21' lz4_c: @@ -45,8 +47,8 @@ lz4_c: numpy: - '1.21' - '1.23' -- '1.20' -- '1.20' +- '1.21' +- '1.21' openssl: - '3' orc: @@ -61,7 +63,7 @@ python: - 3.8.* *_cpython - 3.9.* *_cpython re2: -- 2023.02.02 +- 2023.03.02 snappy: - '1' target_platform: @@ -73,6 +75,7 @@ ucx: zip_keys: - - c_compiler_version - cxx_compiler_version + - cuda_compiler - cuda_compiler_version - cdt_name - docker_image diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNone.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNone.yaml index 39b25b44690d7..08590f860a5c3 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNone.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNone.yaml @@ -1,5 +1,5 @@ aws_crt_cpp: -- 0.19.8 +- 0.20.1 aws_sdk_cpp: - 1.10.57 bzip2: @@ -9,7 +9,7 @@ c_ares: c_compiler: - gcc c_compiler_version: -- '11' +- '12' cdt_name: - cos6 channel_sources: @@ -17,7 +17,7 @@ channel_sources: channel_targets: - conda-forge main cuda_compiler: -- nvcc +- None cuda_compiler_version: - None cuda_compiler_version_min: @@ -25,7 +25,7 @@ cuda_compiler_version_min: cxx_compiler: - gxx cxx_compiler_version: -- '11' +- '12' docker_image: - quay.io/condaforge/linux-anvil-cos7-x86_64 gflags: @@ -33,11 +33,13 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.8.0 +- 2.10.1 libabseil: - '20230125' +libevent: +- 2.1.12 libgrpc: -- '1.52' +- '1.54' libprotobuf: - '3.21' lz4_c: @@ -45,8 +47,8 @@ lz4_c: numpy: - '1.21' - '1.23' -- '1.20' -- '1.20' +- '1.21' +- '1.21' openssl: - '3' orc: @@ -61,7 +63,7 @@ python: - 3.8.* *_cpython - 3.9.* *_cpython re2: -- 2023.02.02 +- 2023.03.02 snappy: - '1' target_platform: @@ -73,6 +75,7 @@ ucx: zip_keys: - - c_compiler_version - cxx_compiler_version + - cuda_compiler - cuda_compiler_version - cdt_name - docker_image diff --git a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNone.yaml b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNone.yaml index af0fc2dcd255e..ef792635029d0 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNone.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNone.yaml @@ -1,7 +1,7 @@ BUILD: - aarch64-conda_cos7-linux-gnu aws_crt_cpp: -- 0.19.8 +- 0.20.1 aws_sdk_cpp: - 1.10.57 bzip2: @@ -11,7 +11,7 @@ c_ares: c_compiler: - gcc c_compiler_version: -- '11' +- '12' cdt_arch: - aarch64 cdt_name: @@ -20,12 +20,16 @@ channel_sources: - conda-forge channel_targets: - conda-forge main +cuda_compiler: +- None cuda_compiler_version: - None +cuda_compiler_version_min: +- '11.2' cxx_compiler: - gxx cxx_compiler_version: -- '11' +- '12' docker_image: - quay.io/condaforge/linux-anvil-cos7-x86_64 gflags: @@ -33,11 +37,13 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.8.0 +- 2.10.1 libabseil: - '20230125' +libevent: +- 2.1.12 libgrpc: -- '1.52' +- '1.54' libprotobuf: - '3.21' lz4_c: @@ -45,8 +51,8 @@ lz4_c: numpy: - '1.21' - '1.23' -- '1.20' -- '1.20' +- '1.21' +- '1.21' openssl: - '3' orc: @@ -61,7 +67,7 @@ python: - 3.8.* *_cpython - 3.9.* *_cpython re2: -- 2023.02.02 +- 2023.03.02 snappy: - '1' target_platform: @@ -73,6 +79,7 @@ ucx: zip_keys: - - c_compiler_version - cxx_compiler_version + - cuda_compiler - cuda_compiler_version - cdt_name - docker_image diff --git a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNone.yaml b/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNone.yaml index 83a1f7f740092..b9bdd01b358cf 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNone.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNone.yaml @@ -1,5 +1,5 @@ aws_crt_cpp: -- 0.19.8 +- 0.20.1 aws_sdk_cpp: - 1.10.57 bzip2: @@ -9,19 +9,23 @@ c_ares: c_compiler: - gcc c_compiler_version: -- '11' +- '12' cdt_name: - cos7 channel_sources: - conda-forge channel_targets: - conda-forge main +cuda_compiler: +- None cuda_compiler_version: - None +cuda_compiler_version_min: +- '11.2' cxx_compiler: - gxx cxx_compiler_version: -- '11' +- '12' docker_image: - quay.io/condaforge/linux-anvil-cos7-x86_64 gflags: @@ -29,11 +33,13 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.8.0 +- 2.10.1 libabseil: - '20230125' +libevent: +- 2.1.12 libgrpc: -- '1.52' +- '1.54' libprotobuf: - '3.21' lz4_c: @@ -41,8 +47,8 @@ lz4_c: numpy: - '1.21' - '1.23' -- '1.20' -- '1.20' +- '1.21' +- '1.21' openssl: - '3' orc: @@ -57,7 +63,7 @@ python: - 3.8.* *_cpython - 3.9.* *_cpython re2: -- 2023.02.02 +- 2023.03.02 snappy: - '1' target_platform: @@ -69,6 +75,7 @@ ucx: zip_keys: - - c_compiler_version - cxx_compiler_version + - cuda_compiler - cuda_compiler_version - cdt_name - docker_image diff --git a/dev/tasks/conda-recipes/.ci_support/osx_64_.yaml b/dev/tasks/conda-recipes/.ci_support/osx_64_.yaml index 0cf990cc113f2..0774679775a61 100644 --- a/dev/tasks/conda-recipes/.ci_support/osx_64_.yaml +++ b/dev/tasks/conda-recipes/.ci_support/osx_64_.yaml @@ -1,7 +1,7 @@ MACOSX_DEPLOYMENT_TARGET: - '10.9' aws_crt_cpp: -- 0.19.8 +- 0.20.1 aws_sdk_cpp: - 1.10.57 bzip2: @@ -11,7 +11,7 @@ c_ares: c_compiler: - clang c_compiler_version: -- '14' +- '15' channel_sources: - conda-forge channel_targets: @@ -21,17 +21,19 @@ cuda_compiler_version: cxx_compiler: - clangxx cxx_compiler_version: -- '14' +- '15' gflags: - '2.2' glog: - '0.6' google_cloud_cpp: -- 2.8.0 +- 2.10.1 libabseil: - '20230125' +libevent: +- 2.1.12 libgrpc: -- '1.52' +- '1.54' libprotobuf: - '3.21' lz4_c: @@ -41,8 +43,8 @@ macos_machine: numpy: - '1.21' - '1.23' -- '1.20' -- '1.20' +- '1.21' +- '1.21' openssl: - '3' orc: @@ -57,7 +59,7 @@ python: - 3.8.* *_cpython - 3.9.* *_cpython re2: -- 2023.02.02 +- 2023.03.02 snappy: - '1' target_platform: diff --git a/dev/tasks/conda-recipes/.ci_support/osx_arm64_.yaml b/dev/tasks/conda-recipes/.ci_support/osx_arm64_.yaml index 3faa6278e81e2..bd834a4b66cb0 100644 --- a/dev/tasks/conda-recipes/.ci_support/osx_arm64_.yaml +++ b/dev/tasks/conda-recipes/.ci_support/osx_arm64_.yaml @@ -1,7 +1,7 @@ MACOSX_DEPLOYMENT_TARGET: - '11.0' aws_crt_cpp: -- 0.19.8 +- 0.20.1 aws_sdk_cpp: - 1.10.57 bzip2: @@ -11,7 +11,7 @@ c_ares: c_compiler: - clang c_compiler_version: -- '14' +- '15' channel_sources: - conda-forge channel_targets: @@ -21,17 +21,19 @@ cuda_compiler_version: cxx_compiler: - clangxx cxx_compiler_version: -- '14' +- '15' gflags: - '2.2' glog: - '0.6' google_cloud_cpp: -- 2.8.0 +- 2.10.1 libabseil: - '20230125' +libevent: +- 2.1.12 libgrpc: -- '1.52' +- '1.54' libprotobuf: - '3.21' lz4_c: @@ -41,8 +43,8 @@ macos_machine: numpy: - '1.21' - '1.23' -- '1.20' -- '1.20' +- '1.21' +- '1.21' openssl: - '3' orc: @@ -57,7 +59,7 @@ python: - 3.8.* *_cpython - 3.9.* *_cpython re2: -- 2023.02.02 +- 2023.03.02 snappy: - '1' target_platform: diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2.yaml index 6ea00e3bd0d3f..dc1c64446b4da 100644 --- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2.yaml @@ -1,3 +1,5 @@ +aws_crt_cpp: +- 0.20.1 aws_sdk_cpp: - 1.10.57 bzip2: @@ -23,15 +25,17 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.8.0 +- 2.10.1 libabseil: - '20230125' libcrc32c: - '1.1' libcurl: -- '7' +- '8' +libevent: +- 2.1.12 libgrpc: -- '1.52' +- '1.54' libprotobuf: - '3.21' lz4_c: @@ -39,8 +43,8 @@ lz4_c: numpy: - '1.21' - '1.23' -- '1.20' -- '1.20' +- '1.21' +- '1.21' openssl: - '3' orc: @@ -55,7 +59,7 @@ python: - 3.8.* *_cpython - 3.9.* *_cpython re2: -- 2023.02.02 +- 2023.03.02 snappy: - '1' target_platform: @@ -63,6 +67,8 @@ target_platform: thrift_cpp: - 0.18.1 zip_keys: +- - cuda_compiler + - cuda_compiler_version - - python - numpy zlib: diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNone.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNone.yaml index 183356662c648..e9dee6a0eac0d 100644 --- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNone.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNone.yaml @@ -1,3 +1,5 @@ +aws_crt_cpp: +- 0.20.1 aws_sdk_cpp: - 1.10.57 bzip2: @@ -11,7 +13,7 @@ channel_sources: channel_targets: - conda-forge main cuda_compiler: -- nvcc +- None cuda_compiler_version: - None cuda_compiler_version_min: @@ -23,15 +25,17 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.8.0 +- 2.10.1 libabseil: - '20230125' libcrc32c: - '1.1' libcurl: -- '7' +- '8' +libevent: +- 2.1.12 libgrpc: -- '1.52' +- '1.54' libprotobuf: - '3.21' lz4_c: @@ -39,8 +43,8 @@ lz4_c: numpy: - '1.21' - '1.23' -- '1.20' -- '1.20' +- '1.21' +- '1.21' openssl: - '3' orc: @@ -55,7 +59,7 @@ python: - 3.8.* *_cpython - 3.9.* *_cpython re2: -- 2023.02.02 +- 2023.03.02 snappy: - '1' target_platform: @@ -63,6 +67,8 @@ target_platform: thrift_cpp: - 0.18.1 zip_keys: +- - cuda_compiler + - cuda_compiler_version - - python - numpy zlib: From 5b8f4a9e7d8fbf796b1e9632053fdd1657fd8d17 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Thu, 18 May 2023 00:26:49 +1100 Subject: [PATCH 2/9] sync current state of pyarrow recipe --- dev/tasks/conda-recipes/arrow-cpp/activate.sh | 33 +++++++++----- .../{bld-arrow.bat => build-arrow.bat} | 3 +- .../conda-recipes/arrow-cpp/build-arrow.sh | 20 +++++---- .../{bld-pyarrow.bat => build-pyarrow.bat} | 14 +----- .../conda-recipes/arrow-cpp/build-pyarrow.sh | 1 + dev/tasks/conda-recipes/arrow-cpp/meta.yaml | 44 +++++++++---------- 6 files changed, 58 insertions(+), 57 deletions(-) rename dev/tasks/conda-recipes/arrow-cpp/{bld-arrow.bat => build-arrow.bat} (96%) rename dev/tasks/conda-recipes/arrow-cpp/{bld-pyarrow.bat => build-pyarrow.bat} (54%) diff --git a/dev/tasks/conda-recipes/arrow-cpp/activate.sh b/dev/tasks/conda-recipes/arrow-cpp/activate.sh index 90210fac0a034..f76d480835cfa 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/activate.sh +++ b/dev/tasks/conda-recipes/arrow-cpp/activate.sh @@ -8,23 +8,32 @@ # is benign and doesn't need to be deleted. # where the GDB wrappers get installed -GDB_PREFIX=$CONDA_PREFIX/share/gdb/auto-load +GDB_PREFIX="$CONDA_PREFIX/share/gdb/auto-load" +# this needs to be in sync with ARROW_GDB_INSTALL_DIR in build.sh +PLACEHOLDER="replace_this_section_with_absolute_slashed_path_to_CONDA_PREFIX" +# the paths here are intentionally stacked, see #935, resp. +# https://github.com/apache/arrow/blob/master/docs/source/cpp/gdb.rst#manual-loading +WRAPPER_DIR="$GDB_PREFIX/$CONDA_PREFIX/lib" + +mkdir -p "$WRAPPER_DIR" || true # If the directory is not writable, nothing can be done -if [ ! -w $GDB_PREFIX ]; then +if [ ! -w "$WRAPPER_DIR" ]; then return fi -# this needs to be in sync with the respective patch -PLACEHOLDER=replace_this_section_with_absolute_slashed_path_to_CONDA_PREFIX -# the paths here are intentionally stacked, see #935, resp. -# https://github.com/apache/arrow/blob/master/docs/source/cpp/gdb.rst#manual-loading -WRAPPER_DIR=$GDB_PREFIX/$CONDA_PREFIX/lib - -mkdir -p $WRAPPER_DIR # there's only one lib in that folder, but the libname changes # based on the version so use a loop instead of hardcoding it. -for f in $GDB_PREFIX/$PLACEHOLDER/lib/*.py; do - # overwrite, because we don't have deactivation (i.e. symlink remains) - ln -sf $f $WRAPPER_DIR/$(basename $f) +for f in "$GDB_PREFIX/$PLACEHOLDER/lib/"*.py; do + if [ ! -e "$f" ]; then + # If the file doesn't exist, skip this iteration of the loop. + # (This happens when no files are found, in which case the + # loop runs with f equal to the pattern itself.) + continue + fi + target="$WRAPPER_DIR/$(basename "$f")" + # We have write permissions to WRAPPER_DIR but not necessarily target. + # Thus it's safest to delete the target in case it already exists. + rm -f "$target" + ln -s "$f" "$WRAPPER_DIR/$(basename "$f")" done diff --git a/dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat b/dev/tasks/conda-recipes/arrow-cpp/build-arrow.bat similarity index 96% rename from dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat rename to dev/tasks/conda-recipes/arrow-cpp/build-arrow.bat index 60c81be741128..50f39dc30dd68 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat +++ b/dev/tasks/conda-recipes/arrow-cpp/build-arrow.bat @@ -18,8 +18,9 @@ if "%cuda_compiler_version%"=="None" ( set "READ_RECIPE_META_YAML_WHY_NOT=OFF" :: for available switches see -:: https://github.com/apache/arrow/blame/apache-arrow-11.0.0/cpp/cmake_modules/DefineOptions.cmake +:: https://github.com/apache/arrow/blame/apache-arrow-12.0.0/cpp/cmake_modules/DefineOptions.cmake cmake -G "Ninja" ^ + -DARROW_ACERO=ON ^ -DARROW_BOOST_USE_SHARED:BOOL=ON ^ -DARROW_BUILD_STATIC:BOOL=OFF ^ -DARROW_BUILD_TESTS:BOOL=OFF ^ diff --git a/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh b/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh index fb8cbade86568..5630ba78987f3 100755 --- a/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh +++ b/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh @@ -42,13 +42,19 @@ then return 1 fi fi - EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_CUDA=ON -DCUDA_TOOLKIT_ROOT_DIR=${CUDA_HOME} -DCMAKE_LIBRARY_PATH=${CUDA_HOME}/lib64/stubs" + EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_CUDA=ON -DCUDA_TOOLKIT_ROOT_DIR=${CUDA_HOME} -DCMAKE_LIBRARY_PATH=${CONDA_BUILD_SYSROOT}/lib" else EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_CUDA=OFF" fi -if [[ "${target_platform}" == "osx-arm64" ]]; then - EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DCLANG_EXECUTABLE=${BUILD_PREFIX}/bin/clang -DLLVM_LINK_EXECUTABLE=${BUILD_PREFIX}/bin/llvm-link" +if [[ "${build_platform}" != "${target_platform}" ]]; then + # point to a usable protoc/grpc_cpp_plugin if we're cross-compiling + EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DProtobuf_PROTOC_EXECUTABLE=$BUILD_PREFIX/bin/protoc" + if [[ ! -f ${BUILD_PREFIX}/bin/${CONDA_TOOLCHAIN_HOST}-clang ]]; then + ln -sf ${BUILD_PREFIX}/bin/clang ${BUILD_PREFIX}/bin/${CONDA_TOOLCHAIN_HOST}-clang + fi + EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DCLANG_EXECUTABLE=${BUILD_PREFIX}/bin/${CONDA_TOOLCHAIN_HOST}-clang" + EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DLLVM_LINK_EXECUTABLE=${BUILD_PREFIX}/bin/llvm-link" sed -ie "s;protoc-gen-grpc.*$;protoc-gen-grpc=${BUILD_PREFIX}/bin/grpc_cpp_plugin\";g" ../src/arrow/flight/CMakeLists.txt sed -ie 's;"--with-jemalloc-prefix\=je_arrow_";"--with-jemalloc-prefix\=je_arrow_" "--with-lg-page\=14";g' ../cmake_modules/ThirdpartyToolchain.cmake fi @@ -64,18 +70,14 @@ if [[ "${target_platform}" == "linux-aarch64" ]] || [[ "${target_platform}" == " export CMAKE_BUILD_PARALLEL_LEVEL=3 fi -# point to a usable protoc if we're running on a different architecture than the target -if [[ "${build_platform}" != "${target_platform}" ]]; then - EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DProtobuf_PROTOC_EXECUTABLE=$BUILD_PREFIX/bin/protoc" -fi - # reusable variable for dependencies we cannot yet unvendor export READ_RECIPE_META_YAML_WHY_NOT=OFF # for available switches see -# https://github.com/apache/arrow/blame/apache-arrow-11.0.0/cpp/cmake_modules/DefineOptions.cmake +# https://github.com/apache/arrow/blame/apache-arrow-12.0.0/cpp/cmake_modules/DefineOptions.cmake # placeholder in ARROW_GDB_INSTALL_DIR must match what's used for replacement in activate.sh cmake -GNinja \ + -DARROW_ACERO=ON \ -DARROW_BOOST_USE_SHARED=ON \ -DARROW_BUILD_BENCHMARKS=OFF \ -DARROW_BUILD_STATIC=OFF \ diff --git a/dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat b/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.bat similarity index 54% rename from dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat rename to dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.bat index 084faf74e4a10..e3eaa32bcf848 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat +++ b/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.bat @@ -2,22 +2,10 @@ pushd "%SRC_DIR%"\python -@rem the symlinks for cmake modules don't work here -@rem NOTE: In contrast to conda-forge, they work here as we clone from git. -@rem del cmake_modules\BuildUtils.cmake -@rem del cmake_modules\SetupCxxFlags.cmake -@rem del cmake_modules\CompilerInfo.cmake -@rem del cmake_modules\FindNumPy.cmake -@rem del cmake_modules\FindPythonLibsNew.cmake -@rem copy /Y "%SRC_DIR%\cpp\cmake_modules\BuildUtils.cmake" cmake_modules\ -@rem copy /Y "%SRC_DIR%\cpp\cmake_modules\SetupCxxFlags.cmake" cmake_modules\ -@rem copy /Y "%SRC_DIR%\cpp\cmake_modules\CompilerInfo.cmake" cmake_modules\ -@rem copy /Y "%SRC_DIR%\cpp\cmake_modules\FindNumPy.cmake" cmake_modules\ -@rem copy /Y "%SRC_DIR%\cpp\cmake_modules\FindPythonLibsNew.cmake" cmake_modules\ - SET ARROW_HOME=%LIBRARY_PREFIX% SET SETUPTOOLS_SCM_PRETEND_VERSION=%PKG_VERSION% SET PYARROW_BUILD_TYPE=release +SET PYARROW_WITH_ACERO=1 SET PYARROW_WITH_DATASET=1 SET PYARROW_WITH_FLIGHT=1 SET PYARROW_WITH_GANDIVA=1 diff --git a/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh b/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh index 14c67ede6324e..e0c3ae96bf55a 100755 --- a/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh +++ b/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh @@ -6,6 +6,7 @@ export ARROW_HOME=$PREFIX export PARQUET_HOME=$PREFIX export SETUPTOOLS_SCM_PRETEND_VERSION=$PKG_VERSION export PYARROW_BUILD_TYPE=release +export PYARROW_WITH_ACERO=1 export PYARROW_WITH_DATASET=1 export PYARROW_WITH_FLIGHT=1 export PYARROW_WITH_GANDIVA=1 diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml index 2f79bbe958c07..2a7a310a13069 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml +++ b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml @@ -4,7 +4,7 @@ {% set build_ext_version = ARROW_VERSION %} {% set build_ext = "cuda" if cuda_enabled else "cpu" %} {% set proc_build_number = "0" %} -{% set llvm_version = "14" %} +{% set llvm_version = "15" %} # see https://github.com/apache/arrow/blob/apache-arrow-10.0.1/cpp/CMakeLists.txt#L88-L90 {% set so_version = (version.split(".")[0] | int * 100 + version.split(".")[1] | int) ~ "." ~ version.split(".")[2] ~ ".0" %} @@ -21,9 +21,6 @@ build: # for cuda support, building with one version is enough to be compatible with # all later versions, since arrow is only using libcuda, and not libcudart. skip: true # [cuda_compiler_version not in ("None", cuda_compiler_version_min)] - # temporary: skip CUDA on aarch/ppc until cross-compilation works, see - # https://github.com/conda-forge/conda-forge-ci-setup-feedstock/pull/210 - skip: true # [(aarch64 or ppc64le) and (cuda_compiler_version != "None")] run_exports: - {{ pin_subpackage("libarrow", max_pin="x.x.x") }} @@ -57,8 +54,8 @@ outputs: - exit 0 - name: libarrow - script: build-arrow.sh # [not win] - script: bld-arrow.bat # [win] + script: build-arrow.sh # [unix] + script: build-arrow.bat # [win] version: {{ version }} build: string: h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}_{{ build_ext }} @@ -68,19 +65,20 @@ outputs: - cudatoolkit track_features: {{ "[arrow-cuda]" if cuda_enabled else "" }} missing_dso_whitelist: - - "*/libcuda.so.*" # [linux] - - "*/nvcuda.dll" # [win] + - '*/libcuda.so.*' # [linux] + - '*/nvcuda.dll' # [win] requirements: build: - {{ compiler("c") }} - {{ compiler("cxx") }} - {{ compiler("cuda") }} # [cuda_compiler_version != "None"] - - clangdev {{ llvm_version }} # [osx and arm64] - - llvmdev {{ llvm_version }} # [osx and arm64] - - gnuconfig # [osx and arm64] # needs to run protoc & grpc_cpp_plugin - libgrpc # [build_platform != target_platform] - libprotobuf # [build_platform != target_platform] + # needed for gandiva + - clangdev {{ llvm_version }} # [build_platform != target_platform] + - llvmdev {{ llvm_version }} # [build_platform != target_platform] + - gnuconfig # [build_platform != target_platform] - cmake - ninja # necessary for vendored jemalloc @@ -91,7 +89,7 @@ outputs: # https://github.com/apache/arrow/blob/apache-arrow-11.0.0/cpp/cmake_modules/ThirdpartyToolchain.cmake#L46-L75 - clangdev {{ llvm_version }} - llvmdev {{ llvm_version }} - - aws-crt-cpp # [unix] + - aws-crt-cpp - aws-sdk-cpp - boost-cpp >=1.70 - brotli @@ -111,6 +109,7 @@ outputs: # its dependencies leak into the build here - libcrc32c # [win] - libcurl # [win] + - libevent - libgrpc - libprotobuf - libutf8proc @@ -136,8 +135,8 @@ outputs: test: commands: {% set headers = [ - "arrow/api.h", "arrow/flight/types.h", "arrow/flight/sql/api.h", - "gandiva/engine.h", "parquet/api/reader.h" + "arrow/api.h", "arrow/acero/groupby.h", "arrow/flight/types.h", + "arrow/flight/sql/api.h", "gandiva/engine.h", "parquet/api/reader.h" ] %} {% for each_header in headers %} # headers @@ -146,8 +145,8 @@ outputs: {% endfor %} {% set libs = (cuda_compiler_version != "None") * ["arrow_cuda"] + [ - "arrow", "arrow_dataset", "arrow_flight", "arrow_flight_sql", - "arrow_substrait", "gandiva", "parquet" + "arrow", "arrow_acero", "arrow_dataset", "arrow_flight", + "arrow_flight_sql", "arrow_substrait", "gandiva", "parquet" ] %} {% for each_lib in libs %} # shared @@ -196,8 +195,8 @@ outputs: - exit 0 - name: pyarrow - script: build-pyarrow.sh # [not win] - script: bld-pyarrow.bat # [win] + script: build-pyarrow.sh # [unix] + script: build-pyarrow.bat # [win] version: {{ version }} build: string: py{{ CONDA_PY }}h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}_{{ build_ext }} @@ -209,8 +208,8 @@ outputs: - {{ SP_DIR }}/pyarrow missing_dso_whitelist: # not actually missing, but installed into SP_DIR, see tests - - "*/arrow_python.dll" # [win] - - "*/arrow_python_flight.dll" # [win] + - '*/arrow_python.dll' # [win] + - '*/arrow_python_flight.dll' # [win] requirements: build: - {{ compiler("c") }} @@ -288,8 +287,8 @@ outputs: summary: Python libraries for Apache Arrow - name: pyarrow-tests - script: build-pyarrow.sh # [not win] - script: bld-pyarrow.bat # [win] + script: build-pyarrow.sh # [unix] + script: build-pyarrow.bat # [win] version: {{ version }} build: string: py{{ CONDA_PY }}h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}_{{ build_ext }} @@ -373,6 +372,7 @@ outputs: {% set tests_to_skip = tests_to_skip + " or test_debug_memory_pool_disabled" %} # [aarch64 or ppc64le] {% set tests_to_skip = tests_to_skip + " or test_env_var_io_thread_count" %} # [aarch64 or ppc64le] # vvvvvvv TESTS THAT SHOULDN'T HAVE TO BE SKIPPED vvvvvvv + {% set tests_to_skip = tests_to_skip + " or test_extension_to_pandas_storage_type" %} # segfaults on OSX: to investigate ASAP {% set tests_to_skip = tests_to_skip + " or test_flight" %} # [osx] # gandiva tests are segfaulting on ppc From 45a24c36017d4b41de28b7ad3a34e4237963870a Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Thu, 18 May 2023 00:29:36 +1100 Subject: [PATCH 3/9] add cross-compiled conda jobs for CUDA on aarch64/ppc64le --- ...nux_aarch64_cuda_compiler_version11.2.yaml | 91 +++++++++++++++++++ ...nux_ppc64le_cuda_compiler_version11.2.yaml | 87 ++++++++++++++++++ dev/tasks/tasks.yml | 24 +++++ 3 files changed, 202 insertions(+) create mode 100644 dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_version11.2.yaml create mode 100644 dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_version11.2.yaml diff --git a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_version11.2.yaml b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_version11.2.yaml new file mode 100644 index 0000000000000..3853add25f5f1 --- /dev/null +++ b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_version11.2.yaml @@ -0,0 +1,91 @@ +BUILD: +- aarch64-conda_cos7-linux-gnu +aws_crt_cpp: +- 0.20.1 +aws_sdk_cpp: +- 1.10.57 +bzip2: +- '1' +c_ares: +- '1' +c_compiler: +- gcc +c_compiler_version: +- '10' +cdt_arch: +- aarch64 +cdt_name: +- cos7 +channel_sources: +- conda-forge +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- '11.2' +cuda_compiler_version_min: +- '11.2' +cxx_compiler: +- gxx +cxx_compiler_version: +- '10' +docker_image: +- quay.io/condaforge/linux-anvil-cuda:11.2 +gflags: +- '2.2' +glog: +- '0.6' +google_cloud_cpp: +- 2.10.1 +libabseil: +- '20230125' +libevent: +- 2.1.12 +libgrpc: +- '1.54' +libprotobuf: +- '3.21' +lz4_c: +- 1.9.3 +numpy: +- '1.21' +- '1.23' +- '1.21' +- '1.21' +openssl: +- '3' +orc: +- 1.8.3 +pin_run_as_build: + python: + min_pin: x.x + max_pin: x.x +python: +- 3.10.* *_cpython +- 3.11.* *_cpython +- 3.8.* *_cpython +- 3.9.* *_cpython +re2: +- 2023.03.02 +snappy: +- '1' +target_platform: +- linux-aarch64 +thrift_cpp: +- 0.18.1 +ucx: +- 1.14.0 +zip_keys: +- - c_compiler_version + - cxx_compiler_version + - cuda_compiler + - cuda_compiler_version + - cdt_name + - docker_image +- - python + - numpy +zlib: +- '1.2' +zstd: +- '1.5' diff --git a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_version11.2.yaml b/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_version11.2.yaml new file mode 100644 index 0000000000000..eef7acde228f5 --- /dev/null +++ b/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_version11.2.yaml @@ -0,0 +1,87 @@ +aws_crt_cpp: +- 0.20.1 +aws_sdk_cpp: +- 1.10.57 +bzip2: +- '1' +c_ares: +- '1' +c_compiler: +- gcc +c_compiler_version: +- '10' +cdt_name: +- cos7 +channel_sources: +- conda-forge +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- '11.2' +cuda_compiler_version_min: +- '11.2' +cxx_compiler: +- gxx +cxx_compiler_version: +- '10' +docker_image: +- quay.io/condaforge/linux-anvil-cuda:11.2 +gflags: +- '2.2' +glog: +- '0.6' +google_cloud_cpp: +- 2.10.1 +libabseil: +- '20230125' +libevent: +- 2.1.12 +libgrpc: +- '1.54' +libprotobuf: +- '3.21' +lz4_c: +- 1.9.3 +numpy: +- '1.21' +- '1.23' +- '1.21' +- '1.21' +openssl: +- '3' +orc: +- 1.8.3 +pin_run_as_build: + python: + min_pin: x.x + max_pin: x.x +python: +- 3.10.* *_cpython +- 3.11.* *_cpython +- 3.8.* *_cpython +- 3.9.* *_cpython +re2: +- 2023.03.02 +snappy: +- '1' +target_platform: +- linux-ppc64le +thrift_cpp: +- 0.18.1 +ucx: +- 1.14.0 +zip_keys: +- - c_compiler_version + - cxx_compiler_version + - cuda_compiler + - cuda_compiler_version + - cdt_name + - docker_image +- - python + - numpy +zlib: +- '1.2' +zstd: +- '1.5' diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 60ed4f440dc06..c0bb1c287282c 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -304,6 +304,18 @@ tasks: - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda + conda-linux-aarch64-cuda-py3: + ci: azure + template: conda-recipes/azure.linux.yml + params: + config: linux_aarch64_cuda_compiler_version11.2 + artifacts: + - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cuda.conda + - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cuda.conda + - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cuda.conda + - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cuda.conda + - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cuda.conda + ########################### Conda Linux (ppc64le) ########################### conda-linux-ppc64le-cpu-py3: @@ -318,6 +330,18 @@ tasks: - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda + conda-linux-ppc64le-cuda-py3: + ci: azure + template: conda-recipes/azure.linux.yml + params: + config: linux_ppc64le_cuda_compiler_version11.2 + artifacts: + - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cuda.conda + - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cuda.conda + - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cuda.conda + - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cuda.conda + - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cuda.conda + ############################## Conda OSX (x64) ############################## conda-osx-x64-cpu-r42: From 03e6af5e784e5c0c8ae90b5d040e47c72040685b Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Thu, 18 May 2023 00:30:42 +1100 Subject: [PATCH 4/9] sync r-arrow recipe --- dev/tasks/conda-recipes/r-arrow/configure.win | 4 ++-- dev/tasks/conda-recipes/r-arrow/meta.yaml | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/dev/tasks/conda-recipes/r-arrow/configure.win b/dev/tasks/conda-recipes/r-arrow/configure.win index fb16a810b8f22..0fc96576bde74 100755 --- a/dev/tasks/conda-recipes/r-arrow/configure.win +++ b/dev/tasks/conda-recipes/r-arrow/configure.win @@ -2,7 +2,7 @@ set -euxo pipefail -echo "PKG_CPPFLAGS=-DNDEBUG -I\"${LIBRARY_PREFIX}/include\" -I\"${PREFIX}/include\" -DARROW_R_WITH_PARQUET -DARROW_R_WITH_DATASET -DARROW_R_WITH_S3 -DARROW_R_WITH_JSON" > src/Makevars.win +echo "PKG_CPPFLAGS=-DNDEBUG -I\"${LIBRARY_PREFIX}/include\" -I\"${PREFIX}/include\" -DARROW_R_WITH_ACERO -DARROW_R_WITH_PARQUET -DARROW_R_WITH_DATASET -DARROW_R_WITH_S3 -DARROW_R_WITH_JSON" > src/Makevars.win echo "PKG_CXXFLAGS=\$(CXX_VISIBILITY)" >> src/Makevars.win echo 'CXX_STD=CXX17' >> src/Makevars.win -echo "PKG_LIBS=-L\"${LIBRARY_PREFIX}/lib\" -larrow_dataset -lparquet -larrow" >> src/Makevars.win +echo "PKG_LIBS=-L\"${LIBRARY_PREFIX}/lib\" -larrow_dataset -larrow_acero -lparquet -larrow" >> src/Makevars.win diff --git a/dev/tasks/conda-recipes/r-arrow/meta.yaml b/dev/tasks/conda-recipes/r-arrow/meta.yaml index 28ee8eb92c921..e8b834254f41c 100644 --- a/dev/tasks/conda-recipes/r-arrow/meta.yaml +++ b/dev/tasks/conda-recipes/r-arrow/meta.yaml @@ -10,8 +10,6 @@ source: path: ../../../../ build: - # 4.1 not usable anymore unless https://github.com/conda-forge/r-base-feedstock/pull/236 gets merged - skip: true # [unix and (r_base == "4.1")] merge_build_host: true # [win] number: 0 rpaths: From 8ef347bb3755415978df3eae499caa8dfc0dce39 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Thu, 18 May 2023 00:33:38 +1100 Subject: [PATCH 5/9] sync ci support for r-arrow, reinstate R 4.1 builds --- .../.ci_support/r/linux_64_r_base4.1.yaml | 27 +++++++++++++ .../.ci_support/r/linux_64_r_base4.2.yaml | 4 +- .../r/linux_aarch64_r_base4.1.yaml | 31 ++++++++++++++ .../r/linux_aarch64_r_base4.2.yaml | 4 +- .../.ci_support/r/osx_64_r_base4.1.yaml | 27 +++++++++++++ .../.ci_support/r/osx_64_r_base4.2.yaml | 4 +- .../.ci_support/r/osx_arm64_r_base4.1.yaml | 27 +++++++++++++ .../.ci_support/r/osx_arm64_r_base4.2.yaml | 4 +- dev/tasks/tasks.yml | 40 +++++++++++++++++++ 9 files changed, 160 insertions(+), 8 deletions(-) create mode 100644 dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.1.yaml create mode 100644 dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.1.yaml create mode 100644 dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.1.yaml create mode 100644 dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.1.yaml diff --git a/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.1.yaml b/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.1.yaml new file mode 100644 index 0000000000000..e63767cbe9771 --- /dev/null +++ b/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.1.yaml @@ -0,0 +1,27 @@ +c_compiler: +- gcc +c_compiler_version: +- '12' +cdt_name: +- cos6 +channel_sources: +- conda-forge +channel_targets: +- conda-forge main +cxx_compiler: +- gxx +cxx_compiler_version: +- '12' +docker_image: +- quay.io/condaforge/linux-anvil-cos7-x86_64 +pin_run_as_build: + r-base: + min_pin: x.x + max_pin: x.x +r_base: +- '4.1' +target_platform: +- linux-64 +zip_keys: +- - c_compiler_version + - cxx_compiler_version diff --git a/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.2.yaml b/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.2.yaml index 38753baa7ed09..6e661e1357d22 100644 --- a/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.2.yaml +++ b/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.2.yaml @@ -1,7 +1,7 @@ c_compiler: - gcc c_compiler_version: -- '11' +- '12' cdt_name: - cos6 channel_sources: @@ -11,7 +11,7 @@ channel_targets: cxx_compiler: - gxx cxx_compiler_version: -- '11' +- '12' docker_image: - quay.io/condaforge/linux-anvil-cos7-x86_64 pin_run_as_build: diff --git a/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.1.yaml b/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.1.yaml new file mode 100644 index 0000000000000..2b80b020fdc0b --- /dev/null +++ b/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.1.yaml @@ -0,0 +1,31 @@ +BUILD: +- aarch64-conda_cos7-linux-gnu +c_compiler: +- gcc +c_compiler_version: +- '12' +cdt_arch: +- aarch64 +cdt_name: +- cos7 +channel_sources: +- conda-forge +channel_targets: +- conda-forge main +cxx_compiler: +- gxx +cxx_compiler_version: +- '12' +docker_image: +- quay.io/condaforge/linux-anvil-cos7-x86_64 +pin_run_as_build: + r-base: + min_pin: x.x + max_pin: x.x +r_base: +- '4.1' +target_platform: +- linux-aarch64 +zip_keys: +- - c_compiler_version + - cxx_compiler_version diff --git a/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.2.yaml b/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.2.yaml index 2913bbb4f141f..9dcd0c34c851c 100644 --- a/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.2.yaml +++ b/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.2.yaml @@ -3,7 +3,7 @@ BUILD: c_compiler: - gcc c_compiler_version: -- '11' +- '12' cdt_arch: - aarch64 cdt_name: @@ -15,7 +15,7 @@ channel_targets: cxx_compiler: - gxx cxx_compiler_version: -- '11' +- '12' docker_image: - quay.io/condaforge/linux-anvil-cos7-x86_64 pin_run_as_build: diff --git a/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.1.yaml b/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.1.yaml new file mode 100644 index 0000000000000..6be6c2f5462c5 --- /dev/null +++ b/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.1.yaml @@ -0,0 +1,27 @@ +MACOSX_DEPLOYMENT_TARGET: +- '10.9' +c_compiler: +- clang +c_compiler_version: +- '15' +channel_sources: +- conda-forge +channel_targets: +- conda-forge main +cxx_compiler: +- clangxx +cxx_compiler_version: +- '15' +macos_machine: +- x86_64-apple-darwin13.4.0 +pin_run_as_build: + r-base: + min_pin: x.x + max_pin: x.x +r_base: +- '4.1' +target_platform: +- osx-64 +zip_keys: +- - c_compiler_version + - cxx_compiler_version diff --git a/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.2.yaml b/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.2.yaml index 25437ee4adcfe..2116eaf7b8b21 100644 --- a/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.2.yaml +++ b/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.2.yaml @@ -3,7 +3,7 @@ MACOSX_DEPLOYMENT_TARGET: c_compiler: - clang c_compiler_version: -- '14' +- '15' channel_sources: - conda-forge channel_targets: @@ -11,7 +11,7 @@ channel_targets: cxx_compiler: - clangxx cxx_compiler_version: -- '14' +- '15' macos_machine: - x86_64-apple-darwin13.4.0 pin_run_as_build: diff --git a/dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.1.yaml b/dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.1.yaml new file mode 100644 index 0000000000000..0ce856fcccf5c --- /dev/null +++ b/dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.1.yaml @@ -0,0 +1,27 @@ +MACOSX_DEPLOYMENT_TARGET: +- '11.0' +c_compiler: +- clang +c_compiler_version: +- '15' +channel_sources: +- conda-forge +channel_targets: +- conda-forge main +cxx_compiler: +- clangxx +cxx_compiler_version: +- '15' +macos_machine: +- arm64-apple-darwin20.0.0 +pin_run_as_build: + r-base: + min_pin: x.x + max_pin: x.x +r_base: +- '4.1' +target_platform: +- osx-arm64 +zip_keys: +- - c_compiler_version + - cxx_compiler_version diff --git a/dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.2.yaml b/dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.2.yaml index 1557b23ff96af..af8a07c42208e 100644 --- a/dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.2.yaml +++ b/dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.2.yaml @@ -3,7 +3,7 @@ MACOSX_DEPLOYMENT_TARGET: c_compiler: - clang c_compiler_version: -- '14' +- '15' channel_sources: - conda-forge channel_targets: @@ -11,7 +11,7 @@ channel_targets: cxx_compiler: - clangxx cxx_compiler_version: -- '14' +- '15' macos_machine: - arm64-apple-darwin20.0.0 pin_run_as_build: diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index c0bb1c287282c..fc8f0d6d245b0 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -246,6 +246,16 @@ tasks: # generated and to be synced regularly from the feedstock. We have no way # yet to generate them inside the arrow repository automatically. + conda-linux-x64-cpu-r41: + ci: azure + template: conda-recipes/azure.linux.yml + params: + config: linux_64_cuda_compiler_versionNone + r_config: linux_64_r_base4.1 + artifacts: + - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda + - r-arrow-{no_rc_version}-r41(h[a-z0-9]+)_0.conda + conda-linux-x64-cpu-r42: ci: azure template: conda-recipes/azure.linux.yml @@ -282,6 +292,16 @@ tasks: ########################### Conda Linux (aarch64) ########################### + conda-linux-aarch64-cpu-r41: + ci: azure + template: conda-recipes/azure.linux.yml + params: + config: linux_aarch64_cuda_compiler_versionNone + r_config: linux_aarch64_r_base4.1 + artifacts: + - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda + - r-arrow-{no_rc_version}-r41(h[a-z0-9]+)_0.conda + conda-linux-aarch64-cpu-r42: ci: azure template: conda-recipes/azure.linux.yml @@ -344,6 +364,16 @@ tasks: ############################## Conda OSX (x64) ############################## + conda-osx-x64-cpu-r41: + ci: azure + template: conda-recipes/azure.osx.yml + params: + config: osx_64_ + r_config: osx_64_r_base4.1 + artifacts: + - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda + - r-arrow-{no_rc_version}-r41(h[a-z0-9]+)_0.conda + conda-osx-x64-cpu-r42: ci: azure template: conda-recipes/azure.osx.yml @@ -368,6 +398,16 @@ tasks: ############################# Conda OSX (arm64) ############################# + conda-osx-arm64-cpu-r41: + ci: azure + template: conda-recipes/azure.osx.yml + params: + config: osx_arm64_ + r_config: osx_arm64_r_base4.1 + artifacts: + - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda + - r-arrow-{no_rc_version}-r41(h[a-z0-9]+)_0.conda + conda-osx-arm64-cpu-r42: ci: azure template: conda-recipes/azure.osx.yml From 1bdaefa13e5e3e678ddae183808d1c0dca8a9f0c Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Sun, 16 Jul 2023 22:09:14 +0200 Subject: [PATCH 6/9] resync arrow-cpp --- dev/tasks/conda-recipes/arrow-cpp/activate.sh | 71 ++++++++++---- .../conda-recipes/arrow-cpp/build-arrow.bat | 11 +-- .../conda-recipes/arrow-cpp/build-arrow.sh | 23 ++--- .../conda-recipes/arrow-cpp/build-pyarrow.sh | 4 +- dev/tasks/conda-recipes/arrow-cpp/meta.yaml | 92 ++++++++++++------- 5 files changed, 126 insertions(+), 75 deletions(-) diff --git a/dev/tasks/conda-recipes/arrow-cpp/activate.sh b/dev/tasks/conda-recipes/arrow-cpp/activate.sh index f76d480835cfa..8757612781bbe 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/activate.sh +++ b/dev/tasks/conda-recipes/arrow-cpp/activate.sh @@ -7,33 +7,70 @@ # doesn't come with a deactivate script, because the symlink # is benign and doesn't need to be deleted. +_la_log() { + if [ "${CF_LIBARROW_ACTIVATE_LOGGING:-}" = "1" ]; then + # The following loop is necessary to handle multi-line strings + # like for the output of `ls -al`. + printf '%s\n' "$*" | while IFS= read -r line + do + echo "$CONDA_PREFIX/etc/conda/activate.d/libarrow_activate.sh DEBUG: $line" + done + fi +} + +_la_log "Beginning libarrow activation." + # where the GDB wrappers get installed -GDB_PREFIX="$CONDA_PREFIX/share/gdb/auto-load" +_la_gdb_prefix="$CONDA_PREFIX/share/gdb/auto-load" # this needs to be in sync with ARROW_GDB_INSTALL_DIR in build.sh -PLACEHOLDER="replace_this_section_with_absolute_slashed_path_to_CONDA_PREFIX" +_la_placeholder="replace_this_section_with_absolute_slashed_path_to_CONDA_PREFIX" # the paths here are intentionally stacked, see #935, resp. # https://github.com/apache/arrow/blob/master/docs/source/cpp/gdb.rst#manual-loading -WRAPPER_DIR="$GDB_PREFIX/$CONDA_PREFIX/lib" +_la_symlink_dir="$_la_gdb_prefix/$CONDA_PREFIX/lib" +_la_orig_install_dir="$_la_gdb_prefix/$_la_placeholder/lib" -mkdir -p "$WRAPPER_DIR" || true -# If the directory is not writable, nothing can be done -if [ ! -w "$WRAPPER_DIR" ]; then - return -fi +_la_log " _la_gdb_prefix: $_la_gdb_prefix" +_la_log " _la_placeholder: $_la_placeholder" +_la_log " _la_symlink_dir: $_la_symlink_dir" +_la_log " _la_orig_install_dir: $_la_orig_install_dir" +_la_log " content of that folder:" +_la_log "$(ls -al "$_la_orig_install_dir" | sed 's/^/ /')" -# there's only one lib in that folder, but the libname changes +# there's only one lib in the _la_orig_install_dir folder, but the libname changes # based on the version so use a loop instead of hardcoding it. -for f in "$GDB_PREFIX/$PLACEHOLDER/lib/"*.py; do - if [ ! -e "$f" ]; then +for _la_target in "$_la_orig_install_dir/"*.py; do + if [ ! -e "$_la_target" ]; then # If the file doesn't exist, skip this iteration of the loop. # (This happens when no files are found, in which case the - # loop runs with f equal to the pattern itself.) + # loop runs with target equal to the pattern itself.) + _la_log 'Folder $_la_orig_install_dir seems to not contain .py files, skipping' + continue + fi + _la_symlink="$_la_symlink_dir/$(basename "$_la_target")" + _la_log " _la_target: $_la_target" + _la_log " _la_symlink: $_la_symlink" + if [ -L "$_la_symlink" ] && [ "$(readlink "$_la_symlink")" = "$_la_target" ]; then + _la_log 'symlink $_la_symlink already exists and points to $_la_target, skipping.' + continue + fi + _la_log 'Creating symlink $_la_symlink pointing to $_la_target' + mkdir -p "$_la_symlink_dir" || true + # this check also creates the symlink; if it fails, we enter the if-branch. + if ! ln -sf "$_la_target" "$_la_symlink"; then + echo -n "${BASH_SOURCE[0]} ERROR: Failed to create symlink from " + echo -n "'$_la_target' to '$_la_symlink'" + echo continue fi - target="$WRAPPER_DIR/$(basename "$f")" - # We have write permissions to WRAPPER_DIR but not necessarily target. - # Thus it's safest to delete the target in case it already exists. - rm -f "$target" - ln -s "$f" "$WRAPPER_DIR/$(basename "$f")" done + +_la_log "Libarrow activation complete." + +unset _la_gdb_prefix +unset _la_log +unset _la_orig_install_dir +unset _la_placeholder +unset _la_symlink +unset _la_symlink_dir +unset _la_target diff --git a/dev/tasks/conda-recipes/arrow-cpp/build-arrow.bat b/dev/tasks/conda-recipes/arrow-cpp/build-arrow.bat index 50f39dc30dd68..1268771643d4f 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/build-arrow.bat +++ b/dev/tasks/conda-recipes/arrow-cpp/build-arrow.bat @@ -1,16 +1,12 @@ @echo on -mkdir "%SRC_DIR%"\cpp\build -pushd "%SRC_DIR%"\cpp\build +mkdir cpp\build +pushd cpp\build :: Enable CUDA support if "%cuda_compiler_version%"=="None" ( set "EXTRA_CMAKE_ARGS=-DARROW_CUDA=OFF" ) else ( - REM this should move to nvcc-feedstock - set "CUDA_PATH=%CUDA_PATH:\=/%" - set "CUDA_HOME=%CUDA_HOME:\=/%" - set "EXTRA_CMAKE_ARGS=-DARROW_CUDA=ON" ) @@ -70,3 +66,6 @@ cmake --build . --target install --config Release if %ERRORLEVEL% neq 0 exit 1 popd + +:: clean up between builds (and to save space) +rmdir /s /q cpp\build diff --git a/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh b/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh index 5630ba78987f3..dc588f9473870 100755 --- a/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh +++ b/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh @@ -30,18 +30,6 @@ fi # Enable CUDA support if [[ ! -z "${cuda_compiler_version+x}" && "${cuda_compiler_version}" != "None" ]] then - if [[ -z "${CUDA_HOME+x}" ]] - then - echo "cuda_compiler_version=${cuda_compiler_version} CUDA_HOME=$CUDA_HOME" - CUDA_GDB_EXECUTABLE=$(which cuda-gdb || exit 0) - if [[ -n "$CUDA_GDB_EXECUTABLE" ]] - then - CUDA_HOME=$(dirname $(dirname $CUDA_GDB_EXECUTABLE)) - else - echo "Cannot determine CUDA_HOME: cuda-gdb not in PATH" - return 1 - fi - fi EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_CUDA=ON -DCUDA_TOOLKIT_ROOT_DIR=${CUDA_HOME} -DCMAKE_LIBRARY_PATH=${CONDA_BUILD_SYSROOT}/lib" else EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_CUDA=OFF" @@ -56,7 +44,7 @@ if [[ "${build_platform}" != "${target_platform}" ]]; then EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DCLANG_EXECUTABLE=${BUILD_PREFIX}/bin/${CONDA_TOOLCHAIN_HOST}-clang" EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DLLVM_LINK_EXECUTABLE=${BUILD_PREFIX}/bin/llvm-link" sed -ie "s;protoc-gen-grpc.*$;protoc-gen-grpc=${BUILD_PREFIX}/bin/grpc_cpp_plugin\";g" ../src/arrow/flight/CMakeLists.txt - sed -ie 's;"--with-jemalloc-prefix\=je_arrow_";"--with-jemalloc-prefix\=je_arrow_" "--with-lg-page\=14";g' ../cmake_modules/ThirdpartyToolchain.cmake + sed -ie 's;"--with-jemalloc-prefix\=je_arrow_";"--with-jemalloc-prefix\=je_arrow_" "--with-lg-page\=16";g' ../cmake_modules/ThirdpartyToolchain.cmake fi # disable -fno-plt, which causes problems with GCC on PPC @@ -65,9 +53,9 @@ if [[ "$target_platform" == "linux-ppc64le" ]]; then CXXFLAGS="$(echo $CXXFLAGS | sed 's/-fno-plt //g')" fi -# Limit number of threads used to avoid hardware oversubscription if [[ "${target_platform}" == "linux-aarch64" ]] || [[ "${target_platform}" == "linux-ppc64le" ]]; then - export CMAKE_BUILD_PARALLEL_LEVEL=3 + # Limit number of threads used to avoid hardware oversubscription + export CMAKE_BUILD_PARALLEL_LEVEL=3 fi # reusable variable for dependencies we cannot yet unvendor @@ -75,7 +63,7 @@ export READ_RECIPE_META_YAML_WHY_NOT=OFF # for available switches see # https://github.com/apache/arrow/blame/apache-arrow-12.0.0/cpp/cmake_modules/DefineOptions.cmake -# placeholder in ARROW_GDB_INSTALL_DIR must match what's used for replacement in activate.sh +# placeholder in ARROW_GDB_INSTALL_DIR must match _la_placeholder in activate.sh cmake -GNinja \ -DARROW_ACERO=ON \ -DARROW_BOOST_USE_SHARED=ON \ @@ -131,3 +119,6 @@ cmake -GNinja \ cmake --build . --target install --config Release popd + +# clean up between builds (and to save space) +rm -rf cpp/build diff --git a/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh b/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh index e0c3ae96bf55a..9c12321a1c115 100755 --- a/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh +++ b/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh @@ -38,9 +38,9 @@ if [[ "${target_platform}" == osx-* ]]; then CXXFLAGS="${CXXFLAGS} -D_LIBCPP_DISABLE_AVAILABILITY" fi -# Limit number of threads used to avoid hardware oversubscription if [[ "${target_platform}" == "linux-aarch64" ]] || [[ "${target_platform}" == "linux-ppc64le" ]]; then - export CMAKE_BUILD_PARALLEL_LEVEL=4 + # Limit number of threads used to avoid hardware oversubscription + export CMAKE_BUILD_PARALLEL_LEVEL=4 fi cd python diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml index 2a7a310a13069..0a34216250af4 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml +++ b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml @@ -21,8 +21,9 @@ build: # for cuda support, building with one version is enough to be compatible with # all later versions, since arrow is only using libcuda, and not libcudart. skip: true # [cuda_compiler_version not in ("None", cuda_compiler_version_min)] + # arrow promises API- & ABI-compatibility along SemVer, see #1096 run_exports: - - {{ pin_subpackage("libarrow", max_pin="x.x.x") }} + - {{ pin_subpackage("libarrow", max_pin="x") }} outputs: - name: apache-arrow-proc @@ -61,8 +62,15 @@ outputs: string: h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}_{{ build_ext }} run_exports: - {{ pin_subpackage("libarrow", max_pin="x.x.x") }} + ignore_run_exports_from: + - {{ compiler("cuda") }} # [cuda_compiler_version != "None"] + # arrow only uses headers, apparently + - gflags + # shared lib linked on unix, not on win + - glog # [win] ignore_run_exports: - - cudatoolkit + # we don't need all of brotli's run-exports + - libbrotlicommon track_features: {{ "[arrow-cuda]" if cuda_enabled else "" }} missing_dso_whitelist: - '*/libcuda.so.*' # [linux] @@ -94,7 +102,6 @@ outputs: - boost-cpp >=1.70 - brotli - bzip2 - - c-ares # not yet: https://github.com/conda-forge/cpp-opentelemetry-sdk-feedstock/issues/38 # - cpp-opentelemetry-sdk # - proto-opentelemetry-proto =={{ cpp_opentelemetry_sdk }} @@ -104,12 +111,6 @@ outputs: # arrow uses a customized jemalloc, see #944 # - jemalloc - libabseil - # since libgoogle-cloud is static on windows, see - # https://github.com/conda-forge/google-cloud-cpp-feedstock/pull/108, - # its dependencies leak into the build here - - libcrc32c # [win] - - libcurl # [win] - - libevent - libgrpc - libprotobuf - libutf8proc @@ -126,11 +127,20 @@ outputs: - xsimd - zlib - zstd + - __cuda >={{ cuda_compiler_version_min }} # [cuda_compiler_version != "None"] + # since libgoogle-cloud is static on windows, see + # https://github.com/conda-forge/google-cloud-cpp-feedstock/pull/108, + # its host deps (which aren't yet covered above) leak into the build here + - libcrc32c # [win] + - libcurl # [win] + # same for libgrpc (before 1.55.0, which is coupled with libprotobuf 4.23.x) + - c-ares # [win and libprotobuf == "3.21"] run_constrained: - apache-arrow-proc =*={{ build_ext }} - - cudatoolkit >={{ cuda_compiler_version_min }} # [cuda_compiler_version != "None"] # make sure we don't co-install with old version of old package name - arrow-cpp ={{ version }} + # old parquet lib output, now part of this feedstock + - parquet-cpp <0.0a0 test: commands: @@ -188,6 +198,8 @@ outputs: requirements: host: - {{ pin_subpackage('libarrow', exact=True) }} + # avoid wrappers for different builds colliding due to identical hashes + - libprotobuf run: - {{ pin_subpackage('libarrow', exact=True) }} test: @@ -200,8 +212,8 @@ outputs: version: {{ version }} build: string: py{{ CONDA_PY }}h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}_{{ build_ext }} - ignore_run_exports: - - cudatoolkit + ignore_run_exports_from: + - {{ compiler("cuda") }} # [cuda_compiler_version != "None"] track_features: {{ "[arrow-cuda]" if cuda_enabled else "" }} rpaths: - lib/ @@ -218,29 +230,28 @@ outputs: - {{ compiler("cuda") }} # [cuda_compiler_version != "None"] - python # [build_platform != target_platform] - cross-python_{{ target_platform }} # [build_platform != target_platform] - - cython # [build_platform != target_platform] + - cython <3 # [build_platform != target_platform] - numpy # [build_platform != target_platform] - cmake - ninja host: - - {{ pin_subpackage('libarrow', exact=True) }} + # we're building for two protobuf versions, cannot pin exactly + # - {{ pin_subpackage('libarrow', exact=True) }} + - libarrow ={{ version }}=*_{{ PKG_BUILDNUM }}_{{ build_ext }} - clangdev {{ llvm_version }} - llvmdev {{ llvm_version }} - - cython - - gflags # [unix] + - cython <3 - numpy - python - setuptools - setuptools_scm run: - - {{ pin_subpackage('libarrow', exact=True) }} + # - {{ pin_subpackage('libarrow', exact=True) }} + - libarrow ={{ version }}=*_{{ PKG_BUILDNUM }}_{{ build_ext }} - {{ pin_compatible('numpy') }} - # empty parquet-cpp metapackage, force old versions to be uninstalled - - parquet-cpp 1.5.1.* - python run_constrained: - apache-arrow-proc =*={{ build_ext }} - - cudatoolkit >={{ cuda_compiler_version_min }} # [cuda_compiler_version != "None"] test: files: @@ -292,8 +303,8 @@ outputs: version: {{ version }} build: string: py{{ CONDA_PY }}h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}_{{ build_ext }} - ignore_run_exports: - - cudatoolkit + ignore_run_exports_from: + - {{ compiler("cuda") }} # [cuda_compiler_version != "None"] track_features: {{ "[arrow-cuda]" if cuda_enabled else "" }} requirements: build: @@ -303,7 +314,7 @@ outputs: - {{ compiler("cuda") }} # [cuda_compiler_version != "None"] - python # [build_platform != target_platform] - cross-python_{{ target_platform }} # [build_platform != target_platform] - - cython # [build_platform != target_platform] + - cython <3 # [build_platform != target_platform] - numpy # [build_platform != target_platform] - cmake - ninja @@ -312,7 +323,7 @@ outputs: - {{ pin_subpackage('pyarrow', exact=True) }} - clangdev {{ llvm_version }} - llvmdev {{ llvm_version }} - - cython + - cython <3 - numpy - python - setuptools @@ -322,27 +333,36 @@ outputs: - python run_constrained: - apache-arrow-proc =*={{ build_ext }} - - cudatoolkit >={{ cuda_compiler_version_min }} # [cuda_compiler_version != "None"] + # crossbow CI: reduce to one python version, except on (unemulated) linux, where it's fast enough + {% if linux64 or py == 311 %} + # {% if not (aarch64 or ppc64le) or py in (310, 311) %} + # only run the full test suite for one python version when in emulation (each run takes ~45min); + # there's essentially zero divergence in behaviour across python versions anyway, and otherwise + # CUDA builds for aarch/ppc consistently run out of disk space on azure for some reason test: requires: + # vary protobuf version in test suite (historically, test failures only have a very + # weak dependency on python version, so we don't lose coverage by doing half & half) + - libprotobuf <4 # [py % 2 == 0] # test_cpp_extension_in_python requires a compiler - {{ compiler("cxx") }} # [linux] - - pytest + # temporary pin due to missing fixture + - pytest <7.4.0 - pytest-lazy-fixture - backports.zoneinfo # [py<39] - cffi - cloudpickle - - cython + - cython <3 - fastparquet - fsspec - hypothesis - pandas - scipy - # not all OSes/arches available in conda-forge - - pytorch * # [unix and not ppc64le] - # not yet rebuilt for libabseil 20230125 - # - tensorflow # [unix and x86_64 and py<311] + # these are generally (far) behind on migrating abseil/grpc/protobuf, + # and using them as test dependencies blocks the migrator unnecessarily + # - pytorch + # - tensorflow # we're not building java bindings # - jpype1 # doesn't get picked up correctly @@ -363,6 +383,8 @@ outputs: # skip tests that raise SIGINT and crash the test suite {% set tests_to_skip = tests_to_skip + " or (test_csv and test_cancellation)" %} # [linux] {% set tests_to_skip = tests_to_skip + " or (test_flight and test_interrupt)" %} # [linux] + # tests that may crash the agent due to out-of-bound memory writes or other risky stuff + {% set tests_to_skip = tests_to_skip + " or test_debug_memory_pool" %} # [aarch64 or ppc64le] # cannot pass -D_LIBCPP_DISABLE_AVAILABILITY to test suite for our older macos sdk {% set tests_to_skip = tests_to_skip + " or test_cpp_extension_in_python" %} # [osx] # skip tests that make invalid(-for-conda) assumptions about the compilers setup @@ -385,10 +407,12 @@ outputs: {% set tests_to_skip = tests_to_skip + " or (test_memory and test_env_var)" %} # [unix] # test is broken; header is in $PREFIX, not $SP_DIR {% set tests_to_skip = tests_to_skip + " or (test_misc and test_get_include)" %} # [unix] + # flaky tests that fail occasionally + {% set tests_to_skip = tests_to_skip + " or test_total_bytes_allocated " %} # [linux] + {% set tests_to_skip = tests_to_skip + " or test_feather_format " %} # [linux] # ^^^^^^^ TESTS THAT SHOULDN'T HAVE TO BE SKIPPED ^^^^^^^ - - # crossbow CI: reduce to one python version, except on (unemulated) linux, where it's fast enough - - pytest -v -rfEs -k "not ({{ tests_to_skip }})" # [linux64 or (py==310 and build_platform==target_platform)] + - pytest -rfEs -k "not ({{ tests_to_skip }})" + {% endif %} about: home: http://github.com/apache/arrow From e9e66054de3fa3a7b6468740ce678951c68de239 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Mon, 24 Jul 2023 18:40:44 +0200 Subject: [PATCH 7/9] sync .ci_support files --- ...> linux_64_cuda_compiler_version11.2.yaml} | 26 +++++++++---------- .../linux_64_cuda_compiler_versionNone.yaml | 16 ++++++------ ...nux_aarch64_cuda_compiler_version11.2.yaml | 14 +++++----- ...nux_aarch64_cuda_compiler_versionNone.yaml | 14 +++++----- ...nux_ppc64le_cuda_compiler_version11.2.yaml | 14 +++++----- ...nux_ppc64le_cuda_compiler_versionNone.yaml | 14 +++++----- .../conda-recipes/.ci_support/osx_64_.yaml | 14 +++++----- .../conda-recipes/.ci_support/osx_arm64_.yaml | 14 +++++----- ... => win_64_cuda_compiler_version11.2.yaml} | 16 +++++++----- .../win_64_cuda_compiler_versionNone.yaml | 14 +++++----- dev/tasks/tasks.yml | 4 +-- 11 files changed, 82 insertions(+), 78 deletions(-) rename dev/tasks/conda-recipes/.ci_support/{linux_64_cuda_compiler_version10.2.yaml => linux_64_cuda_compiler_version11.2.yaml} (85%) rename dev/tasks/conda-recipes/.ci_support/{win_64_cuda_compiler_version10.2.yaml => win_64_cuda_compiler_version11.2.yaml} (89%) diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version11.2.yaml similarity index 85% rename from dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2.yaml rename to dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version11.2.yaml index e135017bbaa61..1cdcec199e7ba 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version11.2.yaml @@ -1,17 +1,15 @@ aws_crt_cpp: -- 0.20.1 +- 0.20.3 aws_sdk_cpp: - 1.10.57 bzip2: - '1' -c_ares: -- '1' c_compiler: - gcc c_compiler_version: -- '7' +- '10' cdt_name: -- cos6 +- cos7 channel_sources: - conda-forge channel_targets: @@ -19,29 +17,29 @@ channel_targets: cuda_compiler: - nvcc cuda_compiler_version: -- '10.2' +- '11.2' cuda_compiler_version_min: -- '10.2' +- '11.2' cxx_compiler: - gxx cxx_compiler_version: -- '7' +- '10' docker_image: -- quay.io/condaforge/linux-anvil-cos7-cuda:10.2 +- quay.io/condaforge/linux-anvil-cuda:11.2 gflags: - '2.2' glog: - '0.6' google_cloud_cpp: -- 2.10.1 +- '2.12' libabseil: - '20230125' -libevent: -- 2.1.12 libgrpc: - '1.54' +- '1.56' libprotobuf: - '3.21' +- 4.23.3 lz4_c: - 1.9.3 numpy: @@ -52,7 +50,7 @@ numpy: openssl: - '3' orc: -- 1.8.3 +- 1.9.0 pin_run_as_build: python: min_pin: x.x @@ -79,6 +77,8 @@ zip_keys: - cuda_compiler_version - cdt_name - docker_image +- - libgrpc + - libprotobuf - - python - numpy zlib: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNone.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNone.yaml index 08590f860a5c3..5be5b58a73932 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNone.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNone.yaml @@ -1,11 +1,9 @@ aws_crt_cpp: -- 0.20.1 +- 0.20.3 aws_sdk_cpp: - 1.10.57 bzip2: - '1' -c_ares: -- '1' c_compiler: - gcc c_compiler_version: @@ -21,7 +19,7 @@ cuda_compiler: cuda_compiler_version: - None cuda_compiler_version_min: -- '10.2' +- '11.2' cxx_compiler: - gxx cxx_compiler_version: @@ -33,15 +31,15 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.10.1 +- '2.12' libabseil: - '20230125' -libevent: -- 2.1.12 libgrpc: - '1.54' +- '1.56' libprotobuf: - '3.21' +- 4.23.3 lz4_c: - 1.9.3 numpy: @@ -52,7 +50,7 @@ numpy: openssl: - '3' orc: -- 1.8.3 +- 1.9.0 pin_run_as_build: python: min_pin: x.x @@ -79,6 +77,8 @@ zip_keys: - cuda_compiler_version - cdt_name - docker_image +- - libgrpc + - libprotobuf - - python - numpy zlib: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_version11.2.yaml b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_version11.2.yaml index 3853add25f5f1..1677b03564c08 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_version11.2.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_version11.2.yaml @@ -1,13 +1,11 @@ BUILD: - aarch64-conda_cos7-linux-gnu aws_crt_cpp: -- 0.20.1 +- 0.20.3 aws_sdk_cpp: - 1.10.57 bzip2: - '1' -c_ares: -- '1' c_compiler: - gcc c_compiler_version: @@ -37,15 +35,15 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.10.1 +- '2.12' libabseil: - '20230125' -libevent: -- 2.1.12 libgrpc: - '1.54' +- '1.56' libprotobuf: - '3.21' +- 4.23.3 lz4_c: - 1.9.3 numpy: @@ -56,7 +54,7 @@ numpy: openssl: - '3' orc: -- 1.8.3 +- 1.9.0 pin_run_as_build: python: min_pin: x.x @@ -83,6 +81,8 @@ zip_keys: - cuda_compiler_version - cdt_name - docker_image +- - libgrpc + - libprotobuf - - python - numpy zlib: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNone.yaml b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNone.yaml index ef792635029d0..88fdf1254e661 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNone.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNone.yaml @@ -1,13 +1,11 @@ BUILD: - aarch64-conda_cos7-linux-gnu aws_crt_cpp: -- 0.20.1 +- 0.20.3 aws_sdk_cpp: - 1.10.57 bzip2: - '1' -c_ares: -- '1' c_compiler: - gcc c_compiler_version: @@ -37,15 +35,15 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.10.1 +- '2.12' libabseil: - '20230125' -libevent: -- 2.1.12 libgrpc: - '1.54' +- '1.56' libprotobuf: - '3.21' +- 4.23.3 lz4_c: - 1.9.3 numpy: @@ -56,7 +54,7 @@ numpy: openssl: - '3' orc: -- 1.8.3 +- 1.9.0 pin_run_as_build: python: min_pin: x.x @@ -83,6 +81,8 @@ zip_keys: - cuda_compiler_version - cdt_name - docker_image +- - libgrpc + - libprotobuf - - python - numpy zlib: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_version11.2.yaml b/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_version11.2.yaml index eef7acde228f5..3585db7b99baa 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_version11.2.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_version11.2.yaml @@ -1,11 +1,9 @@ aws_crt_cpp: -- 0.20.1 +- 0.20.3 aws_sdk_cpp: - 1.10.57 bzip2: - '1' -c_ares: -- '1' c_compiler: - gcc c_compiler_version: @@ -33,15 +31,15 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.10.1 +- '2.12' libabseil: - '20230125' -libevent: -- 2.1.12 libgrpc: - '1.54' +- '1.56' libprotobuf: - '3.21' +- 4.23.3 lz4_c: - 1.9.3 numpy: @@ -52,7 +50,7 @@ numpy: openssl: - '3' orc: -- 1.8.3 +- 1.9.0 pin_run_as_build: python: min_pin: x.x @@ -79,6 +77,8 @@ zip_keys: - cuda_compiler_version - cdt_name - docker_image +- - libgrpc + - libprotobuf - - python - numpy zlib: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNone.yaml b/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNone.yaml index b9bdd01b358cf..c13a522254286 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNone.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNone.yaml @@ -1,11 +1,9 @@ aws_crt_cpp: -- 0.20.1 +- 0.20.3 aws_sdk_cpp: - 1.10.57 bzip2: - '1' -c_ares: -- '1' c_compiler: - gcc c_compiler_version: @@ -33,15 +31,15 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.10.1 +- '2.12' libabseil: - '20230125' -libevent: -- 2.1.12 libgrpc: - '1.54' +- '1.56' libprotobuf: - '3.21' +- 4.23.3 lz4_c: - 1.9.3 numpy: @@ -52,7 +50,7 @@ numpy: openssl: - '3' orc: -- 1.8.3 +- 1.9.0 pin_run_as_build: python: min_pin: x.x @@ -79,6 +77,8 @@ zip_keys: - cuda_compiler_version - cdt_name - docker_image +- - libgrpc + - libprotobuf - - python - numpy zlib: diff --git a/dev/tasks/conda-recipes/.ci_support/osx_64_.yaml b/dev/tasks/conda-recipes/.ci_support/osx_64_.yaml index 0774679775a61..dd4a230760ef2 100644 --- a/dev/tasks/conda-recipes/.ci_support/osx_64_.yaml +++ b/dev/tasks/conda-recipes/.ci_support/osx_64_.yaml @@ -1,13 +1,11 @@ MACOSX_DEPLOYMENT_TARGET: - '10.9' aws_crt_cpp: -- 0.20.1 +- 0.20.3 aws_sdk_cpp: - 1.10.57 bzip2: - '1' -c_ares: -- '1' c_compiler: - clang c_compiler_version: @@ -27,15 +25,15 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.10.1 +- '2.12' libabseil: - '20230125' -libevent: -- 2.1.12 libgrpc: - '1.54' +- '1.56' libprotobuf: - '3.21' +- 4.23.3 lz4_c: - 1.9.3 macos_machine: @@ -48,7 +46,7 @@ numpy: openssl: - '3' orc: -- 1.8.3 +- 1.9.0 pin_run_as_build: python: min_pin: x.x @@ -69,6 +67,8 @@ thrift_cpp: zip_keys: - - c_compiler_version - cxx_compiler_version +- - libgrpc + - libprotobuf - - python - numpy zlib: diff --git a/dev/tasks/conda-recipes/.ci_support/osx_arm64_.yaml b/dev/tasks/conda-recipes/.ci_support/osx_arm64_.yaml index bd834a4b66cb0..6a6713a54fe86 100644 --- a/dev/tasks/conda-recipes/.ci_support/osx_arm64_.yaml +++ b/dev/tasks/conda-recipes/.ci_support/osx_arm64_.yaml @@ -1,13 +1,11 @@ MACOSX_DEPLOYMENT_TARGET: - '11.0' aws_crt_cpp: -- 0.20.1 +- 0.20.3 aws_sdk_cpp: - 1.10.57 bzip2: - '1' -c_ares: -- '1' c_compiler: - clang c_compiler_version: @@ -27,15 +25,15 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.10.1 +- '2.12' libabseil: - '20230125' -libevent: -- 2.1.12 libgrpc: - '1.54' +- '1.56' libprotobuf: - '3.21' +- 4.23.3 lz4_c: - 1.9.3 macos_machine: @@ -48,7 +46,7 @@ numpy: openssl: - '3' orc: -- 1.8.3 +- 1.9.0 pin_run_as_build: python: min_pin: x.x @@ -69,6 +67,8 @@ thrift_cpp: zip_keys: - - c_compiler_version - cxx_compiler_version +- - libgrpc + - libprotobuf - - python - numpy zlib: diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version11.2.yaml similarity index 89% rename from dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2.yaml rename to dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version11.2.yaml index dc1c64446b4da..f75d92e276d9e 100644 --- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version11.2.yaml @@ -1,5 +1,5 @@ aws_crt_cpp: -- 0.20.1 +- 0.20.3 aws_sdk_cpp: - 1.10.57 bzip2: @@ -15,9 +15,9 @@ channel_targets: cuda_compiler: - nvcc cuda_compiler_version: -- '10.2' +- '11.2' cuda_compiler_version_min: -- '10.2' +- '11.2' cxx_compiler: - vs2019 gflags: @@ -25,19 +25,19 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.10.1 +- '2.12' libabseil: - '20230125' libcrc32c: - '1.1' libcurl: - '8' -libevent: -- 2.1.12 libgrpc: - '1.54' +- '1.56' libprotobuf: - '3.21' +- 4.23.3 lz4_c: - 1.9.3 numpy: @@ -48,7 +48,7 @@ numpy: openssl: - '3' orc: -- 1.8.3 +- 1.9.0 pin_run_as_build: python: min_pin: x.x @@ -69,6 +69,8 @@ thrift_cpp: zip_keys: - - cuda_compiler - cuda_compiler_version +- - libgrpc + - libprotobuf - - python - numpy zlib: diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNone.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNone.yaml index e9dee6a0eac0d..6d8fb15b15a2a 100644 --- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNone.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNone.yaml @@ -1,5 +1,5 @@ aws_crt_cpp: -- 0.20.1 +- 0.20.3 aws_sdk_cpp: - 1.10.57 bzip2: @@ -17,7 +17,7 @@ cuda_compiler: cuda_compiler_version: - None cuda_compiler_version_min: -- '10.2' +- '11.2' cxx_compiler: - vs2019 gflags: @@ -25,19 +25,19 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.10.1 +- '2.12' libabseil: - '20230125' libcrc32c: - '1.1' libcurl: - '8' -libevent: -- 2.1.12 libgrpc: - '1.54' +- '1.56' libprotobuf: - '3.21' +- 4.23.3 lz4_c: - 1.9.3 numpy: @@ -48,7 +48,7 @@ numpy: openssl: - '3' orc: -- 1.8.3 +- 1.9.0 pin_run_as_build: python: min_pin: x.x @@ -69,6 +69,8 @@ thrift_cpp: zip_keys: - - cuda_compiler - cuda_compiler_version +- - libgrpc + - libprotobuf - - python - numpy zlib: diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 1e9ce8e9300ad..284946a8cd498 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -282,7 +282,7 @@ tasks: ci: azure template: conda-recipes/azure.linux.yml params: - config: linux_64_cuda_compiler_version10.2 + config: linux_64_cuda_compiler_version11.2 artifacts: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cuda.conda - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cuda.conda @@ -460,7 +460,7 @@ tasks: ci: azure template: conda-recipes/azure.win.yml params: - config: win_64_cuda_compiler_versionNone + config: win_64_cuda_compiler_version11.2 artifacts: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cuda.conda - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cuda.conda From 10e3ceb954aa8f2ab78b2c1fe49d3cf0b7f1f188 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Mon, 24 Jul 2023 22:50:25 +0200 Subject: [PATCH 8/9] free up disk space on linux before build --- dev/tasks/conda-recipes/azure.linux.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/dev/tasks/conda-recipes/azure.linux.yml b/dev/tasks/conda-recipes/azure.linux.yml index b9a54647cc525..279ffb48ccd60 100755 --- a/dev/tasks/conda-recipes/azure.linux.yml +++ b/dev/tasks/conda-recipes/azure.linux.yml @@ -13,6 +13,29 @@ jobs: UPLOAD_PACKAGES: False steps: + - script: | + sudo mkdir -p /opt/empty_dir || true + for d in \ + /opt/ghc \ + /opt/hostedtoolcache \ + /usr/lib/jvm \ + /usr/local/.ghcup \ + /usr/local/lib/android \ + /usr/local/share/powershell \ + /usr/share/dotnet \ + /usr/share/swift \ + ; do + sudo rsync --stats -a --delete /opt/empty_dir/ $d || true + done + sudo apt-get purge -y -f firefox \ + google-chrome-stable \ + microsoft-edge-stable + sudo apt-get autoremove -y >& /dev/null + sudo apt-get autoclean -y >& /dev/null + sudo docker image prune --all --force + df -h + displayName: Manage disk space + # configure qemu binfmt-misc running. This allows us to run docker containers # embedded qemu-static - script: | From 17fc588c431640589d538ffbc7ee119df83153a3 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Mon, 24 Jul 2023 22:57:05 +0200 Subject: [PATCH 9/9] fix test for expected acero header got removed in 0bb2d83a3bba92a28495a67de74d94598e456625 --- dev/tasks/conda-recipes/arrow-cpp/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml index 0a34216250af4..e61034c3075b3 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml +++ b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml @@ -145,7 +145,7 @@ outputs: test: commands: {% set headers = [ - "arrow/api.h", "arrow/acero/groupby.h", "arrow/flight/types.h", + "arrow/api.h", "arrow/acero/api.h", "arrow/flight/types.h", "arrow/flight/sql/api.h", "gandiva/engine.h", "parquet/api/reader.h" ] %} {% for each_header in headers %}