Skip to content

Commit

Permalink
apacheGH-39621: [CI][Packaging] Update vcpkg to 2023.11.20 release (a…
Browse files Browse the repository at this point in the history
…pache#39622)

### Rationale for this change
Old version of vcpkg is blocking apache#39352

### What changes are included in this PR?
- Upgrade vcpkg 
- Fix ports patches
- Upgrade visual studio used in windows wheel builds. VS2019 is now required for the vcpkg `abseil` build. 
- Move `VCPKG_FORCE_SYSTEM_BINARIES` to be set before vcpkg install to fix vcpkg install on linux ARM. 
- Fix for LLVM 17 which requires that an executable exports "llvm_orc_registerEHFrameSectionWrapper()" and "llvm_orc_unregisterEHFrameSectionWrapper()". This effects the java builds that depend on llvm from vcpkg for gandiva. 
- Update image used for python wheel builds on windows to 2024-02-05

### Are these changes tested?
Does not change any behaviour so should be covered by existing tests

### Are there any user-facing changes?
There shouldn't be
* Closes: apache#39621

Lead-authored-by: Thomas Newton <[email protected]>
Co-authored-by: Sutou Kouhei <[email protected]>
Co-authored-by: Raúl Cumplido <[email protected]>
Signed-off-by: Sutou Kouhei <[email protected]>
  • Loading branch information
3 people authored and thisisnic committed Mar 8, 2024
1 parent 1e3e06a commit 8b66b89
Show file tree
Hide file tree
Showing 10 changed files with 57 additions and 41 deletions.
6 changes: 3 additions & 3 deletions .env
Original file line number Diff line number Diff line change
Expand Up @@ -92,13 +92,13 @@ DEVTOOLSET_VERSION=
# Used through docker-compose.yml and serves as the default version for the
# ci/scripts/install_vcpkg.sh script. Prefer to use short SHAs to keep the
# docker tags more readable.
VCPKG="501db0f17ef6df184fcdbfbe0f87cde2313b6ab1" # 2023.04.15 Release
VCPKG="a42af01b72c28a8e1d7b48107b33e4f286a55ef6" # 2023.11.20 Release

# This must be updated when we update
# ci/docker/python-wheel-windows-vs2017.dockerfile.
# ci/docker/python-wheel-windows-vs2019.dockerfile.
# This is a workaround for our CI problem that "archery docker build" doesn't
# use pulled built images in dev/tasks/python-wheels/github.windows.yml.
PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2023-08-02
PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2024-02-05

# Use conanio/${CONAN} for "docker-compose run --rm conan". See
# https://github.com/conan-io/conan-docker-tools#readme for available
Expand Down
7 changes: 4 additions & 3 deletions ci/docker/python-wheel-manylinux.dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -62,15 +62,16 @@ COPY ci/vcpkg/*.patch \
COPY ci/scripts/install_vcpkg.sh \
arrow/ci/scripts/
ENV VCPKG_ROOT=/opt/vcpkg
RUN arrow/ci/scripts/install_vcpkg.sh ${VCPKG_ROOT} ${vcpkg}
ENV PATH="${PATH}:${VCPKG_ROOT}"

ARG build_type=release
ENV CMAKE_BUILD_TYPE=${build_type} \
VCPKG_FORCE_SYSTEM_BINARIES=1 \
VCPKG_OVERLAY_TRIPLETS=/arrow/ci/vcpkg \
VCPKG_DEFAULT_TRIPLET=${arch_short}-linux-static-${build_type} \
VCPKG_FEATURE_FLAGS="manifests"

RUN arrow/ci/scripts/install_vcpkg.sh ${VCPKG_ROOT} ${vcpkg}
ENV PATH="${PATH}:${VCPKG_ROOT}"

COPY ci/vcpkg/vcpkg.json arrow/ci/vcpkg/
# cannot use the S3 feature here because while aws-sdk-cpp=1.9.160 contains
# ssl related fixes as well as we can patch the vcpkg portfile to support
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@
# when you update this file.

# based on mcr.microsoft.com/windows/servercore:ltsc2019
# contains choco and vs2017 preinstalled
FROM abrarov/msvc-2017:2.11.0
# contains choco and vs2019 preinstalled
FROM abrarov/msvc-2019:2.11.0

# Add unix tools to path
RUN setx path "%path%;C:\Program Files\Git\usr\bin"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@
# when you update this file.

# based on mcr.microsoft.com/windows/servercore:ltsc2019
# contains choco and vs2017 preinstalled
FROM abrarov/msvc-2017:2.11.0
# contains choco and vs2019 preinstalled
FROM abrarov/msvc-2019:2.11.0

# Install CMake and Ninja
ARG cmake=3.21.4
Expand Down
8 changes: 5 additions & 3 deletions ci/scripts/python_wheel_windows_build.bat
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

echo "Building windows wheel..."

call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvars64.bat"
call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvars64.bat"

echo "=== (%PYTHON_VERSION%) Clear output directories and leftovers ==="
del /s /q C:\arrow-build
Expand Down Expand Up @@ -50,7 +50,8 @@ set ARROW_WITH_SNAPPY=ON
set ARROW_WITH_ZLIB=ON
set ARROW_WITH_ZSTD=ON
set CMAKE_UNITY_BUILD=ON
set CMAKE_GENERATOR=Visual Studio 15 2017 Win64
set CMAKE_GENERATOR=Visual Studio 16 2019
set CMAKE_PLATFORM=x64
set VCPKG_ROOT=C:\vcpkg
set VCPKG_FEATURE_FLAGS=-manifests
set VCGPK_TARGET_TRIPLET=amd64-windows-static-md-%CMAKE_BUILD_TYPE%
Expand Down Expand Up @@ -96,6 +97,7 @@ cmake ^
-DVCPKG_MANIFEST_MODE=OFF ^
-DVCPKG_TARGET_TRIPLET=%VCGPK_TARGET_TRIPLET% ^
-G "%CMAKE_GENERATOR%" ^
-A "%CMAKE_PLATFORM%" ^
C:\arrow\cpp || exit /B 1
cmake --build . --config %CMAKE_BUILD_TYPE% --target install || exit /B 1
popd
Expand All @@ -121,6 +123,6 @@ set CMAKE_PREFIX_PATH=C:\arrow-dist

pushd C:\arrow\python
@REM bundle the msvc runtime
cp "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Redist\MSVC\14.16.27012\x64\Microsoft.VC141.CRT\msvcp140.dll" pyarrow\
cp "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Redist\MSVC\14.28.29325\x64\Microsoft.VC142.CRT\msvcp140.dll" pyarrow\
python setup.py bdist_wheel || exit /B 1
popd
27 changes: 14 additions & 13 deletions ci/vcpkg/ports.patch
Original file line number Diff line number Diff line change
@@ -1,43 +1,44 @@
diff --git a/ports/curl/portfile.cmake b/ports/curl/portfile.cmake
index 5a14562..924b1b7 100644
index bdc544e9e..53f6bbc3b 100644
--- a/ports/curl/portfile.cmake
+++ b/ports/curl/portfile.cmake
@@ -87,8 +87,11 @@ vcpkg_cmake_configure(
@@ -74,9 +74,12 @@ vcpkg_cmake_configure(
-DENABLE_MANUAL=OFF
-DCURL_CA_FALLBACK=ON
-DCURL_USE_LIBPSL=OFF
+ -DCURL_CA_PATH=none
+ -DCURL_CA_BUNDLE=none
-DCMAKE_DISABLE_FIND_PACKAGE_Perl=ON
OPTIONS_DEBUG
-DENABLE_DEBUG=ON
+ ${EXTRA_ARGS_DEBUG}
)
vcpkg_cmake_install()
vcpkg_copy_pdbs()
diff --git a/ports/snappy/portfile.cmake b/ports/snappy/portfile.cmake
index 8f3f3f9..745b0fb 100644
index 0c7098082..c603c3653 100644
--- a/ports/snappy/portfile.cmake
+++ b/ports/snappy/portfile.cmake
@@ -9,6 +9,7 @@ vcpkg_from_github(
HEAD_REF master
@@ -10,6 +10,7 @@ vcpkg_from_github(
PATCHES
fix_clang-cl_build.patch
no-werror.patch
+ "snappy-disable-bmi.patch"
)

vcpkg_cmake_configure(
diff --git a/ports/snappy/snappy-disable-bmi.patch b/ports/snappy/snappy-disable-bmi.patch
new file mode 100644
index 0000000..a57ce0c
index 000000000..e839c93a4
--- /dev/null
+++ b/ports/snappy/snappy-disable-bmi.patch
@@ -0,0 +1,19 @@
+diff --git a/snappy.cc b/snappy.cc
+index 79dc0e8..f3153ea 100644
+index d414718..7b49d2a 100644
+--- a/snappy.cc
++++ b/snappy.cc
+@@ -965,14 +965,10 @@ static inline void Report(const char *algorithm, size_t compressed_size,
+ static inline uint32_t ExtractLowBytes(uint32_t v, int n) {
+@@ -1014,14 +1014,10 @@ static inline void Report(const char *algorithm, size_t compressed_size,
+ static inline uint32_t ExtractLowBytes(const uint32_t& v, int n) {
+ assert(n >= 0);
+ assert(n <= 4);
+-#if SNAPPY_HAVE_BMI2
Expand All @@ -52,13 +53,13 @@ index 0000000..a57ce0c
+
+ static inline bool LeftShiftOverflows(uint8_t value, uint32_t shift) {
diff --git a/ports/llvm/portfile.cmake b/ports/llvm/portfile.cmake
index 4d7e26a..1f054a2 100644
index bf9397b66..c3112b673 100644
--- a/ports/llvm/portfile.cmake
+++ b/ports/llvm/portfile.cmake
@@ -274,6 +274,8 @@ vcpkg_cmake_configure(
@@ -293,6 +293,8 @@ vcpkg_cmake_configure(
${FEATURE_OPTIONS}
MAYBE_UNUSED_VARIABLES
COMPILER_RT_ENABLE_IOS
OPENMP_TOOLS_INSTALL_DIR
MLIR_TOOLS_INSTALL_DIR
+ BOLT_TOOLS_INSTALL_DIR
+ LIBOMP_INSTALL_ALIASES
)
Expand Down
5 changes: 4 additions & 1 deletion ci/vcpkg/vcpkg.json
Original file line number Diff line number Diff line change
Expand Up @@ -81,8 +81,11 @@
"default-features": false,
"features": [
"clang",
"default-options",
"default-targets",
"enable-bindings",
"enable-terminfo",
"enable-zlib",
"enable-zstd",
"enable-rtti",
"lld",
"tools"
Expand Down
9 changes: 9 additions & 0 deletions cpp/src/gandiva/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,15 @@ function(ADD_GANDIVA_TEST REL_TEST_NAME)

set(TEST_NAME gandiva-${REL_TEST_NAME})
string(REPLACE "_" "-" TEST_NAME ${TEST_NAME})

if(ARG_USE_STATIC_LINKING OR ARROW_TEST_LINKAGE STREQUAL "static")
# LLVM 17 or later requires that an executable exports
# "llvm_orc_registerEHFrameSectionWrapper()" and
# "llvm_orc_unregisterEHFrameSectionWrapper()". We need to do
# nothing when we use libLLVM.so. But we need to export symbols
# explicitly when we use libLLVM*.a.
set_target_properties(${TEST_NAME} PROPERTIES ENABLE_EXPORTS TRUE)
endif()
endfunction()

add_gandiva_test(internals-test
Expand Down
12 changes: 6 additions & 6 deletions dev/tasks/python-wheels/github.windows.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ jobs:
# this is a private repository at the moment (mostly because of licensing
# consideration of windows images with visual studio), but anyone can
# recreate the image by manually building it via:
# `archery build python-wheel-windows-vs2017`
# `archery build python-wheel-windows-vs2019`
# note that we don't run docker build since there wouldn't be a cache hit
# and rebuilding the dependencies takes a fair amount of time
REPO: ghcr.io/ursacomputing/arrow
Expand All @@ -46,17 +46,17 @@ jobs:
run: |
cd arrow
@rem We want to use only
@rem archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-windows-vs2017
@rem archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-windows-vs2019
@rem but it doesn't use pulled caches.
@rem It always build an image from scratch.
@rem We can remove this workaround once we find a way to use
@rem pulled caches when build an image.
echo on
archery docker pull --no-ignore-pull-failures python-wheel-windows-vs2017
archery docker pull --no-ignore-pull-failures python-wheel-windows-vs2019
if errorlevel 1 (
archery docker build --no-pull python-wheel-windows-vs2017 || exit /B 1
archery docker build --no-pull python-wheel-windows-vs2019 || exit /B 1
)
archery docker run --no-build -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-windows-vs2017
archery docker run --no-build -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-windows-vs2019
- uses: actions/upload-artifact@v3
with:
Expand All @@ -77,5 +77,5 @@ jobs:
shell: cmd
run: |
cd arrow
archery docker push python-wheel-windows-vs2017
archery docker push python-wheel-windows-vs2019
{% endif %}
16 changes: 8 additions & 8 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,7 @@ x-hierarchy:
- python-wheel-manylinux-2-28
- python-wheel-manylinux-test-imports
- python-wheel-manylinux-test-unittests
- python-wheel-windows-vs2017
- python-wheel-windows-vs2019
- python-wheel-windows-test

volumes:
Expand Down Expand Up @@ -1098,19 +1098,19 @@ services:
CHECK_UNITTESTS: "ON"
command: /arrow/ci/scripts/python_wheel_unix_test.sh /arrow

python-wheel-windows-vs2017:
image: ${REPO}:python-${PYTHON}-wheel-windows-vs2017-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
python-wheel-windows-vs2019:
image: ${REPO}:python-${PYTHON}-wheel-windows-vs2019-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
build:
args:
vcpkg: ${VCPKG}
python: ${PYTHON}
context: .
dockerfile: ci/docker/python-wheel-windows-vs2017.dockerfile
dockerfile: ci/docker/python-wheel-windows-vs2019.dockerfile
# This should make the pushed images reusable, but the image gets rebuilt.
# Uncomment if no local cache is available.
# cache_from:
# - abrarov/msvc-2017:2.11.0
# - ${REPO}:python-${PYTHON}-wheel-windows-vs2017-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
# - abrarov/msvc-2019:2.11.0
# - ${REPO}:python-${PYTHON}-wheel-windows-vs2019-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
volumes:
- "${DOCKER_VOLUME_PREFIX}python-wheel-windows-clcache:C:/clcache"
- type: bind
Expand All @@ -1119,12 +1119,12 @@ services:
command: arrow\\ci\\scripts\\python_wheel_windows_build.bat

python-wheel-windows-test:
image: ${REPO}:python-${PYTHON}-wheel-windows-test-vs2017-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
image: ${REPO}:python-${PYTHON}-wheel-windows-test-vs2019-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
build:
args:
python: ${PYTHON}
context: .
dockerfile: ci/docker/python-wheel-windows-test-vs2017.dockerfile
dockerfile: ci/docker/python-wheel-windows-test-vs2019.dockerfile
volumes:
- "${DOCKER_VOLUME_PREFIX}python-wheel-windows-clcache:C:/clcache"
- type: bind
Expand Down

0 comments on commit 8b66b89

Please sign in to comment.