Skip to content

Commit

Permalink
apacheGH-43951: [CI][Python] Use GitHub Packages for vcpkg cache
Browse files Browse the repository at this point in the history
  • Loading branch information
kou committed Nov 5, 2024
1 parent 4274db8 commit c931193
Show file tree
Hide file tree
Showing 4 changed files with 46 additions and 13 deletions.
13 changes: 9 additions & 4 deletions ci/docker/python-wheel-manylinux.dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,8 @@ COPY ci/scripts/install_ccache.sh arrow/ci/scripts/
RUN /arrow/ci/scripts/install_ccache.sh ${ccache} /usr/local

# Install vcpkg
ARG GITHUB_ACTOR
ARG GITHUB_TOKEN
ARG vcpkg
COPY ci/vcpkg/*.patch \
ci/vcpkg/*linux*.cmake \
Expand All @@ -75,10 +77,13 @@ COPY ci/scripts/install_vcpkg.sh \
ENV VCPKG_ROOT=/opt/vcpkg
ARG build_type=release
ENV CMAKE_BUILD_TYPE=${build_type} \
VCPKG_FORCE_SYSTEM_BINARIES=1 \
VCPKG_OVERLAY_TRIPLETS=/arrow/ci/vcpkg \
GITHUB_ACTOR="${GITHUB_ACTOR}" \
GITHUB_TOKEN="${GITHUB_TOKEN}" \
VCPKG_BINARY_SOURCES="clear;nuget,https://nuget.pkg.github.com/${GITHUB_ACTOR}/index.json,readwrite" \
VCPKG_DEFAULT_TRIPLET=${arch_short}-linux-static-${build_type} \
VCPKG_FEATURE_FLAGS="manifests"
VCPKG_FEATURE_FLAGS="manifests" \
VCPKG_FORCE_SYSTEM_BINARIES=1 \
VCPKG_OVERLAY_TRIPLETS=/arrow/ci/vcpkg

RUN arrow/ci/scripts/install_vcpkg.sh ${VCPKG_ROOT} ${vcpkg}
ENV PATH="${PATH}:${VCPKG_ROOT}"
Expand All @@ -93,7 +98,7 @@ RUN vcpkg install \
--clean-after-build \
--x-install-root=${VCPKG_ROOT}/installed \
--x-manifest-root=/arrow/ci/vcpkg \
--x-feature=azure \
--x-feature=azure \
--x-feature=flight \
--x-feature=gcs \
--x-feature=json \
Expand Down
19 changes: 17 additions & 2 deletions ci/scripts/install_vcpkg.sh
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
# specific language governing permissions and limitations
# under the License.

set -e
set -eu

if [ "$#" -lt 1 ]; then
echo "Usage: $0 ``<target-directory> [<vcpkg-version> [<vcpkg-ports-patch>]]"
Expand All @@ -42,7 +42,7 @@ pushd ${vcpkg_destination}

git checkout "${vcpkg_version}"

if [[ "$OSTYPE" == "msys" ]]; then
if [[ "${OSTYPE:-}" == "msys" ]]; then
./bootstrap-vcpkg.bat -disableMetrics
else
./bootstrap-vcpkg.sh -disableMetrics
Expand All @@ -53,4 +53,19 @@ if [ -f "${vcpkg_ports_patch}" ]; then
echo "Patch successfully applied to the VCPKG port files!"
fi

if [ -n "${GITHUB_TOKEN:-}" ] && [ -n "${GITHUB_ACTOR:-}" ]; then
PATH="${PATH}:${VCPKG_ROOT}"
nuget_url="https://nuget.pkg.github.com/${GITHUB_ACTOR}/index.json"
"$(vcpkg fetch nuget)" \
sources add \
-Source "${nuget_url}" \
-StorePasswordInClearText \
-Name GitHubPackages \
-UserName "${GITHUB_ACTOR}" \
-Password "${GITHUB_TOKEN}"
"$(vcpkg fetch nuget)" \
setapikey "${GITHUB_TOKEN}" \
-Source "${nuget_url}"
fi

popd
10 changes: 9 additions & 1 deletion dev/tasks/python-wheels/github.linux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@

{{ macros.github_header() }}

permissions:
packages: write

jobs:
build:
name: "Build wheel for manylinux {{ manylinux_version }}"
Expand Down Expand Up @@ -49,7 +52,12 @@ jobs:

- name: Build wheel
shell: bash
run: archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-manylinux-{{ manylinux_version }}
run: |
archery docker run \
-e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} \
python-wheel-manylinux-{{ manylinux_version }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

- uses: actions/upload-artifact@v4
with:
Expand Down
17 changes: 11 additions & 6 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -53,26 +53,30 @@
#
# See more in cpp/build-support/run-test.sh::print_coredumps

x-common: &common
GITHUB_ACTIONS:

x-ccache: &ccache
CCACHE_COMPILERCHECK: content
CCACHE_COMPRESS: 1
CCACHE_COMPRESSLEVEL: 6
CCACHE_MAXSIZE: 1G
CCACHE_DIR: /ccache

x-common: &common
GITHUB_ACTIONS:

x-cpp: &cpp
ARROW_RUNTIME_SIMD_LEVEL:
ARROW_SIMD_LEVEL:

x-sccache: &sccache
AWS_ACCESS_KEY_ID:
AWS_SECRET_ACCESS_KEY:
SCCACHE_BUCKET:
SCCACHE_REGION:
SCCACHE_S3_KEY_PREFIX: ${SCCACHE_S3_KEY_PREFIX:-sccache}

x-cpp: &cpp
ARROW_RUNTIME_SIMD_LEVEL:
ARROW_SIMD_LEVEL:
x-vcpkg: &vcpkg
GITHUB_ACTOR:
GITHUB_TOKEN:

# CPU/memory limit presets to pass to Docker.
#
Expand Down Expand Up @@ -1144,6 +1148,7 @@ services:
image: ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-2-28-vcpkg-${VCPKG}
build:
args:
<<: [*vcpkg]
arch: ${ARCH}
arch_short: ${ARCH_SHORT}
base: quay.io/pypa/manylinux_2_28_${ARCH_ALIAS}:2024-08-03-32dfa47
Expand Down

0 comments on commit c931193

Please sign in to comment.