From 4ef2e5f730d71833c675cc5c8b9120f6e315b8de Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 1 Feb 2024 14:53:35 +0100 Subject: [PATCH] GH-39555: [Packaging][Python] Enable building pyarrow against numpy 2.0 (#39557) ### Rationale for this change Ensure we can build pyarrow against numpy 2.0 nightly (update pyproject.toml to allow this), and test this by building our nightly wheels with numpy nightly. This also ensures that other projects that use our nightly wheels to test together with numpy nightly can do that (numpy 2.0 changes the ABI, so to run with numpy 2.0, your package needs to be built with numpy 2.x; currently pyarrow installed with our nightly wheel will fail to import when also numpy nightly is installed). See the parent issue https://github.com/apache/arrow/issues/39532 for details, and https://numpy.org/devdocs/dev/depending_on_numpy.html#numpy-2-0-specific-advice for a direct link to the NumPy guidelines on updating build dependencies for NumPy 2.0. * Closes: #39555 Lead-authored-by: Joris Van den Bossche Co-authored-by: Antoine Pitrou Signed-off-by: Joris Van den Bossche --- ci/docker/python-wheel-manylinux.dockerfile | 5 +++-- ci/docker/python-wheel-windows-vs2017.dockerfile | 3 ++- ci/scripts/python_wheel_macos_build.sh | 5 ++++- python/pyproject.toml | 7 ++++++- python/requirements-build.txt | 3 ++- python/requirements-wheel-build.txt | 3 ++- python/setup.py | 2 +- 7 files changed, 20 insertions(+), 8 deletions(-) diff --git a/ci/docker/python-wheel-manylinux.dockerfile b/ci/docker/python-wheel-manylinux.dockerfile index 0a50d450c225a..a07c727ac76fa 100644 --- a/ci/docker/python-wheel-manylinux.dockerfile +++ b/ci/docker/python-wheel-manylinux.dockerfile @@ -28,7 +28,7 @@ ENV MANYLINUX_VERSION=${manylinux} RUN yum install -y dnf # Install basic dependencies -RUN dnf install -y git flex curl autoconf zip perl-IPC-Cmd wget kernel-headers +RUN dnf install -y git flex curl autoconf zip perl-IPC-Cmd wget # A system Python is required for ninja and vcpkg in this Dockerfile. # On manylinux2014 base images, system Python is 2.7.5, while @@ -97,4 +97,5 @@ SHELL ["/bin/bash", "-i", "-c"] ENTRYPOINT ["/bin/bash", "-i", "-c"] COPY python/requirements-wheel-build.txt /arrow/python/ -RUN pip install -r /arrow/python/requirements-wheel-build.txt +# TODO(GH-39848) Remove the `--pre --extra-index-url` for numpy nightly again before the 16.0 release +RUN pip install -r /arrow/python/requirements-wheel-build.txt --pre --extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" diff --git a/ci/docker/python-wheel-windows-vs2017.dockerfile b/ci/docker/python-wheel-windows-vs2017.dockerfile index faf07800c956a..067105b3a7995 100644 --- a/ci/docker/python-wheel-windows-vs2017.dockerfile +++ b/ci/docker/python-wheel-windows-vs2017.dockerfile @@ -88,7 +88,8 @@ RUN choco install -r -y --no-progress python --version=%PYTHON_VERSION% RUN python -m pip install -U pip setuptools COPY python/requirements-wheel-build.txt arrow/python/ -RUN python -m pip install -r arrow/python/requirements-wheel-build.txt +# TODO(GH-39848) Remove the `--pre --extra-index-url` for numpy nightly again before the 16.0 release +RUN python -m pip install -r arrow/python/requirements-wheel-build.txt --pre --extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" # ENV CLCACHE_DIR="C:\clcache" # ENV CLCACHE_COMPRESS=1 diff --git a/ci/scripts/python_wheel_macos_build.sh b/ci/scripts/python_wheel_macos_build.sh index fd845c512dcdb..8123a9fdf1c48 100755 --- a/ci/scripts/python_wheel_macos_build.sh +++ b/ci/scripts/python_wheel_macos_build.sh @@ -50,12 +50,15 @@ echo "=== (${PYTHON_VERSION}) Install Python build dependencies ===" export PIP_SITE_PACKAGES=$(python -c 'import site; print(site.getsitepackages()[0])') export PIP_TARGET_PLATFORM="macosx_${MACOSX_DEPLOYMENT_TARGET//./_}_${arch}" +# TODO(GH-39848) Remove the `--pre --extra-index-url` for numpy nightly again before the 16.0 release pip install \ --upgrade \ --only-binary=:all: \ --target $PIP_SITE_PACKAGES \ --platform $PIP_TARGET_PLATFORM \ - -r ${source_dir}/python/requirements-wheel-build.txt + -r ${source_dir}/python/requirements-wheel-build.txt \ + --pre \ + --extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" pip install "delocate>=0.10.3" echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ===" diff --git a/python/pyproject.toml b/python/pyproject.toml index 437de105ab8e7..9079618ad1c7d 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -18,7 +18,12 @@ [build-system] requires = [ "cython >= 0.29.31", - "oldest-supported-numpy>=0.14", + # Starting with NumPy 1.25, NumPy is (by default) as far back compatible + # as oldest-support-numpy was (customizable with a NPY_TARGET_VERSION + # define). For older Python versions (where NumPy 1.25 is not yet avaiable) + # continue using oldest-support-numpy. + "oldest-supported-numpy>=0.14; python_version<'3.9'", + "numpy>=1.25; python_version>='3.9'", "setuptools_scm < 8.0.0", "setuptools >= 40.1.0", "wheel" diff --git a/python/requirements-build.txt b/python/requirements-build.txt index 56e9d479ee9ba..e1372e807f88d 100644 --- a/python/requirements-build.txt +++ b/python/requirements-build.txt @@ -1,4 +1,5 @@ cython>=0.29.31 -oldest-supported-numpy>=0.14 +oldest-supported-numpy>=0.14; python_version<'3.9' +numpy>=1.25; python_version>='3.9' setuptools_scm<8.0.0 setuptools>=38.6.0 diff --git a/python/requirements-wheel-build.txt b/python/requirements-wheel-build.txt index f42ee4a018f3c..044f9de5f8214 100644 --- a/python/requirements-wheel-build.txt +++ b/python/requirements-wheel-build.txt @@ -1,5 +1,6 @@ cython>=0.29.31 -oldest-supported-numpy>=0.14 +oldest-supported-numpy>=0.14; python_version<'3.9' +numpy>=1.25; python_version>='3.9' setuptools_scm<8.0.0 setuptools>=58 wheel diff --git a/python/setup.py b/python/setup.py index d7a2da2077cdd..098d75a3186af 100755 --- a/python/setup.py +++ b/python/setup.py @@ -449,7 +449,7 @@ def has_ext_modules(foo): install_requires = ( - 'numpy >= 1.16.6, <2', + 'numpy >= 1.16.6', )