ARROW-2461: [Python] Build manylinux2010 wheels

Author: Antoine Pitrou <[email protected]> Author: Krisztián Szűcs <[email protected]> Closes apache#4675 from pitrou/ARROW-2461-manylinux2010 and squashes the following commits: 2f218d5 <Krisztián Szűcs> remove multibuild's clean_code function call 1186b52 <Krisztián Szűcs> use FETCH_HEAD in the CI templates 17a4c7f <Antoine Pitrou> Nits cb5f285 <Antoine Pitrou> Attempt to factor out package scripts bcd0761 <Antoine Pitrou> ARROW-2461: Build manylinux2010 wheels
kszucs · Jun 25, 2019 · 0568544 · 0568544
1 parent a7f354f
commit 0568544
Show file tree

Hide file tree

Showing 44 changed files with 1,388 additions and 38 deletions.
diff --git a/dev/tasks/conda-recipes/appveyor.yml b/dev/tasks/conda-recipes/appveyor.yml
@@ -45,7 +45,7 @@ test_script:
   # Clone arrow
   - git clone --no-checkout {{ arrow.remote }} arrow || exit /B
   - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} || exit /B
-  - git -C arrow checkout {{ arrow.head }} || exit /B
+  - git -C arrow checkout FETCH_HEAD || exit /B
 
   - pushd arrow\dev\tasks\conda-recipes
 

diff --git a/dev/tasks/conda-recipes/travis.linux.yml b/dev/tasks/conda-recipes/travis.linux.yml
@@ -49,7 +49,7 @@ install:
 before_script:
   - git clone --no-checkout {{ arrow.remote }} arrow
   - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
-  - git -C arrow checkout {{ arrow.head }}
+  - git -C arrow checkout FETCH_HEAD
 
   - pushd arrow/dev/tasks/conda-recipes
 

diff --git a/dev/tasks/conda-recipes/travis.osx.yml b/dev/tasks/conda-recipes/travis.osx.yml
@@ -48,7 +48,7 @@ install:
 before_script:
   - git clone --no-checkout {{ arrow.remote }} arrow
   - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
-  - git -C arrow checkout {{ arrow.head }}
+  - git -C arrow checkout FETCH_HEAD
 
   - pushd arrow/dev/tasks/conda-recipes
 

diff --git a/dev/tasks/docker-tests/circle.linux.yml b/dev/tasks/docker-tests/circle.linux.yml
@@ -25,7 +25,7 @@ jobs:
       - run: docker-compose -v
       - run: git clone --no-checkout {{ arrow.remote }} arrow
       - run: git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
-      - run: git -C arrow checkout {{ arrow.head }}
+      - run: git -C arrow checkout FETCH_HEAD
       - run: git -C arrow submodule update --init --recursive
       - run: |
           pushd arrow

diff --git a/dev/tasks/docker-tests/travis.linux.yml b/dev/tasks/docker-tests/travis.linux.yml
@@ -45,7 +45,7 @@ before_install:
 before_script:
   - git clone --no-checkout {{ arrow.remote }} arrow
   - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
-  - git -C arrow checkout {{ arrow.head }}
+  - git -C arrow checkout FETCH_HEAD
   - git -C arrow submodule update --init --recursive
 
 script:

diff --git a/dev/tasks/gandiva-jars/travis.linux.yml b/dev/tasks/gandiva-jars/travis.linux.yml
@@ -43,7 +43,7 @@ before_install:
 before_script:
   - git clone --no-checkout {{ arrow.remote }} arrow
   - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
-  - git -C arrow checkout {{ arrow.head }}
+  - git -C arrow checkout FETCH_HEAD
 
   - export TRAVIS_BUILD_DIR=$TRAVIS_BUILD_DIR/arrow
 

diff --git a/dev/tasks/gandiva-jars/travis.osx.yml b/dev/tasks/gandiva-jars/travis.osx.yml
@@ -35,7 +35,7 @@ env:
 before_script:
   - git clone --no-checkout {{ arrow.remote }} arrow
   - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
-  - git -C arrow checkout {{ arrow.head }}
+  - git -C arrow checkout FETCH_HEAD
 
   - export TRAVIS_BUILD_DIR=$TRAVIS_BUILD_DIR/arrow
   - brew update

diff --git a/dev/tasks/linux-packages/travis.linux.arm64.yml b/dev/tasks/linux-packages/travis.linux.arm64.yml
@@ -40,7 +40,7 @@ env:
 before_script:
   - git clone --no-checkout {{ arrow.remote }} arrow
   - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
-  - git -C arrow checkout {{ arrow.head }}
+  - git -C arrow checkout FETCH_HEAD
 
 script:
   - pushd arrow/dev/tasks/linux-packages

diff --git a/dev/tasks/linux-packages/travis.linux.yml b/dev/tasks/linux-packages/travis.linux.yml
@@ -34,7 +34,7 @@ env:
 before_script:
   - git clone --no-checkout {{ arrow.remote }} arrow
   - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
-  - git -C arrow checkout {{ arrow.head }}
+  - git -C arrow checkout FETCH_HEAD
 
 script:
   - pushd arrow/dev/tasks/linux-packages

diff --git a/dev/tasks/python-wheels/appveyor.yml b/dev/tasks/python-wheels/appveyor.yml
@@ -36,7 +36,7 @@ build_script:
 
   - git clone --no-checkout {{ arrow.remote }} %ARROW_SRC% || exit /B
   - git -C %ARROW_SRC% fetch -t {{ arrow.remote }} {{ arrow.branch }} || exit /B
-  - git -C %ARROW_SRC% checkout {{ arrow.head }} || exit /B
+  - git -C %ARROW_SRC% checkout FETCH_HEAD || exit /B
   - git -C %ARROW_SRC% submodule update --init || exit /B
 
   # Avoid picking up AppVeyor-installed OpenSSL (linker errors with gRPC)

diff --git a/dev/tasks/python-wheels/linux-test.sh → dev/tasks/python-wheels/manylinux-test.sh b/dev/tasks/python-wheels/linux-test.sh → dev/tasks/python-wheels/manylinux-test.sh
@@ -20,7 +20,7 @@
 set -e
 
 # Install built wheel
-pip install -q /arrow/python/manylinux1/dist/*.whl
+pip install -q /arrow/python/$WHEEL_TAG/dist/*.whl
 
 # Runs tests on installed distribution from an empty directory
 python --version
@@ -34,6 +34,7 @@ import pyarrow.parquet
 import pyarrow.plasma
 
 if sys.version_info.major > 2:
+    import pyarrow.flight
     import pyarrow.gandiva
 "
 

diff --git a/dev/tasks/python-wheels/travis.linux.yml → dev/tasks/python-wheels/travis.manylinux.yml b/dev/tasks/python-wheels/travis.linux.yml → dev/tasks/python-wheels/travis.manylinux.yml
@@ -39,7 +39,7 @@ before_script:
 script:
   - git clone --no-checkout {{ arrow.remote }} arrow
   - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
-  - git -C arrow checkout {{ arrow.head }}
+  - git -C arrow checkout FETCH_HEAD
   - mkdir -p dist
 
   # build wheel
@@ -51,22 +51,25 @@ script:
 
   # don't attempt to build it, because the base image by pypa is updated
   # regularly and would cause undeterministic builds
-  - docker-compose pull python-manylinux1
+  - BUILD_IMAGE=python-{{ wheel_tag }}
+  - docker-compose pull $BUILD_IMAGE
   - docker-compose run
       -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }}
       -e PYTHON_VERSION="{{ python_version }}"
       -e UNICODE_WIDTH="{{ unicode_width }}"
-      python-manylinux1
+      $BUILD_IMAGE
   - popd
 
   # test on multiple distributions
   {%- for image in test_docker_images %}
-  - docker run -it --shm-size 2G --volume $(pwd)/arrow:/arrow {{ image }}
-      /arrow/dev/tasks/python-wheels/linux-test.sh
+  - docker run -it --shm-size 2G --volume $(pwd)/arrow:/arrow
+      --env WHEEL_TAG="{{ wheel_tag }}"
+      {{ image }}
+      /arrow/dev/tasks/python-wheels/manylinux-test.sh
   {%- endfor %}
 
   # prepare for deployment
-  - sudo mv arrow/python/manylinux1/dist/* dist/
+  - sudo mv arrow/python/{{ wheel_tag }}/dist/* dist/
 
 deploy:
   provider: releases

diff --git a/dev/tasks/python-wheels/travis.osx.yml b/dev/tasks/python-wheels/travis.osx.yml
@@ -36,7 +36,7 @@ before_install:
 
   - git clone --no-checkout {{ arrow.remote }} arrow
   - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
-  - git -C arrow checkout {{ arrow.head }}
+  - git -C arrow checkout FETCH_HEAD
 
   # ARROW-3976 Old versions of git can cause failures when Homebrew prints a
   # donation solicitation. Attempt to update git
@@ -60,9 +60,6 @@ before_install:
 install:
   - mkdir -p dist
 
-  # multibuild's default clean_code function
-  - clean_code arrow {{ arrow.head }}
-
   # the following functions are defined in osx-build.sh
   - build_wheel arrow
 

diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
@@ -28,11 +28,16 @@ groups:
     - conda-win-vs2015-py36
     - conda-win-vs2015-py37
   wheel:
-    - wheel-linux-cp27m
-    - wheel-linux-cp27mu
-    - wheel-linux-cp35m
-    - wheel-linux-cp36m
-    - wheel-linux-cp37m
+    - wheel-manylinux1-cp27m
+    - wheel-manylinux1-cp27mu
+    - wheel-manylinux1-cp35m
+    - wheel-manylinux1-cp36m
+    - wheel-manylinux1-cp37m
+    - wheel-manylinux2010-cp27m
+    - wheel-manylinux2010-cp27mu
+    - wheel-manylinux2010-cp35m
+    - wheel-manylinux2010-cp36m
+    - wheel-manylinux2010-cp37m
     - wheel-osx-cp27m
     - wheel-osx-cp35m
     - wheel-osx-cp36m
@@ -151,65 +156,134 @@ tasks:
 
   ############################## Wheel Linux ##################################
 
-  wheel-linux-cp27m:
+  wheel-manylinux1-cp27m:
     ci: travis
     platform: linux
-    template: python-wheels/travis.linux.yml
+    template: python-wheels/travis.manylinux.yml
     params:
       python_version: 2.7
       unicode_width: 16
+      wheel_tag: manylinux1
       test_docker_images: []
     artifacts:
       - pyarrow-{no_rc_version}-cp27-cp27m-manylinux1_x86_64.whl
 
-  wheel-linux-cp27mu:
+  wheel-manylinux1-cp27mu:
     ci: travis
     platform: linux
-    template: python-wheels/travis.linux.yml
+    template: python-wheels/travis.manylinux.yml
     params:
       python_version: 2.7
       unicode_width: 32
+      wheel_tag: manylinux1
       test_docker_images:
         - python:2.7-slim  # debian ucs4
     artifacts:
       - pyarrow-{no_rc_version}-cp27-cp27mu-manylinux1_x86_64.whl
 
-  wheel-linux-cp35m:
+  wheel-manylinux1-cp35m:
     ci: travis
     platform: linux
-    template: python-wheels/travis.linux.yml
+    template: python-wheels/travis.manylinux.yml
     params:
       python_version: 3.5
       unicode_width: 16
+      wheel_tag: manylinux1
       test_docker_images:
         - python:3.5-slim
     artifacts:
       - pyarrow-{no_rc_version}-cp35-cp35m-manylinux1_x86_64.whl
 
-  wheel-linux-cp36m:
+  wheel-manylinux1-cp36m:
     ci: travis
     platform: linux
-    template: python-wheels/travis.linux.yml
+    template: python-wheels/travis.manylinux.yml
     params:
       python_version: 3.6
       unicode_width: 16
+      wheel_tag: manylinux1
       test_docker_images:
         - python:3.6-slim
     artifacts:
       - pyarrow-{no_rc_version}-cp36-cp36m-manylinux1_x86_64.whl
 
-  wheel-linux-cp37m:
+  wheel-manylinux1-cp37m:
     ci: travis
     platform: linux
-    template: python-wheels/travis.linux.yml
+    template: python-wheels/travis.manylinux.yml
     params:
       python_version: 3.7
       unicode_width: 16
+      wheel_tag: manylinux1
       test_docker_images:
         - python:3.7-slim
     artifacts:
       - pyarrow-{no_rc_version}-cp37-cp37m-manylinux1_x86_64.whl
 
+  wheel-manylinux2010-cp27m:
+    ci: travis
+    platform: linux
+    template: python-wheels/travis.manylinux.yml
+    params:
+      python_version: 2.7
+      unicode_width: 16
+      wheel_tag: manylinux2010
+      test_docker_images: []
+    artifacts:
+      - pyarrow-{no_rc_version}-cp27-cp27m-manylinux2010_x86_64.whl
+
+  wheel-manylinux2010-cp27mu:
+    ci: travis
+    platform: linux
+    template: python-wheels/travis.manylinux.yml
+    params:
+      python_version: 2.7
+      unicode_width: 32
+      wheel_tag: manylinux2010
+      test_docker_images:
+        - python:2.7-slim  # debian ucs4
+    artifacts:
+      - pyarrow-{no_rc_version}-cp27-cp27mu-manylinux2010_x86_64.whl
+
+  wheel-manylinux2010-cp35m:
+    ci: travis
+    platform: linux
+    template: python-wheels/travis.manylinux.yml
+    params:
+      python_version: 3.5
+      unicode_width: 16
+      wheel_tag: manylinux2010
+      test_docker_images:
+        - python:3.5-slim
+    artifacts:
+      - pyarrow-{no_rc_version}-cp35-cp35m-manylinux2010_x86_64.whl
+
+  wheel-manylinux2010-cp36m:
+    ci: travis
+    platform: linux
+    template: python-wheels/travis.manylinux.yml
+    params:
+      python_version: 3.6
+      unicode_width: 16
+      wheel_tag: manylinux2010
+      test_docker_images:
+        - python:3.6-slim
+    artifacts:
+      - pyarrow-{no_rc_version}-cp36-cp36m-manylinux2010_x86_64.whl
+
+  wheel-manylinux2010-cp37m:
+    ci: travis
+    platform: linux
+    template: python-wheels/travis.manylinux.yml
+    params:
+      python_version: 3.7
+      unicode_width: 16
+      wheel_tag: manylinux2010
+      test_docker_images:
+        - python:3.7-slim
+    artifacts:
+      - pyarrow-{no_rc_version}-cp37-cp37m-manylinux2010_x86_64.whl
+
   ############################## Wheel OSX ####################################
 
   wheel-osx-cp27m:

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -439,6 +439,28 @@ services:
       - ./python/manylinux1:/io:delegated
     command: /io/build_arrow.sh
 
+  python-manylinux2010:
+    # Usage:
+    #   either build:
+    #     $ docker-compose build python-manylinux2010
+    #   or pull:
+    #     $ docker-compose pull python-manylinux2010
+    #   an then run:
+    #     $ docker-compose run -e PYTHON_VERSION=3.7 python-manylinux2010
+    image: quay.io/ursa-labs/arrow_manylinux2010_x86_64_base:latest
+    build:
+      context: python/manylinux2010
+      dockerfile: Dockerfile-x86_64_base
+    shm_size: 2G
+    environment:
+      PYARROW_PARALLEL: 3
+      PYTHON_VERSION: ${PYTHON_VERSION:-3.6}
+      UNICODE_WIDTH: ${UNICODE_WIDTH:-16}
+    volumes:
+      - .:/arrow:delegated
+      - ./python/manylinux2010:/io:delegated
+    command: /io/build_arrow.sh
+
   ######################### Integration Tests #################################
 
   # impala:

diff --git a/python/.gitignore b/python/.gitignore
@@ -36,8 +36,10 @@ htmlcov
 .asv
 pyarrow/_table_api.h
 
-# manylinux1 temporary files
+# manylinux temporary files
 manylinux1/arrow
+nm_arrow.log
+visible_symbols.log
 
 # plasma store
 pyarrow/plasma_store_server
diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh
@@ -57,12 +57,12 @@ PIP="${CPYTHON_PATH}/bin/pip"
 PATH="${PATH}:${CPYTHON_PATH}"
 
 if [ "${PYTHON_VERSION}" != "2.7" ]; then
-  # Gandiva is not supported on Python 2.7
   export PYARROW_WITH_FLIGHT=1
   export PYARROW_WITH_GANDIVA=1
   export BUILD_ARROW_FLIGHT=ON
   export BUILD_ARROW_GANDIVA=ON
 else
+  # Flight and Gandiva are not supported on Python 2.7
   export PYARROW_WITH_FLIGHT=0
   export PYARROW_WITH_GANDIVA=0
   export BUILD_ARROW_FLIGHT=OFF