ARROW-2461: [Python] Build manylinux2010 wheels

dev/tasks/python-wheels/linux-test.sh → dev/tasks/python-wheels/manylinux-test.sh

@@ -20,7 +20,7 @@
 set -e
 
 # Install built wheel
-pip install -q /arrow/python/manylinux1/dist/*.whl
+pip install -q /arrow/python/$WHEEL_TAG/dist/*.whl
 
 # Runs tests on installed distribution from an empty directory
 python --version
@@ -34,6 +34,7 @@ import pyarrow.parquet
 import pyarrow.plasma
 
 if sys.version_info.major > 2:
+    import pyarrow.flight
     import pyarrow.gandiva
 "
 

dev/tasks/python-wheels/travis.linux.yml → dev/tasks/python-wheels/travis.manylinux.yml

@@ -51,22 +51,25 @@ script:
 
   # don't attempt to build it, because the base image by pypa is updated
   # regularly and would cause undeterministic builds
-  - docker-compose pull python-manylinux1
+  - BUILD_IMAGE=python-{{ wheel_tag }}
+  - docker-compose pull $BUILD_IMAGE
   - docker-compose run
       -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }}
       -e PYTHON_VERSION="{{ python_version }}"
       -e UNICODE_WIDTH="{{ unicode_width }}"
-      python-manylinux1
+      $BUILD_IMAGE
   - popd
 
   # test on multiple distributions
   {%- for image in test_docker_images %}
-  - docker run -it --shm-size 2G --volume $(pwd)/arrow:/arrow {{ image }}
-      /arrow/dev/tasks/python-wheels/linux-test.sh
+  - docker run -it --shm-size 2G --volume $(pwd)/arrow:/arrow
+      --env WHEEL_TAG="{{ wheel_tag }}"
+      {{ image }}
+      /arrow/dev/tasks/python-wheels/manylinux-test.sh
   {%- endfor %}
 
   # prepare for deployment
-  - sudo mv arrow/python/manylinux1/dist/* dist/
+  - sudo mv arrow/python/{{ wheel_tag }}/dist/* dist/
 
 deploy:
   provider: releases

dev/tasks/tasks.yml

@@ -28,11 +28,16 @@ groups:
     - conda-win-vs2015-py36
     - conda-win-vs2015-py37
   wheel:
-    - wheel-linux-cp27m
-    - wheel-linux-cp27mu
-    - wheel-linux-cp35m
-    - wheel-linux-cp36m
-    - wheel-linux-cp37m
+    - wheel-manylinux1-cp27m
+    - wheel-manylinux1-cp27mu
+    - wheel-manylinux1-cp35m
+    - wheel-manylinux1-cp36m
+    - wheel-manylinux1-cp37m
+    - wheel-manylinux2010-cp27m
+    - wheel-manylinux2010-cp27mu
+    - wheel-manylinux2010-cp35m
+    - wheel-manylinux2010-cp36m
+    - wheel-manylinux2010-cp37m
     - wheel-osx-cp27m
     - wheel-osx-cp35m
     - wheel-osx-cp36m
@@ -151,65 +156,134 @@ tasks:
 
   ############################## Wheel Linux ##################################
 
-  wheel-linux-cp27m:
+  wheel-manylinux1-cp27m:
     ci: travis
     platform: linux
-    template: python-wheels/travis.linux.yml
+    template: python-wheels/travis.manylinux.yml
     params:
       python_version: 2.7
       unicode_width: 16
+      wheel_tag: manylinux1
       test_docker_images: []
     artifacts:
       - pyarrow-{no_rc_version}-cp27-cp27m-manylinux1_x86_64.whl
 
-  wheel-linux-cp27mu:
+  wheel-manylinux1-cp27mu:
     ci: travis
     platform: linux
-    template: python-wheels/travis.linux.yml
+    template: python-wheels/travis.manylinux.yml
     params:
       python_version: 2.7
       unicode_width: 32
+      wheel_tag: manylinux1
       test_docker_images:
         - python:2.7-slim  # debian ucs4
     artifacts:
       - pyarrow-{no_rc_version}-cp27-cp27mu-manylinux1_x86_64.whl
 
-  wheel-linux-cp35m:
+  wheel-manylinux1-cp35m:
     ci: travis
     platform: linux
-    template: python-wheels/travis.linux.yml
+    template: python-wheels/travis.manylinux.yml
     params:
       python_version: 3.5
       unicode_width: 16
+      wheel_tag: manylinux1
       test_docker_images:
         - python:3.5-slim
     artifacts:
       - pyarrow-{no_rc_version}-cp35-cp35m-manylinux1_x86_64.whl
 
-  wheel-linux-cp36m:
+  wheel-manylinux1-cp36m:
     ci: travis
     platform: linux
-    template: python-wheels/travis.linux.yml
+    template: python-wheels/travis.manylinux.yml
     params:
       python_version: 3.6
       unicode_width: 16
+      wheel_tag: manylinux1
       test_docker_images:
         - python:3.6-slim
     artifacts:
       - pyarrow-{no_rc_version}-cp36-cp36m-manylinux1_x86_64.whl
 
-  wheel-linux-cp37m:
+  wheel-manylinux1-cp37m:
     ci: travis
     platform: linux
-    template: python-wheels/travis.linux.yml
+    template: python-wheels/travis.manylinux.yml
     params:
       python_version: 3.7
       unicode_width: 16
+      wheel_tag: manylinux1
       test_docker_images:
         - python:3.7-slim
     artifacts:
       - pyarrow-{no_rc_version}-cp37-cp37m-manylinux1_x86_64.whl
 
+  wheel-manylinux2010-cp27m:
+    ci: travis
+    platform: linux
+    template: python-wheels/travis.manylinux.yml
+    params:
+      python_version: 2.7
+      unicode_width: 16
+      wheel_tag: manylinux2010
+      test_docker_images: []
+    artifacts:
+      - pyarrow-{no_rc_version}-cp27-cp27m-manylinux2010_x86_64.whl
+
+  wheel-manylinux2010-cp27mu:
+    ci: travis
+    platform: linux
+    template: python-wheels/travis.manylinux.yml
+    params:
+      python_version: 2.7
+      unicode_width: 32
+      wheel_tag: manylinux2010
+      test_docker_images:
+        - python:2.7-slim  # debian ucs4
+    artifacts:
+      - pyarrow-{no_rc_version}-cp27-cp27mu-manylinux2010_x86_64.whl
+
+  wheel-manylinux2010-cp35m:
+    ci: travis
+    platform: linux
+    template: python-wheels/travis.manylinux.yml
+    params:
+      python_version: 3.5
+      unicode_width: 16
+      wheel_tag: manylinux2010
+      test_docker_images:
+        - python:3.5-slim
+    artifacts:
+      - pyarrow-{no_rc_version}-cp35-cp35m-manylinux2010_x86_64.whl
+
+  wheel-manylinux2010-cp36m:
+    ci: travis
+    platform: linux
+    template: python-wheels/travis.manylinux.yml
+    params:
+      python_version: 3.6
+      unicode_width: 16
+      wheel_tag: manylinux2010
+      test_docker_images:
+        - python:3.6-slim
+    artifacts:
+      - pyarrow-{no_rc_version}-cp36-cp36m-manylinux2010_x86_64.whl
+
+  wheel-manylinux2010-cp37m:
+    ci: travis
+    platform: linux
+    template: python-wheels/travis.manylinux.yml
+    params:
+      python_version: 3.7
+      unicode_width: 16
+      wheel_tag: manylinux2010
+      test_docker_images:
+        - python:3.7-slim
+    artifacts:
+      - pyarrow-{no_rc_version}-cp37-cp37m-manylinux2010_x86_64.whl
+
   ############################## Wheel OSX ####################################
 
   wheel-osx-cp27m:

docker-compose.yml

@@ -439,6 +439,28 @@ services:
       - ./python/manylinux1:/io:delegated
     command: /io/build_arrow.sh
 
+  python-manylinux2010:
+    # Usage:
+    #   either build:
+    #     $ docker-compose build python-manylinux2010
+    #   or pull:
+    #     $ docker-compose pull python-manylinux2010
+    #   an then run:
+    #     $ docker-compose run -e PYTHON_VERSION=3.7 python-manylinux2010
+    image: quay.io/ursa-labs/arrow_manylinux2010_x86_64_base:latest
+    build:
+      context: python/manylinux2010
+      dockerfile: Dockerfile-x86_64_base
+    shm_size: 2G
+    environment:
+      PYARROW_PARALLEL: 3
+      PYTHON_VERSION: ${PYTHON_VERSION:-3.6}
+      UNICODE_WIDTH: ${UNICODE_WIDTH:-16}
+    volumes:
+      - .:/arrow:delegated
+      - ./python/manylinux2010:/io:delegated
+    command: /io/build_arrow.sh
+
   ######################### Integration Tests #################################
 
   # impala:

python/.gitignore

@@ -36,8 +36,10 @@ htmlcov
 .asv
 pyarrow/_table_api.h
 
-# manylinux1 temporary files
+# manylinux temporary files
 manylinux1/arrow
+nm_arrow.log
+visible_symbols.log
 
 # plasma store
 pyarrow/plasma_store_server

python/manylinux2010/Dockerfile-x86_64_base

@@ -0,0 +1,105 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM quay.io/pypa/manylinux2010_x86_64:latest
+
+# Install build dependencies
+RUN yum install -y xz bison ccache flex wget
+RUN yum clean all
+
+# Install up-to-date CMake and Ninja versions
+# A fresh CMake is required to recognize recent Boost versions...
+ADD scripts/install_cmake.sh /
+RUN /install_cmake.sh
+
+ADD scripts/build_zlib.sh /
+RUN /build_zlib.sh
+
+WORKDIR /
+RUN git clone https://github.com/matthew-brett/multibuild.git && cd multibuild && git checkout 1a7f31be677185f2dface2643284846e14130c3f
+
+ADD scripts/build_llvm.sh /
+RUN /build_llvm.sh
+
+# clang is only used to precompile Gandiva bitcode
+ADD scripts/build_clang.sh /
+RUN /build_clang.sh
+
+ADD scripts/build_openssl.sh /
+RUN /build_openssl.sh
+
+ADD scripts/build_boost.sh /
+RUN /build_boost.sh
+
+ADD scripts/build_thrift.sh /
+RUN /build_thrift.sh
+ENV THRIFT_HOME /usr/local
+
+ADD scripts/build_gflags.sh /
+RUN /build_gflags.sh
+
+ADD scripts/build_protobuf.sh /
+RUN /build_protobuf.sh
+ENV PROTOBUF_HOME /usr/local
+
+ADD scripts/build_cares.sh /
+RUN /build_cares.sh
+
+ADD scripts/build_grpc.sh /
+RUN /build_grpc.sh
+
+ADD scripts/build_flatbuffers.sh /
+RUN /build_flatbuffers.sh
+ENV FLATBUFFERS_HOME /usr/local
+
+ADD scripts/build_brotli.sh /
+RUN /build_brotli.sh
+ENV BROTLI_HOME /usr/local
+
+ADD scripts/build_snappy.sh /
+RUN /build_snappy.sh
+ENV SNAPPY_HOME /usr/local
+
+ADD scripts/build_lz4.sh /
+RUN /build_lz4.sh
+ENV LZ4_HOME /usr/local
+
+ADD scripts/build_zstd.sh /
+RUN /build_zstd.sh
+ENV ZSTD_HOME /usr/local
+
+ADD scripts/build_glog.sh /
+RUN /build_glog.sh
+ENV GLOG_HOME /usr/local
+
+ADD scripts/build_double_conversion.sh /
+RUN /build_double_conversion.sh
+
+ADD scripts/build_rapidjson.sh /
+RUN /build_rapidjson.sh
+
+ADD scripts/build_re2.sh /
+RUN /build_re2.sh
+
+ADD scripts/build_uriparser.sh /
+RUN /build_uriparser.sh
+
+ADD scripts/build_bz2.sh /
+RUN /build_bz2.sh
+
+ADD scripts/build_virtualenvs.sh /
+RUN /build_virtualenvs.sh