From edd39a2a8ddc1b2aef2397fff0af817e639bb30f Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Mon, 24 Jun 2019 16:19:17 +0200 Subject: [PATCH] ARROW-2461: [Python] Build manylinux2010 wheels --- .../{linux-test.sh => manylinux1-test.sh} | 1 + dev/tasks/python-wheels/manylinux2010-test.sh | 45 ++++++ ...travis.linux.yml => travis.manylinux1.yml} | 2 +- .../python-wheels/travis.manylinux2010.yml | 82 ++++++++++ dev/tasks/tasks.yml | 94 +++++++++-- docker-compose.yml | 22 +++ python/.gitignore | 4 +- python/manylinux2010/Dockerfile-x86_64_base | 105 ++++++++++++ python/manylinux2010/README.md | 109 +++++++++++++ python/manylinux2010/build_arrow.sh | 150 ++++++++++++++++++ python/manylinux2010/scripts/build_boost.sh | 43 +++++ python/manylinux2010/scripts/build_brotli.sh | 34 ++++ python/manylinux2010/scripts/build_bz2.sh | 30 ++++ python/manylinux2010/scripts/build_cares.sh | 34 ++++ python/manylinux2010/scripts/build_clang.sh | 43 +++++ .../scripts/build_double_conversion.sh | 34 ++++ .../scripts/build_flatbuffers.sh | 32 ++++ python/manylinux2010/scripts/build_gflags.sh | 38 +++++ python/manylinux2010/scripts/build_glog.sh | 35 ++++ python/manylinux2010/scripts/build_grpc.sh | 41 +++++ python/manylinux2010/scripts/build_llvm.sh | 48 ++++++ python/manylinux2010/scripts/build_lz4.sh | 33 ++++ python/manylinux2010/scripts/build_openssl.sh | 32 ++++ .../manylinux2010/scripts/build_protobuf.sh | 29 ++++ .../manylinux2010/scripts/build_rapidjson.sh | 37 +++++ python/manylinux2010/scripts/build_re2.sh | 35 ++++ python/manylinux2010/scripts/build_snappy.sh | 31 ++++ python/manylinux2010/scripts/build_thrift.sh | 46 ++++++ .../manylinux2010/scripts/build_uriparser.sh | 37 +++++ .../scripts/build_virtualenvs.sh | 72 +++++++++ python/manylinux2010/scripts/build_zlib.sh | 26 +++ python/manylinux2010/scripts/build_zstd.sh | 39 +++++ .../scripts/check_arrow_visibility.sh | 35 ++++ python/manylinux2010/scripts/install_cmake.sh | 22 +++ 34 files changed, 1483 insertions(+), 17 deletions(-) rename dev/tasks/python-wheels/{linux-test.sh => manylinux1-test.sh} (98%) create mode 100755 dev/tasks/python-wheels/manylinux2010-test.sh rename dev/tasks/python-wheels/{travis.linux.yml => travis.manylinux1.yml} (97%) create mode 100644 dev/tasks/python-wheels/travis.manylinux2010.yml create mode 100644 python/manylinux2010/Dockerfile-x86_64_base create mode 100644 python/manylinux2010/README.md create mode 100755 python/manylinux2010/build_arrow.sh create mode 100755 python/manylinux2010/scripts/build_boost.sh create mode 100755 python/manylinux2010/scripts/build_brotli.sh create mode 100755 python/manylinux2010/scripts/build_bz2.sh create mode 100755 python/manylinux2010/scripts/build_cares.sh create mode 100755 python/manylinux2010/scripts/build_clang.sh create mode 100755 python/manylinux2010/scripts/build_double_conversion.sh create mode 100755 python/manylinux2010/scripts/build_flatbuffers.sh create mode 100755 python/manylinux2010/scripts/build_gflags.sh create mode 100755 python/manylinux2010/scripts/build_glog.sh create mode 100755 python/manylinux2010/scripts/build_grpc.sh create mode 100755 python/manylinux2010/scripts/build_llvm.sh create mode 100755 python/manylinux2010/scripts/build_lz4.sh create mode 100755 python/manylinux2010/scripts/build_openssl.sh create mode 100755 python/manylinux2010/scripts/build_protobuf.sh create mode 100755 python/manylinux2010/scripts/build_rapidjson.sh create mode 100755 python/manylinux2010/scripts/build_re2.sh create mode 100755 python/manylinux2010/scripts/build_snappy.sh create mode 100755 python/manylinux2010/scripts/build_thrift.sh create mode 100755 python/manylinux2010/scripts/build_uriparser.sh create mode 100755 python/manylinux2010/scripts/build_virtualenvs.sh create mode 100755 python/manylinux2010/scripts/build_zlib.sh create mode 100755 python/manylinux2010/scripts/build_zstd.sh create mode 100755 python/manylinux2010/scripts/check_arrow_visibility.sh create mode 100755 python/manylinux2010/scripts/install_cmake.sh diff --git a/dev/tasks/python-wheels/linux-test.sh b/dev/tasks/python-wheels/manylinux1-test.sh similarity index 98% rename from dev/tasks/python-wheels/linux-test.sh rename to dev/tasks/python-wheels/manylinux1-test.sh index 1fdabc735da29..776747e334e9d 100755 --- a/dev/tasks/python-wheels/linux-test.sh +++ b/dev/tasks/python-wheels/manylinux1-test.sh @@ -34,6 +34,7 @@ import pyarrow.parquet import pyarrow.plasma if sys.version_info.major > 2: + import pyarrow.flight import pyarrow.gandiva " diff --git a/dev/tasks/python-wheels/manylinux2010-test.sh b/dev/tasks/python-wheels/manylinux2010-test.sh new file mode 100755 index 0000000000000..4d61f90e6c4ee --- /dev/null +++ b/dev/tasks/python-wheels/manylinux2010-test.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e + +# Install built wheel +pip install -q /arrow/python/manylinux2010/dist/*.whl + +# Runs tests on installed distribution from an empty directory +python --version + +# Test optional dependencies +python -c " +import sys +import pyarrow +import pyarrow.orc +import pyarrow.parquet +import pyarrow.plasma + +if sys.version_info.major > 2: + import pyarrow.flight + import pyarrow.gandiva +" + +export ARROW_TEST_DATA=/arrow/testing/data + +# Run pyarrow tests +pip install -q -r /arrow/python/requirements-test.txt +pytest -v --pyargs pyarrow diff --git a/dev/tasks/python-wheels/travis.linux.yml b/dev/tasks/python-wheels/travis.manylinux1.yml similarity index 97% rename from dev/tasks/python-wheels/travis.linux.yml rename to dev/tasks/python-wheels/travis.manylinux1.yml index 31e633e6752e9..e6098759ded94 100644 --- a/dev/tasks/python-wheels/travis.linux.yml +++ b/dev/tasks/python-wheels/travis.manylinux1.yml @@ -62,7 +62,7 @@ script: # test on multiple distributions {%- for image in test_docker_images %} - docker run -it --shm-size 2G --volume $(pwd)/arrow:/arrow {{ image }} - /arrow/dev/tasks/python-wheels/linux-test.sh + /arrow/dev/tasks/python-wheels/manylinux1-test.sh {%- endfor %} # prepare for deployment diff --git a/dev/tasks/python-wheels/travis.manylinux2010.yml b/dev/tasks/python-wheels/travis.manylinux2010.yml new file mode 100644 index 0000000000000..d6c7ab24d515d --- /dev/null +++ b/dev/tasks/python-wheels/travis.manylinux2010.yml @@ -0,0 +1,82 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +os: linux + +services: + - docker + +# Update docker to support newer docker-compose versions +addons: + apt: + packages: + - docker-ce + +# don't build twice +if: tag IS blank + +env: + global: + - PLAT=x86_64 + - TRAVIS_TAG={{ task.tag }} + +before_script: + - sudo pip install -U docker-compose + +script: + - git clone --no-checkout {{ arrow.remote }} arrow + - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} + - git -C arrow checkout {{ arrow.head }} + - mkdir -p dist + + # build wheel + - pushd arrow + + # Pull testing resources + - git submodule init + - git submodule update + + # don't attempt to build it, because the base image by pypa is updated + # regularly and would cause undeterministic builds + - docker-compose pull python-manylinux2010 + - docker-compose run + -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} + -e PYTHON_VERSION="{{ python_version }}" + -e UNICODE_WIDTH="{{ unicode_width }}" + python-manylinux2010 + - popd + + # test on multiple distributions + {%- for image in test_docker_images %} + - docker run -it --shm-size 2G --volume $(pwd)/arrow:/arrow {{ image }} + /arrow/dev/tasks/python-wheels/manylinux2010-test.sh + {%- endfor %} + + # prepare for deployment + - sudo mv arrow/python/manylinux2010/dist/* dist/ + +deploy: + provider: releases + api_key: $CROSSBOW_GITHUB_TOKEN + file_glob: true + file: dist/*.whl + skip_cleanup: true + on: + tags: true + +notifications: + email: + - {{ job.email }} diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 5e74a5ed032ee..f3b19dc06104f 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -28,11 +28,16 @@ groups: - conda-win-vs2015-py36 - conda-win-vs2015-py37 wheel: - - wheel-linux-cp27m - - wheel-linux-cp27mu - - wheel-linux-cp35m - - wheel-linux-cp36m - - wheel-linux-cp37m + - wheel-manylinux1-cp27m + - wheel-manylinux1-cp27mu + - wheel-manylinux1-cp35m + - wheel-manylinux1-cp36m + - wheel-manylinux1-cp37m + - wheel-manylinux2010-cp27m + - wheel-manylinux2010-cp27mu + - wheel-manylinux2010-cp35m + - wheel-manylinux2010-cp36m + - wheel-manylinux2010-cp37m - wheel-osx-cp27m - wheel-osx-cp35m - wheel-osx-cp36m @@ -151,10 +156,10 @@ tasks: ############################## Wheel Linux ################################## - wheel-linux-cp27m: + wheel-manylinux1-cp27m: ci: travis platform: linux - template: python-wheels/travis.linux.yml + template: python-wheels/travis.manylinux1.yml params: python_version: 2.7 unicode_width: 16 @@ -162,10 +167,10 @@ tasks: artifacts: - pyarrow-{no_rc_version}-cp27-cp27m-manylinux1_x86_64.whl - wheel-linux-cp27mu: + wheel-manylinux1-cp27mu: ci: travis platform: linux - template: python-wheels/travis.linux.yml + template: python-wheels/travis.manylinux1.yml params: python_version: 2.7 unicode_width: 32 @@ -174,10 +179,10 @@ tasks: artifacts: - pyarrow-{no_rc_version}-cp27-cp27mu-manylinux1_x86_64.whl - wheel-linux-cp35m: + wheel-manylinux1-cp35m: ci: travis platform: linux - template: python-wheels/travis.linux.yml + template: python-wheels/travis.manylinux1.yml params: python_version: 3.5 unicode_width: 16 @@ -186,10 +191,10 @@ tasks: artifacts: - pyarrow-{no_rc_version}-cp35-cp35m-manylinux1_x86_64.whl - wheel-linux-cp36m: + wheel-manylinux1-cp36m: ci: travis platform: linux - template: python-wheels/travis.linux.yml + template: python-wheels/travis.manylinux1.yml params: python_version: 3.6 unicode_width: 16 @@ -198,10 +203,10 @@ tasks: artifacts: - pyarrow-{no_rc_version}-cp36-cp36m-manylinux1_x86_64.whl - wheel-linux-cp37m: + wheel-manylinux1-cp37m: ci: travis platform: linux - template: python-wheels/travis.linux.yml + template: python-wheels/travis.manylinux1.yml params: python_version: 3.7 unicode_width: 16 @@ -210,6 +215,65 @@ tasks: artifacts: - pyarrow-{no_rc_version}-cp37-cp37m-manylinux1_x86_64.whl + wheel-manylinux2010-cp27m: + ci: travis + platform: linux + template: python-wheels/travis.manylinux2010.yml + params: + python_version: 2.7 + unicode_width: 16 + test_docker_images: [] + artifacts: + - pyarrow-{no_rc_version}-cp27-cp27m-manylinux2010_x86_64.whl + + wheel-manylinux2010-cp27mu: + ci: travis + platform: linux + template: python-wheels/travis.manylinux2010.yml + params: + python_version: 2.7 + unicode_width: 32 + test_docker_images: + - python:2.7-slim # debian ucs4 + artifacts: + - pyarrow-{no_rc_version}-cp27-cp27mu-manylinux2010_x86_64.whl + + wheel-manylinux2010-cp35m: + ci: travis + platform: linux + template: python-wheels/travis.manylinux2010.yml + params: + python_version: 3.5 + unicode_width: 16 + test_docker_images: + - python:3.5-slim + artifacts: + - pyarrow-{no_rc_version}-cp35-cp35m-manylinux2010_x86_64.whl + + wheel-manylinux2010-cp36m: + ci: travis + platform: linux + template: python-wheels/travis.manylinux2010.yml + params: + python_version: 3.6 + unicode_width: 16 + test_docker_images: + - python:3.6-slim + artifacts: + - pyarrow-{no_rc_version}-cp36-cp36m-manylinux2010_x86_64.whl + + wheel-manylinux2010-cp37m: + ci: travis + platform: linux + template: python-wheels/travis.manylinux2010.yml + params: + python_version: 3.7 + unicode_width: 16 + test_docker_images: + - python:3.7-slim + artifacts: + - pyarrow-{no_rc_version}-cp37-cp37m-manylinux2010_x86_64.whl + ############################## Wheel OSX #################################### wheel-osx-cp27m: diff --git a/docker-compose.yml b/docker-compose.yml index 6ff601deba45c..adad951a2c08d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -439,6 +439,28 @@ services: - ./python/manylinux1:/io:delegated command: /io/build_arrow.sh + python-manylinux2010: + # Usage: + # either build: + # $ docker-compose build python-manylinux2010 + # or pull: + # $ docker-compose pull python-manylinux2010 + # an then run: + # $ docker-compose run -e PYTHON_VERSION=3.7 python-manylinux2010 + image: quay.io/ursa-labs/arrow_manylinux2010_x86_64_base:latest + build: + context: python/manylinux2010 + dockerfile: Dockerfile-x86_64_base + shm_size: 2G + environment: + PYARROW_PARALLEL: 3 + PYTHON_VERSION: ${PYTHON_VERSION:-3.6} + UNICODE_WIDTH: ${UNICODE_WIDTH:-16} + volumes: + - .:/arrow:delegated + - ./python/manylinux2010:/io:delegated + command: /io/build_arrow.sh + ######################### Integration Tests ################################# # impala: diff --git a/python/.gitignore b/python/.gitignore index 8f08f9377bf6e..8fdd7c4ebfb65 100644 --- a/python/.gitignore +++ b/python/.gitignore @@ -36,8 +36,10 @@ htmlcov .asv pyarrow/_table_api.h -# manylinux1 temporary files +# manylinux temporary files manylinux1/arrow +nm_arrow.log +visible_symbols.log # plasma store pyarrow/plasma_store_server diff --git a/python/manylinux2010/Dockerfile-x86_64_base b/python/manylinux2010/Dockerfile-x86_64_base new file mode 100644 index 0000000000000..cd6d445b4dadd --- /dev/null +++ b/python/manylinux2010/Dockerfile-x86_64_base @@ -0,0 +1,105 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM quay.io/pypa/manylinux2010_x86_64:latest + +# Install build dependencies +RUN yum install -y xz bison ccache flex wget +RUN yum clean all + +# Install up-to-date CMake and Ninja versions +# A fresh CMake is required to recognize recent Boost versions... +ADD scripts/install_cmake.sh / +RUN /install_cmake.sh + +ADD scripts/build_zlib.sh / +RUN /build_zlib.sh + +WORKDIR / +RUN git clone https://github.com/matthew-brett/multibuild.git && cd multibuild && git checkout 1a7f31be677185f2dface2643284846e14130c3f + +ADD scripts/build_llvm.sh / +RUN /build_llvm.sh + +# clang is only used to precompile Gandiva bitcode +ADD scripts/build_clang.sh / +RUN /build_clang.sh + +ADD scripts/build_openssl.sh / +RUN /build_openssl.sh + +ADD scripts/build_boost.sh / +RUN /build_boost.sh + +ADD scripts/build_thrift.sh / +RUN /build_thrift.sh +ENV THRIFT_HOME /usr/local + +ADD scripts/build_gflags.sh / +RUN /build_gflags.sh + +ADD scripts/build_protobuf.sh / +RUN /build_protobuf.sh +ENV PROTOBUF_HOME /usr/local + +ADD scripts/build_cares.sh / +RUN /build_cares.sh + +ADD scripts/build_grpc.sh / +RUN /build_grpc.sh + +ADD scripts/build_flatbuffers.sh / +RUN /build_flatbuffers.sh +ENV FLATBUFFERS_HOME /usr/local + +ADD scripts/build_brotli.sh / +RUN /build_brotli.sh +ENV BROTLI_HOME /usr/local + +ADD scripts/build_snappy.sh / +RUN /build_snappy.sh +ENV SNAPPY_HOME /usr/local + +ADD scripts/build_lz4.sh / +RUN /build_lz4.sh +ENV LZ4_HOME /usr/local + +ADD scripts/build_zstd.sh / +RUN /build_zstd.sh +ENV ZSTD_HOME /usr/local + +ADD scripts/build_glog.sh / +RUN /build_glog.sh +ENV GLOG_HOME /usr/local + +ADD scripts/build_double_conversion.sh / +RUN /build_double_conversion.sh + +ADD scripts/build_rapidjson.sh / +RUN /build_rapidjson.sh + +ADD scripts/build_re2.sh / +RUN /build_re2.sh + +ADD scripts/build_uriparser.sh / +RUN /build_uriparser.sh + +ADD scripts/build_bz2.sh / +RUN /build_bz2.sh + +ADD scripts/build_virtualenvs.sh / +RUN /build_virtualenvs.sh diff --git a/python/manylinux2010/README.md b/python/manylinux2010/README.md new file mode 100644 index 0000000000000..2ed1d54a7dff6 --- /dev/null +++ b/python/manylinux2010/README.md @@ -0,0 +1,109 @@ + + +## Manylinux2010 wheels for Apache Arrow + +This folder provides base Docker images and an infrastructure to build +`manylinux2010` compatible Python wheels that should be installable on all +Linux distributions published in last four years. + +The process is split up in two parts: + +1. There are base Docker images that contain the build dependencies for + Arrow. Those images do not need to be rebuilt frequently, and are hosted + on the public quay.io service. + +2. Based on on these images, there is a bash script (`build_arrow.sh`) that + the PyArrow wheels for all supported Python versions, and place them + in the `dist` folder. + +### Building PyArrow + +You can build the PyArrow wheels by running the following command in this +directroy (this is for Python 2.7 with unicode width 16, similarly you can pass +in `PYTHON_VERSION="3.5"`, `PYTHON_VERSION="3.6"` or `PYTHON_VERSION="3.7"` or +use `PYTHON_VERSION="2.7"` with `UNICODE_WIDTH=32`): + +```bash +# Build the python packages +docker run --env PYTHON_VERSION="2.7" --env UNICODE_WIDTH=16 --shm-size=2g --rm -t -i -v $PWD:/io -v $PWD/../../:/arrow quay.io/ursa-labs/arrow_manylinux2010_x86_64_base:latest /io/build_arrow.sh +# Now the new packages are located in the dist/ folder +ls -l dist/ +``` + +### Re-building the build image + +In case you want to make changes to the base Docker image (for example because +you want to update a dependency to a new version), you must re-build it. +The Docker configuration is in `Dockerfile-x86_64_base`, and it calls into +scripts stored under the `scripts` directory. + +```bash +docker build -t arrow_manylinux2010_x86_64_base -f Dockerfile-x86_64_base . +``` + +For each dependency, a bash script in the `scripts/` directory downloads the +source code, builds it and installs the build results. At the end of each +dependency build the sources are removed again so that only the binary +installation of a dependency is persisted in the Docker image. + +### Publishing a new build image + +If you have write access to the `quay.io` Ursa Labs account, you can directly +publish a build image that you built locally. + +For that you need to first tag your image for quay.io upload: +```bash +$ docker image tag arrow_manylinux2010_x86_64_base:latest quay.io/ursa-labs/arrow_manylinux2010_x86_64_base +``` + +Then you can push it: +```bash +$ docker image push quay.io/ursa-labs/arrow_manylinux2010_x86_64_base +The push refers to repository [quay.io/ursa-labs/arrow_manylinux2010_x86_64_base] +a1ab88d27acc: Pushing [==============> ] 492.5MB/1.645GB +[... etc. ...] +``` + +### Using quay.io to trigger and build the docker image + +You can also create your own `quay.io` repository and trigger builds there from +your Github fork of the Arrow repository. + +1. Make the change in the build scripts (eg. to modify the boost build, update `scripts/boost.sh`). + +2. Setup an account on quay.io and link to your GitHub account + +3. In quay.io, Add a new repository using : + + 1. Link to GitHub repository push + 2. Trigger build on changes to a specific branch (eg. myquay) of the repo (eg. `pravindra/arrow`) + 3. Set Dockerfile location to `/python/manylinux2010/Dockerfile-x86_64_base` + 4. Set Context location to `/python/manylinux2010` + +4. Push change (in step 1) to the branch specified in step 3.ii + + * This should trigger a build in quay.io, the build takes about 2 hrs to finish. + +5. Add a tag `latest` to the build after step 4 finishes, save the build ID (eg. `quay.io/pravindra/arrow_manylinux2010_x86_64_base:latest`) + +6. In your arrow PR, + + * include the change from 1. + * modify `travis_script_manylinux.sh` to switch to the location from step 5 for the docker image. diff --git a/python/manylinux2010/build_arrow.sh b/python/manylinux2010/build_arrow.sh new file mode 100755 index 0000000000000..0626a330f377d --- /dev/null +++ b/python/manylinux2010/build_arrow.sh @@ -0,0 +1,150 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Usage: +# docker run --rm -v $PWD:/io arrow-base-x86_64 /io/build_arrow.sh + +# Build upon the scripts in https://github.com/matthew-brett/manylinux-builds +# * Copyright (c) 2013-2016, Matt Terry and Matthew Brett (BSD 2-clause) + +source /multibuild/manylinux_utils.sh + +# Quit on failure +set -e + +# Print commands for debugging +# set -x + +cd /arrow/python + +# PyArrow build configuration +export PYARROW_BUILD_TYPE='release' +export PYARROW_CMAKE_GENERATOR='Ninja' +export PYARROW_WITH_ORC=1 +export PYARROW_WITH_PARQUET=1 +export PYARROW_WITH_PLASMA=1 +export PYARROW_BUNDLE_ARROW_CPP=1 +export PYARROW_BUNDLE_BOOST=1 +export PYARROW_BOOST_NAMESPACE=arrow_boost +export PKG_CONFIG_PATH=/usr/lib/pkgconfig:/arrow-dist/lib/pkgconfig + +export PYARROW_CMAKE_OPTIONS='-DBoost_NAMESPACE=arrow_boost -DBOOST_ROOT=/arrow_boost_dist' +# Ensure the target directory exists +mkdir -p /io/dist + +# Must pass PYTHON_VERSION and UNICODE_WIDTH env variables +# possible values are: 2.7,16 2.7,32 3.5,16 3.6,16 3.7,16 + +CPYTHON_PATH="$(cpython_path ${PYTHON_VERSION} ${UNICODE_WIDTH})" +PYTHON_INTERPRETER="${CPYTHON_PATH}/bin/python" +PIP="${CPYTHON_PATH}/bin/pip" +PATH="${PATH}:${CPYTHON_PATH}" + +if [ "${PYTHON_VERSION}" != "2.7" ]; then + # Gandiva is not supported on Python 2.7 + export PYARROW_WITH_FLIGHT=1 + export PYARROW_WITH_GANDIVA=1 + export BUILD_ARROW_FLIGHT=ON + export BUILD_ARROW_GANDIVA=ON +else + export PYARROW_WITH_FLIGHT=0 + export PYARROW_WITH_GANDIVA=0 + export BUILD_ARROW_FLIGHT=OFF + export BUILD_ARROW_GANDIVA=OFF +fi + +# ARROW-3052(wesm): ORC is being bundled until it can be added to the +# manylinux1 image + +echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ===" +ARROW_BUILD_DIR=/tmp/build-PY${PYTHON_VERSION}-${UNICODE_WIDTH} +mkdir -p "${ARROW_BUILD_DIR}" +pushd "${ARROW_BUILD_DIR}" +PATH="${CPYTHON_PATH}/bin:${PATH}" cmake -DCMAKE_BUILD_TYPE=Release \ + -DARROW_DEPENDENCY_SOURCE="SYSTEM" \ + -DZLIB_ROOT=/usr/local \ + -DCMAKE_INSTALL_PREFIX=/arrow-dist \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DARROW_BUILD_TESTS=OFF \ + -DARROW_BUILD_SHARED=ON \ + -DARROW_BUILD_STATIC=OFF \ + -DARROW_BOOST_USE_SHARED=ON \ + -DARROW_GANDIVA_PC_CXX_FLAGS="-isystem;/opt/rh/devtoolset-8/root/usr/include/c++/8/;-isystem;/opt/rh/devtoolset-8/root/usr/include/c++/8/x86_64-redhat-linux/" \ + -DARROW_JEMALLOC=ON \ + -DARROW_RPATH_ORIGIN=ON \ + -DARROW_PYTHON=ON \ + -DARROW_PARQUET=ON \ + -DPythonInterp_FIND_VERSION=${PYTHON_VERSION} \ + -DARROW_PLASMA=ON \ + -DARROW_TENSORFLOW=ON \ + -DARROW_ORC=ON \ + -DARROW_WITH_BZ2=ON \ + -DARROW_FLIGHT=${BUILD_ARROW_FLIGHT} \ + -DARROW_GANDIVA=${BUILD_ARROW_GANDIVA} \ + -DARROW_GANDIVA_JAVA=OFF \ + -DBoost_NAMESPACE=arrow_boost \ + -DBOOST_ROOT=/arrow_boost_dist \ + -DOPENSSL_USE_STATIC_LIBS=ON \ + -DORC_SOURCE=BUNDLED \ + -GNinja /arrow/cpp +ninja install +popd + +# Check that we don't expose any unwanted symbols +/io/scripts/check_arrow_visibility.sh + +echo "=== (${PYTHON_VERSION}) Install the wheel build dependencies ===" +$PIP install -r requirements-wheel.txt + +# Clear output directories and leftovers +rm -rf dist/ +rm -rf build/ +rm -rf repaired_wheels/ +find -name "*.so" -delete + +echo "=== (${PYTHON_VERSION}) Building wheel ===" +PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER setup.py build_ext --inplace +PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER setup.py bdist_wheel +# Source distribution is used for debian pyarrow packages. +PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER setup.py sdist + +echo "=== (${PYTHON_VERSION}) Tag the wheel with manylinux2010 ===" +mkdir -p repaired_wheels/ +auditwheel repair --plat manylinux2010_x86_64 -L . dist/pyarrow-*.whl -w repaired_wheels/ + +# Install the built wheels +$PIP install repaired_wheels/*.whl + +# Test that the modules are importable +$PYTHON_INTERPRETER -c " +import sys +import pyarrow +import pyarrow.orc +import pyarrow.parquet +import pyarrow.plasma + +if sys.version_info.major > 2: + import pyarrow.flight + import pyarrow.gandiva +" + +# More thorough testing happens outsite of the build to prevent +# packaging issues like ARROW-4372 +mv dist/*.tar.gz /io/dist +mv repaired_wheels/*.whl /io/dist diff --git a/python/manylinux2010/scripts/build_boost.sh b/python/manylinux2010/scripts/build_boost.sh new file mode 100755 index 0000000000000..ef3aef25c11c8 --- /dev/null +++ b/python/manylinux2010/scripts/build_boost.sh @@ -0,0 +1,43 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +BOOST_VERSION=1.68.0 +BOOST_VERSION_UNDERSCORE=${BOOST_VERSION//\./_} +NCORES=$(($(grep -c ^processor /proc/cpuinfo) + 1)) + +curl -sL https://dl.bintray.com/boostorg/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_UNDERSCORE}.tar.gz -o /boost_${BOOST_VERSION_UNDERSCORE}.tar.gz +tar xf boost_${BOOST_VERSION_UNDERSCORE}.tar.gz +mkdir /arrow_boost +pushd /boost_${BOOST_VERSION_UNDERSCORE} +./bootstrap.sh +./b2 -j${NCORES} tools/bcp +./dist/bin/bcp --namespace=arrow_boost --namespace-alias filesystem date_time system regex build predef algorithm locale format variant multiprecision/cpp_int /arrow_boost +popd + +pushd /arrow_boost +ls -l +./bootstrap.sh +./bjam -j${NCORES} dll-path="'\$ORIGIN/'" cxxflags='-std=c++11 -fPIC' cflags=-fPIC linkflags="-std=c++11" variant=release link=shared --prefix=/arrow_boost_dist --with-filesystem --with-date_time --with-system --with-regex install +popd +rm -rf boost_${BOOST_VERSION_UNDERSCORE}.tar.gz boost_${BOOST_VERSION_UNDERSCORE} arrow_boost +# Boost always install header-only parts but they also take up quite some space. +# We don't need them in array, so don't persist them in the docker layer. +# fusion 16.7 MiB +rm -r /arrow_boost_dist/include/boost/fusion +# spirit 8.2 MiB +rm -r /arrow_boost_dist/include/boost/spirit diff --git a/python/manylinux2010/scripts/build_brotli.sh b/python/manylinux2010/scripts/build_brotli.sh new file mode 100755 index 0000000000000..58715ca62325f --- /dev/null +++ b/python/manylinux2010/scripts/build_brotli.sh @@ -0,0 +1,34 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +export BROTLI_VERSION="1.0.7" +curl -sL "https://github.com/google/brotli/archive/v${BROTLI_VERSION}.tar.gz" -o brotli-${BROTLI_VERSION}.tar.gz +tar xf brotli-${BROTLI_VERSION}.tar.gz +pushd brotli-${BROTLI_VERSION} +mkdir build +pushd build +cmake -DCMAKE_BUILD_TYPE=release \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DBUILD_SHARED_LIBS=OFF \ + -GNinja \ + .. +ninja install +popd +popd +rm -rf brotli-${BROTLI_VERSION}.tar.gz brotli-${BROTLI_VERSION} diff --git a/python/manylinux2010/scripts/build_bz2.sh b/python/manylinux2010/scripts/build_bz2.sh new file mode 100755 index 0000000000000..75ff7a2f8911d --- /dev/null +++ b/python/manylinux2010/scripts/build_bz2.sh @@ -0,0 +1,30 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +NCORES=$(($(grep -c ^processor /proc/cpuinfo) + 1)) +export BZ2_VERSION="1.0.6" +export CFLAGS="-Wall -Winline -O2 -fPIC -D_FILE_OFFSET_BITS=64" + +curl -sL "https://www.sourceware.org/pub/bzip2/bzip2-${BZ2_VERSION}.tar.gz" -o bzip2-${BZ2_VERSION}.tar.gz +tar xf bzip2-${BZ2_VERSION}.tar.gz + +pushd bzip2-${BZ2_VERSION} +make PREFIX=/usr/local CFLAGS="$CFLAGS" install -j${NCORES} +popd + +rm -rf bzip2-${BZ2_VERSION}.tar.gz bzip2-${BZ2_VERSION} diff --git a/python/manylinux2010/scripts/build_cares.sh b/python/manylinux2010/scripts/build_cares.sh new file mode 100755 index 0000000000000..a3897d7eb929a --- /dev/null +++ b/python/manylinux2010/scripts/build_cares.sh @@ -0,0 +1,34 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +export CARES_VERSION="1.15.0" +export CFLAGS="-fPIC" +export PREFIX="/usr/local" +curl -sL "https://c-ares.haxx.se/download/c-ares-$CARES_VERSION.tar.gz" -o c-ares-${CARES_VERSION}.tar.gz +tar xf c-ares-${CARES_VERSION}.tar.gz +pushd c-ares-${CARES_VERSION} + +cmake -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=${PREFIX} \ + -DCARES_STATIC=ON \ + -DCARES_SHARED=OFF \ + -DCMAKE_C_FLAGS=${CFLAGS} \ + -GNinja . +ninja install +popd +rm -rf c-ares-${CARES_VERSION}.tar.gz c-ares-${CARES_VERSION} diff --git a/python/manylinux2010/scripts/build_clang.sh b/python/manylinux2010/scripts/build_clang.sh new file mode 100755 index 0000000000000..970802963cdba --- /dev/null +++ b/python/manylinux2010/scripts/build_clang.sh @@ -0,0 +1,43 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source /multibuild/manylinux_utils.sh + +export LLVM_VERSION="7.0.1" +curl -sL http://releases.llvm.org/${LLVM_VERSION}/cfe-${LLVM_VERSION}.src.tar.xz -o cfe-${LLVM_VERSION}.src.tar.xz +unxz cfe-${LLVM_VERSION}.src.tar.xz +tar xf cfe-${LLVM_VERSION}.src.tar +pushd cfe-${LLVM_VERSION}.src +mkdir build +pushd build +cmake \ + -DCMAKE_BUILD_TYPE=Release \ + -DCLANG_INCLUDE_TESTS=OFF \ + -DCLANG_INCLUDE_DOCS=OFF \ + -DCLANG_ENABLE_ARCMT=OFF \ + -DCLANG_ENABLE_STATIC_ANALYZER=OFF \ + -DLLVM_ENABLE_RTTI=ON \ + -DLLVM_INSTALL_TOOLCHAIN_ONLY=ON \ + -DLLVM_INCLUDE_TESTS=OFF \ + -DLLVM_INCLUDE_DOCS=OFF \ + -GNinja \ + .. +ninja install +popd +popd +rm -rf cfe-${LLVM_VERSION}.src.tar.xz cfe-${LLVM_VERSION}.src.tar cfe-${LLVM_VERSION}.src diff --git a/python/manylinux2010/scripts/build_double_conversion.sh b/python/manylinux2010/scripts/build_double_conversion.sh new file mode 100755 index 0000000000000..bf4ea0773fd50 --- /dev/null +++ b/python/manylinux2010/scripts/build_double_conversion.sh @@ -0,0 +1,34 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +DC_VERSION=3.1.4 + +curl -sL https://github.com/google/double-conversion/archive/v${DC_VERSION}.tar.gz -o double-conversion-${DC_VERSION}.tar.gz +tar xf double-conversion-${DC_VERSION}.tar.gz +pushd double-conversion-${DC_VERSION} + +cmake . \ + -GNinja \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DBUILD_SHARED_LIBS=OFF \ + -DBUILD_TESTING=OFF +ninja install + +popd +rm -rf double-conversion-${DC_VERSION}.tar.gz double-conversion-${DC_VERSION} diff --git a/python/manylinux2010/scripts/build_flatbuffers.sh b/python/manylinux2010/scripts/build_flatbuffers.sh new file mode 100755 index 0000000000000..f5af7cc9e5ada --- /dev/null +++ b/python/manylinux2010/scripts/build_flatbuffers.sh @@ -0,0 +1,32 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +export FLATBUFFERS_VERSION=1.10.0 +curl -sL https://github.com/google/flatbuffers/archive/v${FLATBUFFERS_VERSION}.tar.gz \ + -o flatbuffers-${FLATBUFFERS_VERSION}.tar.gz +tar xf flatbuffers-${FLATBUFFERS_VERSION}.tar.gz +pushd flatbuffers-${FLATBUFFERS_VERSION} +cmake \ + -DCMAKE_CXX_FLAGS=-fPIC \ + -DCMAKE_INSTALL_PREFIX:PATH=/usr/local \ + -DFLATBUFFERS_BUILD_TESTS=OFF \ + -DCMAKE_BUILD_TYPE=Release \ + -GNinja . +ninja install +popd +rm -rf flatbuffers-${FLATBUFFERS_VERSION}.tar.gz flatbuffers-${FLATBUFFERS_VERSION} diff --git a/python/manylinux2010/scripts/build_gflags.sh b/python/manylinux2010/scripts/build_gflags.sh new file mode 100755 index 0000000000000..7f40437a83f84 --- /dev/null +++ b/python/manylinux2010/scripts/build_gflags.sh @@ -0,0 +1,38 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +export GFLAGS_VERSION="2.2.1" +export CFLAGS="-fPIC" +export CXXFLAGS="-fPIC" + +curl -sL "https://github.com/gflags/gflags/archive/v${GFLAGS_VERSION}.tar.gz" -o gflags-${GFLAGS_VERSION}.tar.gz +tar xf gflags-${GFLAGS_VERSION}.tar.gz +pushd gflags-${GFLAGS_VERSION} + +cmake . \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/usr \ + -DINSTALL_HEADERS=on \ + -DBUILD_SHARED_LIBS=off \ + -DBUILD_STATIC_LIBS=on \ + -DBUILD_TESTING=off \ + -GNinja + +ninja install +popd +rm -rf gflags-${GFLAGS_VERSION}.tar.gz gflags-${GFLAGS_VERSION} diff --git a/python/manylinux2010/scripts/build_glog.sh b/python/manylinux2010/scripts/build_glog.sh new file mode 100755 index 0000000000000..5e3b8c24ebeb7 --- /dev/null +++ b/python/manylinux2010/scripts/build_glog.sh @@ -0,0 +1,35 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +export GLOG_VERSION="0.3.5" +export PREFIX="/usr/local" +curl -sL "https://github.com/google/glog/archive/v${GLOG_VERSION}.tar.gz" -o glog-${GLOG_VERSION}.tar.gz +tar xf glog-${GLOG_VERSION}.tar.gz +pushd glog-${GLOG_VERSION} + +cmake -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=${PREFIX} \ + -DCMAKE_POSITION_INDEPENDENT_CODE=1 \ + -DBUILD_SHARED_LIBS=OFF \ + -DBUILD_TESTING=OFF \ + -DWITH_GFLAGS=OFF \ + -GNinja . +ninja install +popd +rm -rf glog-${GLOG_VERSION}.tar.gz glog-${GLOG_VERSION} + diff --git a/python/manylinux2010/scripts/build_grpc.sh b/python/manylinux2010/scripts/build_grpc.sh new file mode 100755 index 0000000000000..6b906862e5a54 --- /dev/null +++ b/python/manylinux2010/scripts/build_grpc.sh @@ -0,0 +1,41 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +export GRPC_VERSION="1.20.0" +export CFLAGS="-fPIC -DGPR_MANYLINUX1=1" +export PREFIX="/usr/local" + +curl -sL "https://github.com/grpc/grpc/archive/v${GRPC_VERSION}.tar.gz" -o grpc-${GRPC_VERSION}.tar.gz +tar xf grpc-${GRPC_VERSION}.tar.gz +pushd grpc-${GRPC_VERSION} + +cmake -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=${PREFIX} \ + -DBUILD_SHARED_LIBS=OFF \ + -DCMAKE_C_FLAGS="${CFLAGS}" \ + -DCMAKE_CXX_FLAGS="${CFLAGS}" \ + -DgRPC_CARES_PROVIDER=package \ + -DgRPC_GFLAGS_PROVIDER=package \ + -DgRPC_PROTOBUF_PROVIDER=package \ + -DgRPC_SSL_PROVIDER=package \ + -DgRPC_ZLIB_PROVIDER=package \ + -DOPENSSL_USE_STATIC_LIBS=ON \ + -GNinja . +ninja install +popd +rm -rf grpc-${GRPC_VERSION}.tar.gz grpc-${GRPC_VERSION} diff --git a/python/manylinux2010/scripts/build_llvm.sh b/python/manylinux2010/scripts/build_llvm.sh new file mode 100755 index 0000000000000..1cc66455a351f --- /dev/null +++ b/python/manylinux2010/scripts/build_llvm.sh @@ -0,0 +1,48 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source /multibuild/manylinux_utils.sh + +LLVM_VERSION="7.0.1" +PREFIX="/usr/local" + +curl -sL http://releases.llvm.org/${LLVM_VERSION}/llvm-${LLVM_VERSION}.src.tar.xz -o llvm-${LLVM_VERSION}.src.tar.xz +unxz llvm-${LLVM_VERSION}.src.tar.xz +tar xf llvm-${LLVM_VERSION}.src.tar +pushd llvm-${LLVM_VERSION}.src +mkdir build +pushd build +cmake -DCMAKE_INSTALL_PREFIX=$PREFIX \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_TARGETS_TO_BUILD=host \ + -DLLVM_INCLUDE_DOCS=OFF \ + -DLLVM_INCLUDE_EXAMPLES=OFF \ + -DLLVM_INCLUDE_TESTS=OFF \ + -DLLVM_INCLUDE_UTILS=OFF \ + -DLLVM_ENABLE_TERMINFO=OFF \ + -DLLVM_ENABLE_ASSERTIONS=ON \ + -DLLVM_ENABLE_RTTI=ON \ + -DLLVM_ENABLE_OCAMLDOC=OFF \ + -DLLVM_USE_INTEL_JITEVENTS=ON \ + -DPYTHON_EXECUTABLE="$(cpython_path 2.7 32)/bin/python" \ + -GNinja \ + .. +ninja install +popd +popd +rm -rf llvm-${LLVM_VERSION}.src.tar.xz llvm-${LLVM_VERSION}.src.tar llvm-${LLVM_VERSION}.src diff --git a/python/manylinux2010/scripts/build_lz4.sh b/python/manylinux2010/scripts/build_lz4.sh new file mode 100755 index 0000000000000..7b3d4416dba50 --- /dev/null +++ b/python/manylinux2010/scripts/build_lz4.sh @@ -0,0 +1,33 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +NCORES=$(($(grep -c ^processor /proc/cpuinfo) + 1)) +export LZ4_VERSION="1.8.3" +export PREFIX="/usr/local" +export CFLAGS="${CFLAGS} -O3 -fPIC" +export LDFLAGS="${LDFLAGS} -Wl,-rpath,${PREFIX}/lib -L${PREFIX}/lib" +curl -sL "https://github.com/lz4/lz4/archive/v${LZ4_VERSION}.tar.gz" -o lz4-${LZ4_VERSION}.tar.gz +tar xf lz4-${LZ4_VERSION}.tar.gz +pushd lz4-${LZ4_VERSION} + +make -j$NCORES PREFIX=${PREFIX} +make install PREFIX=${PREFIX} +popd +rm -rf lz4-${LZ4_VERSION}.tar.gz lz4-${LZ4_VERSION} +# We don't want to link against shared libs +rm -rf /usr/lib/liblz4.so* diff --git a/python/manylinux2010/scripts/build_openssl.sh b/python/manylinux2010/scripts/build_openssl.sh new file mode 100755 index 0000000000000..d9ce0a86c409f --- /dev/null +++ b/python/manylinux2010/scripts/build_openssl.sh @@ -0,0 +1,32 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# XXX OpenSSL 1.1.1 needs Perl 5.10 to compile. +OPENSSL_VERSION="1.1.1c" +NCORES=$(($(grep -c ^processor /proc/cpuinfo) + 1)) + +wget --no-check-certificate https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz -O openssl-${OPENSSL_VERSION}.tar.gz +tar xf openssl-${OPENSSL_VERSION}.tar.gz + +pushd openssl-${OPENSSL_VERSION} +./config -fpic no-shared no-tests --prefix=/usr/local +make -j${NCORES} +make install_sw +popd + +rm -rf openssl-${OPENSSL_VERSION}.tar.gz openssl-${OPENSSL_VERSION} diff --git a/python/manylinux2010/scripts/build_protobuf.sh b/python/manylinux2010/scripts/build_protobuf.sh new file mode 100755 index 0000000000000..abb31c2f700f9 --- /dev/null +++ b/python/manylinux2010/scripts/build_protobuf.sh @@ -0,0 +1,29 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +NCORES=$(($(grep -c ^processor /proc/cpuinfo) + 1)) +PROTOBUF_VERSION="3.7.1" + +curl -sL "https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-all-${PROTOBUF_VERSION}.tar.gz" -o protobuf-${PROTOBUF_VERSION}.tar.gz +tar xf protobuf-${PROTOBUF_VERSION}.tar.gz +pushd protobuf-${PROTOBUF_VERSION} +./configure --disable-shared --prefix=/usr/local "CXXFLAGS=-O2 -fPIC" +make -j$NCORES +make install +popd +rm -rf protobuf-${PROTOBUF_VERSION}.tar.gz protobuf-${PROTOBUF_VERSION} diff --git a/python/manylinux2010/scripts/build_rapidjson.sh b/python/manylinux2010/scripts/build_rapidjson.sh new file mode 100755 index 0000000000000..4a93bd4276535 --- /dev/null +++ b/python/manylinux2010/scripts/build_rapidjson.sh @@ -0,0 +1,37 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +export RAPIDJSON_VERSION="1.1.0" + +curl -sL "https://github.com/miloyip/rapidjson/archive/v${RAPIDJSON_VERSION}.tar.gz" -o rapidjson-${RAPIDJSON_VERSION}.tar.gz +tar xf rapidjson-${RAPIDJSON_VERSION}.tar.gz +pushd rapidjson-${RAPIDJSON_VERSION} +mkdir build +pushd build +cmake -GNinja \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DRAPIDJSON_HAS_STDSTRING=ON \ + -DRAPIDJSON_BUILD_TESTS=OFF \ + -DRAPIDJSON_BUILD_EXAMPLES=OFF \ + -DRAPIDJSON_BUILD_DOC=OFF \ + -DCMAKE_BUILD_TYPE=release \ + .. +ninja install +popd +popd +rm -rf rapidjson-${RAPIDJSON_VERSION}.tar.gz rapidjson-${RAPIDJSON_VERSION} diff --git a/python/manylinux2010/scripts/build_re2.sh b/python/manylinux2010/scripts/build_re2.sh new file mode 100755 index 0000000000000..143d8dccb1bdb --- /dev/null +++ b/python/manylinux2010/scripts/build_re2.sh @@ -0,0 +1,35 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +export RE2_VERSION="2019-04-01" +NCORES=$(($(grep -c ^processor /proc/cpuinfo) + 1)) + +curl -sL "http://github.com/google/re2/archive/${RE2_VERSION}.tar.gz" -o re2-${RE2_VERSION}.tar.gz +tar xf re2-${RE2_VERSION}.tar.gz +pushd re2-${RE2_VERSION} + +export CXXFLAGS="-fPIC -O2 ${CXXFLAGS}" +export CFLAGS="-fPIC -O2 ${CFLAGS}" + +# Build shared libraries +make prefix=/usr/local -j${NCORES} install + +popd + +# Need to remove shared library to make sure the static library is picked up by Arrow +rm -rf re2-${RE2_VERSION}.tar.gz re2-${RE2_VERSION} /usr/local/lib/libre2.so* diff --git a/python/manylinux2010/scripts/build_snappy.sh b/python/manylinux2010/scripts/build_snappy.sh new file mode 100755 index 0000000000000..a67ac7fd7a177 --- /dev/null +++ b/python/manylinux2010/scripts/build_snappy.sh @@ -0,0 +1,31 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +export SNAPPY_VERSION="1.1.7" +curl -sL "https://github.com/google/snappy/archive/${SNAPPY_VERSION}.tar.gz" -o snappy-${SNAPPY_VERSION}.tar.gz +tar xf snappy-${SNAPPY_VERSION}.tar.gz +pushd snappy-${SNAPPY_VERSION} +CXXFLAGS='-DNDEBUG -O2' cmake -GNinja \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DCMAKE_POSITION_INDEPENDENT_CODE=1 \ + -DBUILD_SHARED_LIBS=OFF \ + -DSNAPPY_BUILD_TESTS=OFF \ + . +ninja install +popd +rm -rf snappy-${SNAPPY_VERSION}.tar.gz snappy-${SNAPPY_VERSION} diff --git a/python/manylinux2010/scripts/build_thrift.sh b/python/manylinux2010/scripts/build_thrift.sh new file mode 100755 index 0000000000000..3a0b9e807746a --- /dev/null +++ b/python/manylinux2010/scripts/build_thrift.sh @@ -0,0 +1,46 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +export THRIFT_VERSION=0.12.0 +wget http://archive.apache.org/dist/thrift/${THRIFT_VERSION}/thrift-${THRIFT_VERSION}.tar.gz +tar xf thrift-${THRIFT_VERSION}.tar.gz +pushd thrift-${THRIFT_VERSION} +mkdir build-tmp +pushd build-tmp +cmake -DCMAKE_BUILD_TYPE=release \ + -DCMAKE_CXX_FLAGS=-fPIC \ + -DCMAKE_C_FLAGS=-fPIC \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DCMAKE_INSTALL_RPATH=/usr/local/lib \ + -DBUILD_EXAMPLES=OFF \ + -DBUILD_TESTING=OFF \ + -DWITH_QT4=OFF \ + -DWITH_C_GLIB=OFF \ + -DWITH_JAVA=OFF \ + -DWITH_PYTHON=OFF \ + -DWITH_HASKELL=OFF \ + -DWITH_CPP=ON \ + -DWITH_STATIC_LIB=ON \ + -DWITH_SHARED_LIB=OFF \ + -DBoost_NAMESPACE=arrow_boost \ + -DBOOST_ROOT=/arrow_boost_dist \ + -GNinja .. +ninja install +popd +popd +rm -rf thrift-${THRIFT_VERSION}.tar.gz thrift-${THRIFT_VERSION} diff --git a/python/manylinux2010/scripts/build_uriparser.sh b/python/manylinux2010/scripts/build_uriparser.sh new file mode 100755 index 0000000000000..840df5ad57adc --- /dev/null +++ b/python/manylinux2010/scripts/build_uriparser.sh @@ -0,0 +1,37 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +export URIPARSER_VERSION="0.9.2" +export CFLAGS="-fPIC" +export PREFIX="/usr/local" +curl -sL "https://github.com/uriparser/uriparser/archive/uriparser-${URIPARSER_VERSION}.tar.gz" -o uriparser-${URIPARSER_VERSION}.tar.gz +tar xf uriparser-${URIPARSER_VERSION}.tar.gz +pushd uriparser-uriparser-${URIPARSER_VERSION} + +cmake -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=${PREFIX} \ + -DURIPARSER_BUILD_DOCS=off \ + -DURIPARSER_BUILD_TESTS=off \ + -DURIPARSER_BUILD_TOOLS=off \ + -DURIPARSER_BUILD_WCHAR_T=off \ + -DBUILD_SHARED_LIBS=off \ + -DCMAKE_POSITION_INDEPENDENT_CODE=on \ + -GNinja . +ninja install +popd +rm -rf uriparser-${URIPARSER_VERSION}.tar.gz uriparser-uriparser-${URIPARSER_VERSION} diff --git a/python/manylinux2010/scripts/build_virtualenvs.sh b/python/manylinux2010/scripts/build_virtualenvs.sh new file mode 100755 index 0000000000000..67ec961678e9d --- /dev/null +++ b/python/manylinux2010/scripts/build_virtualenvs.sh @@ -0,0 +1,72 @@ +#!/bin/bash -e +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Build upon the scripts in https://github.com/matthew-brett/manylinux-builds +# * Copyright (c) 2013-2016, Matt Terry and Matthew Brett (BSD 2-clause) + +PYTHON_VERSIONS="${PYTHON_VERSIONS:-2.7,16 2.7,32 3.5,16 3.6,16, 3.7,16}" + +source /multibuild/manylinux_utils.sh + +for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do + IFS="," + set -- $PYTHON_TUPLE; + PYTHON=$1 + U_WIDTH=$2 + PYTHON_INTERPRETER="$(cpython_path $PYTHON ${U_WIDTH})/bin/python" + PIP="$(cpython_path $PYTHON ${U_WIDTH})/bin/pip" + PATH="$PATH:$(cpython_path $PYTHON ${U_WIDTH})" + + echo "=== (${PYTHON}, ${U_WIDTH}) Installing build dependencies ===" + $PIP install "numpy==1.14.*" "cython==0.29.8" "virtualenv==16.3.0" + # Pandas requires numpy and cython + $PIP install "pandas==0.24.0" + + # TensorFlow is not supported for Python 2.7 with unicode width 16 or with Python 3.7 + if [ $PYTHON != "2.7" ] || [ $U_WIDTH = "32" ]; then + if [ $PYTHON != "3.7" ]; then + $PIP install "tensorflow==1.11.0" "Keras-Preprocessing==1.0.5" + fi + fi + + echo "=== (${PYTHON}, ${U_WIDTH}) Preparing virtualenv for tests ===" + "$(cpython_path $PYTHON ${U_WIDTH})/bin/virtualenv" -p ${PYTHON_INTERPRETER} --no-download /venv-test-${PYTHON}-${U_WIDTH} + source /venv-test-${PYTHON}-${U_WIDTH}/bin/activate + pip install pytest hypothesis "numpy==1.14.*" "pandas==0.24.0" + deactivate +done + +# Remove debug symbols from libraries that were installed via wheel. +find /venv-test-*/lib/*/site-packages/pandas -name '*.so' -exec strip '{}' ';' +find /venv-test-*/lib/*/site-packages/numpy -name '*.so' -exec strip '{}' ';' +find /opt/_internal/cpython-*/lib/*/site-packages/pandas -name '*.so' -exec strip '{}' ';' +find /opt/_internal/cpython-*/lib/*/site-packages/tensorflow -name '*.so' -exec strip '{}' ';' +# Only Python 3.6+ packages are stripable as they are built inside of the image +find /opt/_internal/cpython-3.6.*/lib/python3.6/site-packages/numpy -name '*.so' -exec strip '{}' ';' +find /opt/_internal/cpython-3.7.*/lib/python3.7/site-packages/numpy -name '*.so' -exec strip '{}' ';' +find /opt/_internal/*/lib/*/site-packages/Cython -name '*.so' -exec strip '{}' ';' + +# Remove pip cache again. It's useful during the virtualenv creation but we +# don't want it persisted in the docker layer, ~264MiB +rm -rf /root/.cache +# Remove pandas' tests module as it includes a lot of data, ~27MiB per Python +# venv, i.e. 216MiB in total +rm -rf /opt/_internal/*/lib/*/site-packages/pandas/tests +rm -rf /venv-test-*/lib/*/site-packages/pandas/tests +# Remove unused Python versions +rm -rf /opt/_internal/cpython-3.4* diff --git a/python/manylinux2010/scripts/build_zlib.sh b/python/manylinux2010/scripts/build_zlib.sh new file mode 100755 index 0000000000000..272b6c4b2b0bb --- /dev/null +++ b/python/manylinux2010/scripts/build_zlib.sh @@ -0,0 +1,26 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +curl -sL https://zlib.net/zlib-1.2.11.tar.gz -o /zlib-1.2.11.tar.gz +tar xf zlib-1.2.11.tar.gz +pushd zlib-1.2.11 +./configure +make -j8 +make install +popd +rm -rf zlib-1.2.11.tar.gz zlib-1.2.11 diff --git a/python/manylinux2010/scripts/build_zstd.sh b/python/manylinux2010/scripts/build_zstd.sh new file mode 100755 index 0000000000000..d6b0b09320f2d --- /dev/null +++ b/python/manylinux2010/scripts/build_zstd.sh @@ -0,0 +1,39 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +export ZSTD_VERSION="1.4.0" + +curl -sL "https://github.com/facebook/zstd/archive/v${ZSTD_VERSION}.tar.gz" -o zstd-${ZSTD_VERSION}.tar.gz +tar xf zstd-${ZSTD_VERSION}.tar.gz +pushd zstd-${ZSTD_VERSION} +mkdir build_cmake +pushd build_cmake + +cmake -GNinja -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DZSTD_BUILD_PROGRAMS=off \ + -DZSTD_BUILD_SHARED=off \ + -DZSTD_BUILD_STATIC=on \ + -DZSTD_MULTITHREAD_SUPPORT=off \ + -DCMAKE_POSITION_INDEPENDENT_CODE=1 \ + ../build/cmake +ninja install + +popd +popd +rm -rf zstd-${ZSTD_VERSION}.tar.gz zstd-${ZSTD_VERSION} diff --git a/python/manylinux2010/scripts/check_arrow_visibility.sh b/python/manylinux2010/scripts/check_arrow_visibility.sh new file mode 100755 index 0000000000000..ed464e0779a0e --- /dev/null +++ b/python/manylinux2010/scripts/check_arrow_visibility.sh @@ -0,0 +1,35 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +nm --demangle --dynamic /arrow-dist/lib/libarrow.so > nm_arrow.log + +# Filter out Arrow symbols and see if anything remains. +# '_init' and '_fini' symbols may or not be present, we don't care. +# (note we must ignore the grep exit status when no match is found) +grep ' T ' nm_arrow.log | grep -v -E '(arrow|\b_init\b|\b_fini\b)' | cat - > visible_symbols.log + +if [[ -f visible_symbols.log && `cat visible_symbols.log | wc -l` -eq 0 ]] +then + exit 0 +fi + +echo "== Unexpected symbols exported by libarrow.so ==" +cat visible_symbols.log +echo "================================================" + +exit 1 diff --git a/python/manylinux2010/scripts/install_cmake.sh b/python/manylinux2010/scripts/install_cmake.sh new file mode 100755 index 0000000000000..637ea05340eff --- /dev/null +++ b/python/manylinux2010/scripts/install_cmake.sh @@ -0,0 +1,22 @@ +#!/bin/bash -e +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +/opt/python/cp37-cp37m/bin/pip install cmake ninja +ln -s /opt/python/cp37-cp37m/bin/cmake /usr/local/bin/cmake +ln -s /opt/python/cp37-cp37m/bin/ninja /usr/local/bin/ninja +strip /opt/_internal/cpython-3.*/lib/python3.7/site-packages/cmake/data/bin/*