From 665aa23506aa4cd8739f531ccf78e8480e8901e0 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Mon, 27 May 2019 12:09:20 +0200 Subject: [PATCH] ARROW-2461: [Python] Build manylinux2010 wheels --- docker-compose.yml | 22 ++ python/manylinux2010/.dockerignore | 1 + python/manylinux2010/Dockerfile-x86_64_base | 99 ++++++++ python/manylinux2010/README.md | 117 ++++++++++ python/manylinux2010/build_arrow.sh | 143 ++++++++++++ python/manylinux2010/scripts/build_bison.sh | 26 +++ python/manylinux2010/scripts/build_boost.sh | 43 ++++ python/manylinux2010/scripts/build_brotli.sh | 35 +++ python/manylinux2010/scripts/build_ccache.sh | 29 +++ python/manylinux2010/scripts/build_clang.sh | 39 ++++ .../scripts/build_double_conversion.sh | 29 +++ .../scripts/build_flatbuffers.sh | 32 +++ python/manylinux2010/scripts/build_gflags.sh | 38 +++ python/manylinux2010/scripts/build_glog.sh | 36 +++ python/manylinux2010/scripts/build_gtest.sh | 39 ++++ python/manylinux2010/scripts/build_llvm.sh | 42 ++++ python/manylinux2010/scripts/build_lz4.sh | 32 +++ python/manylinux2010/scripts/build_openssl.sh | 29 +++ .../manylinux2010/scripts/build_protobuf.sh | 28 +++ python/manylinux2010/scripts/build_python.sh | 218 ++++++++++++++++++ .../manylinux2010/scripts/build_rapidjson.sh | 37 +++ python/manylinux2010/scripts/build_re2.sh | 32 +++ python/manylinux2010/scripts/build_snappy.sh | 26 +++ python/manylinux2010/scripts/build_thrift.sh | 45 ++++ .../scripts/build_virtualenvs.sh | 70 ++++++ python/manylinux2010/scripts/build_zlib.sh | 26 +++ python/manylinux2010/scripts/build_zstd.sh | 32 +++ .../scripts/check_arrow_visibility.sh | 35 +++ python/manylinux2010/scripts/install_cmake.sh | 23 ++ .../scripts/python-tag-abi-tag.py | 30 +++ python/manylinux2010/scripts/requirements.txt | 34 +++ 31 files changed, 1467 insertions(+) create mode 100644 python/manylinux2010/.dockerignore create mode 100644 python/manylinux2010/Dockerfile-x86_64_base create mode 100644 python/manylinux2010/README.md create mode 100755 python/manylinux2010/build_arrow.sh create mode 100755 python/manylinux2010/scripts/build_bison.sh create mode 100755 python/manylinux2010/scripts/build_boost.sh create mode 100755 python/manylinux2010/scripts/build_brotli.sh create mode 100755 python/manylinux2010/scripts/build_ccache.sh create mode 100755 python/manylinux2010/scripts/build_clang.sh create mode 100755 python/manylinux2010/scripts/build_double_conversion.sh create mode 100755 python/manylinux2010/scripts/build_flatbuffers.sh create mode 100755 python/manylinux2010/scripts/build_gflags.sh create mode 100755 python/manylinux2010/scripts/build_glog.sh create mode 100755 python/manylinux2010/scripts/build_gtest.sh create mode 100755 python/manylinux2010/scripts/build_llvm.sh create mode 100755 python/manylinux2010/scripts/build_lz4.sh create mode 100755 python/manylinux2010/scripts/build_openssl.sh create mode 100755 python/manylinux2010/scripts/build_protobuf.sh create mode 100755 python/manylinux2010/scripts/build_python.sh create mode 100755 python/manylinux2010/scripts/build_rapidjson.sh create mode 100755 python/manylinux2010/scripts/build_re2.sh create mode 100755 python/manylinux2010/scripts/build_snappy.sh create mode 100755 python/manylinux2010/scripts/build_thrift.sh create mode 100755 python/manylinux2010/scripts/build_virtualenvs.sh create mode 100755 python/manylinux2010/scripts/build_zlib.sh create mode 100755 python/manylinux2010/scripts/build_zstd.sh create mode 100755 python/manylinux2010/scripts/check_arrow_visibility.sh create mode 100755 python/manylinux2010/scripts/install_cmake.sh create mode 100644 python/manylinux2010/scripts/python-tag-abi-tag.py create mode 100644 python/manylinux2010/scripts/requirements.txt diff --git a/docker-compose.yml b/docker-compose.yml index 361eeb97b1b1d..1caac6d080832 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -389,6 +389,28 @@ services: - ./python/manylinux1:/io:delegated command: /io/build_arrow.sh + python-manylinux2010: + # Usage: + # either build: + # $ docker-compose build python-manylinux2010 + # or pull: + # $ docker-compose pull python-manylinux2010 + # an then run: + # $ docker-compose run -e PYTHON_VERSION=3.7 python-manylinux1 + image: quay.io/xhochy/arrow_manylinux2010_x86_64_base:latest + build: + context: python/manylinux2010 + dockerfile: Dockerfile-x86_64_base + shm_size: 2G + environment: + PYARROW_PARALLEL: 3 + PYTHON_VERSION: ${PYTHON_VERSION:-3.6} + UNICODE_WIDTH: ${UNICODE_WIDTH:-16} + volumes: + - .:/arrow:delegated + - ./python/manylinux2010:/io:delegated + command: /io/build_arrow.sh + ######################### Integration Tests ################################# # impala: diff --git a/python/manylinux2010/.dockerignore b/python/manylinux2010/.dockerignore new file mode 100644 index 0000000000000..1521c8b7652b1 --- /dev/null +++ b/python/manylinux2010/.dockerignore @@ -0,0 +1 @@ +dist diff --git a/python/manylinux2010/Dockerfile-x86_64_base b/python/manylinux2010/Dockerfile-x86_64_base new file mode 100644 index 0000000000000..82a1a50cb237c --- /dev/null +++ b/python/manylinux2010/Dockerfile-x86_64_base @@ -0,0 +1,99 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +FROM quay.io/pypa/manylinux2010_x86_64:latest + +# Install dependencies +RUN yum install -y xz ccache flex wget && yum clean all + +ADD scripts/build_zlib.sh / +RUN /build_zlib.sh + +ADD scripts/build_openssl.sh / +RUN /build_openssl.sh + +ADD scripts/build_boost.sh / +RUN /build_boost.sh + +# Install cmake manylinux1 package +ADD scripts/install_cmake.sh / +RUN /install_cmake.sh + +ADD scripts/build_gtest.sh / +RUN /build_gtest.sh +ENV GTEST_HOME /usr + +ADD scripts/build_flatbuffers.sh / +RUN /build_flatbuffers.sh +ENV FLATBUFFERS_HOME /usr + +ADD scripts/build_bison.sh / +RUN /build_bison.sh + +ADD scripts/build_thrift.sh / +RUN /build_thrift.sh +ENV THRIFT_HOME /usr + +ADD scripts/build_brotli.sh / +RUN /build_brotli.sh +ENV BROTLI_HOME /usr + +ADD scripts/build_snappy.sh / +RUN /build_snappy.sh +ENV SNAPPY_HOME /usr + +ADD scripts/build_lz4.sh / +RUN /build_lz4.sh +ENV LZ4_HOME /usr + +ADD scripts/build_zstd.sh / +RUN /build_zstd.sh +ENV ZSTD_HOME /usr + +ADD scripts/build_ccache.sh / +RUN /build_ccache.sh + +ADD scripts/build_protobuf.sh / +RUN /build_protobuf.sh +ENV PROTOBUF_HOME /usr + +ADD scripts/build_glog.sh / +RUN /build_glog.sh +ENV GLOG_HOME /usr + +WORKDIR / +RUN git clone https://github.com/matthew-brett/multibuild.git && cd multibuild && git checkout ffe59955ad8690c2f8bb74766cb7e9b0d0ee3963 + +ADD scripts/build_virtualenvs.sh / +RUN /build_virtualenvs.sh + +ADD scripts/build_llvm.sh / +RUN /build_llvm.sh + +ADD scripts/build_clang.sh / +RUN /build_clang.sh + +ADD scripts/build_double_conversion.sh / +RUN /build_double_conversion.sh + +ADD scripts/build_rapidjson.sh / +RUN /build_rapidjson.sh + +ADD scripts/build_re2.sh / +RUN /build_re2.sh + +ADD scripts/build_gflags.sh / +RUN /build_gflags.sh diff --git a/python/manylinux2010/README.md b/python/manylinux2010/README.md new file mode 100644 index 0000000000000..3fcc4f7e15d92 --- /dev/null +++ b/python/manylinux2010/README.md @@ -0,0 +1,117 @@ + + +## Manylinux1 wheels for Apache Arrow + +This folder provides base Docker images and an infrastructure to build +`manylinux1` compatible Python wheels that should be installable on all +Linux distributions published in last four years. + +The process is split up in two parts: There are base Docker images that build +the native, Python-indenpendent dependencies. For these you can select if you +want to also build the dependencies used for the Parquet support. Depending on +these images, there is also a bash script that will build the pyarrow wheels +for all supported Python versions and place them in the `dist` folder. + +### Build instructions + +You can build the wheels with the following +command (this is for Python 2.7 with unicode width 16, similarly you can pass +in `PYTHON_VERSION="3.5"`, `PYTHON_VERSION="3.6"` or `PYTHON_VERSION="3.7"` or +use `PYTHON_VERSION="2.7"` with `UNICODE_WIDTH=32`): + +```bash +# Build the python packages +docker run --env PYTHON_VERSION="2.7" --env UNICODE_WIDTH=16 --shm-size=2g --rm -t -i -v $PWD:/io -v $PWD/../../:/arrow quay.io/xhochy/arrow_manylinux1_x86_64_base:latest /io/build_arrow.sh +# Now the new packages are located in the dist/ folder +ls -l dist/ +``` + +### Updating the build environment +The base docker image is less often updated. In the case we want to update +a dependency to a new version, we also need to adjust it. You can rebuild +this image using + +```bash +docker build -t arrow_manylinux1_x86_64_base -f Dockerfile-x86_64_base . +``` + +For each dependency, we have a bash script in the directory `scripts/` that +downloads the sources, builds and installs them. At the end of each dependency +build the sources are removed again so that only the binary installation of a +dependency is persisted in the docker image. When you do local adjustments to +this image, you need to change the name of the docker image in the `docker run` +command. + +### Using quay.io to trigger and build the docker image + +1. Make the change in the build scripts (eg. to modify the boost build, update `scripts/boost.sh`). + +2. Setup an account on quay.io and link to your GitHub account + +3. In quay.io, Add a new repository using : + + 1. Link to GitHub repository push + 2. Trigger build on changes to a specific branch (eg. myquay) of the repo (eg. `pravindra/arrow`) + 3. Set Dockerfile location to `/python/manylinux1/Dockerfile-x86_64_base` + 4. Set Context location to `/python/manylinux1` + +4. Push change (in step 1) to the branch specified in step 3.ii + + * This should trigger a build in quay.io, the build takes about 2 hrs to finish. + +5. Add a tag `latest` to the build after step 4 finishes, save the build ID (eg. `quay.io/pravindra/arrow_manylinux1_x86_64_base:latest`) + +6. In your arrow PR, + + * include the change from 1. + * modify `travis_script_manylinux.sh` to switch to the location from step 5 for the docker image. + +## TensorFlow compatible wheels for Arrow + +As TensorFlow is not compatible with the manylinux1 standard, the above +wheels can cause segfaults if they are used together with the TensorFlow wheels +from https://www.tensorflow.org/install/pip. We do not recommend using +TensorFlow wheels with pyarrow manylinux1 wheels until these incompatibilities +are addressed by the TensorFlow team [1]. For most end-users, the recommended +way to use Arrow together with TensorFlow is through conda. +If this is not an option for you, there is also a way to produce TensorFlow +compatible Arrow wheels that however do not conform to the manylinux1 standard +and are not officially supported by the Arrow community. + +Similar to the manylinux1 wheels, there is a base image that can be built with + +```bash +docker build -t arrow_linux_x86_64_base -f Dockerfile-x86_64_ubuntu . +``` + +Once the image has been built, you can then build the wheels with the following +command (this is for Python 2.7 with unicode width 16, similarly you can pass +in `PYTHON_VERSION="3.5"`, `PYTHON_VERSION="3.6"` or `PYTHON_VERSION="3.7"` or +use `PYTHON_VERSION="2.7"` with `UNICODE_WIDTH=32`) + +```bash +# Build the python packages +sudo docker run --env UBUNTU_WHEELS=1 --env PYTHON_VERSION="2.7" --env UNICODE_WIDTH=16 --rm -t -i -v $PWD:/io -v $PWD/../../:/arrow arrow_linux_x86_64_base:latest /io/build_arrow.sh +# Now the new packages are located in the dist/ folder +ls -l dist/ +echo "Please note that these wheels are not manylinux1 compliant" +``` + +[1] https://groups.google.com/a/tensorflow.org/d/topic/developers/TMqRaT-H2bI/discussion diff --git a/python/manylinux2010/build_arrow.sh b/python/manylinux2010/build_arrow.sh new file mode 100755 index 0000000000000..50b0720784ac2 --- /dev/null +++ b/python/manylinux2010/build_arrow.sh @@ -0,0 +1,143 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Usage: +# docker run --rm -v $PWD:/io arrow-base-x86_64 /io/build_arrow.sh + +# Build upon the scripts in https://github.com/matthew-brett/manylinux-builds +# * Copyright (c) 2013-2016, Matt Terry and Matthew Brett (BSD 2-clause) + +source /multibuild/manylinux_utils.sh + +# Quit on failure +set -e + +# Print commands for debugging +set -x + +cd /arrow/python + +# PyArrow build configuration +export PYARROW_BUILD_TYPE='release' +export PYARROW_CMAKE_GENERATOR='Ninja' +export PYARROW_WITH_ORC=1 +export PYARROW_WITH_PARQUET=1 +export PYARROW_WITH_PLASMA=1 +export PYARROW_BUNDLE_ARROW_CPP=1 +export PYARROW_BUNDLE_BOOST=1 +export PYARROW_BOOST_NAMESPACE=arrow_boost +export PKG_CONFIG_PATH=/usr/lib/pkgconfig:/arrow-dist/lib/pkgconfig + +export PYARROW_CMAKE_OPTIONS='-DTHRIFT_HOME=/usr -DBoost_NAMESPACE=arrow_boost -DBOOST_ROOT=/arrow_boost_dist' +# Ensure the target directory exists +mkdir -p /io/dist + +# Must pass PYTHON_VERSION and UNICODE_WIDTH env variables +# possible values are: 2.7,16 2.7,32 3.5,16 3.6,16 3.7,16 + +CPYTHON_PATH="$(cpython_path ${PYTHON_VERSION} ${UNICODE_WIDTH})" +PYTHON_INTERPRETER="${CPYTHON_PATH}/bin/python" +PIP="${CPYTHON_PATH}/bin/pip" +PATH="${PATH}:${CPYTHON_PATH}" + +#if [ "${PYTHON_VERSION}" != "2.7" ]; then +# # Gandiva is not supported on Python 2.7 +# export PYARROW_WITH_GANDIVA=1 +# export BUILD_ARROW_GANDIVA=ON +#else + export PYARROW_WITH_GANDIVA=0 + export BUILD_ARROW_GANDIVA=OFF +#fi + +echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ===" +ARROW_BUILD_DIR=/tmp/build-PY${PYTHON_VERSION}-${UNICODE_WIDTH} +mkdir -p "${ARROW_BUILD_DIR}" +pushd "${ARROW_BUILD_DIR}" +#-DARROW_GANDIVA_PC_CXX_FLAGS="-isystem;/opt/rh/devtoolset-8/root/usr/include/c++/8/" \ +PATH="${CPYTHON_PATH}/bin:${PATH}" cmake -DCMAKE_BUILD_TYPE=Release \ + -DARROW_DEPENDENCY_SOURCE="SYSTEM" \ + -DZLIB_ROOT=/usr/local \ + -DCMAKE_INSTALL_PREFIX=/arrow-dist \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DARROW_BUILD_TESTS=OFF \ + -DARROW_BUILD_SHARED=ON \ + -DARROW_BOOST_USE_SHARED=ON \ + -DARROW_JEMALLOC=ON \ + -DARROW_RPATH_ORIGIN=ON \ + -DARROW_PYTHON=ON \ + -DARROW_PARQUET=ON \ + -DPythonInterp_FIND_VERSION=${PYTHON_VERSION} \ + -DARROW_PLASMA=ON \ + -DARROW_TENSORFLOW=ON \ + -DARROW_ORC=ON \ + -DARROW_GANDIVA=${BUILD_ARROW_GANDIVA} \ + -DARROW_GANDIVA_JAVA=OFF \ + -DBoost_NAMESPACE=arrow_boost \ + -DBOOST_ROOT=/arrow_boost_dist \ + -GNinja /arrow/cpp +ninja install +popd + +# Check that we don't expose any unwanted symbols +/io/scripts/check_arrow_visibility.sh + +echo "=== (${PYTHON_VERSION}) Install the wheel build dependencies ===" +$PIP install -r requirements-wheel.txt + +# Clear output directory +rm -rf dist/ +echo "=== (${PYTHON_VERSION}) Building wheel ===" +# Remove build directory to ensure CMake gets a clean run +rm -rf build/ +PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER setup.py build_ext \ + --inplace \ + --bundle-arrow-cpp \ + --bundle-boost \ + --boost-namespace=arrow_boost +PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER setup.py bdist_wheel +PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER setup.py sdist + +if [ -n "$UBUNTU_WHEELS" ]; then + echo "=== (${PYTHON_VERSION}) Wheels are not compatible with manylinux1 ===" + mv dist/pyarrow-*.whl /io/dist +else + echo "=== (${PYTHON_VERSION}) Tag the wheel with manylinux2010 ===" + mkdir -p repaired_wheels/ + auditwheel -v repair --plat manylinux2010_x86_64 -L . dist/pyarrow-*.whl -w repaired_wheels/ + + # Install the built wheels + $PIP install repaired_wheels/*.whl + + # Test that the modules are importable + $PYTHON_INTERPRETER -c " +import sys +import pyarrow +import pyarrow.orc +import pyarrow.parquet +import pyarrow.plasma + +#if sys.version_info.major > 2: +# import pyarrow.gandiva + " + + # More thorough testing happens outsite of the build to prevent + # packaging issues like ARROW-4372 + mv dist/*.tar.gz /io/dist + mv repaired_wheels/*.whl /io/dist +fi diff --git a/python/manylinux2010/scripts/build_bison.sh b/python/manylinux2010/scripts/build_bison.sh new file mode 100755 index 0000000000000..29cc0be6adf6c --- /dev/null +++ b/python/manylinux2010/scripts/build_bison.sh @@ -0,0 +1,26 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +wget http://ftp.gnu.org/gnu/bison/bison-3.0.4.tar.gz +tar xf bison-3.0.4.tar.gz +pushd bison-3.0.4 +./configure --prefix=/usr +make -j4 +make install +popd +rm -rf bison-3.0.4 bison-3.0.4.tar.gz diff --git a/python/manylinux2010/scripts/build_boost.sh b/python/manylinux2010/scripts/build_boost.sh new file mode 100755 index 0000000000000..666968d80fb3f --- /dev/null +++ b/python/manylinux2010/scripts/build_boost.sh @@ -0,0 +1,43 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +BOOST_VERSION=1.66.0 +BOOST_VERSION_UNDERSCORE=${BOOST_VERSION//\./_} +NCORES=$(($(grep -c ^processor /proc/cpuinfo) + 1)) + +curl -sL https://dl.bintray.com/boostorg/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_UNDERSCORE}.tar.gz -o /boost_${BOOST_VERSION_UNDERSCORE}.tar.gz +tar xf boost_${BOOST_VERSION_UNDERSCORE}.tar.gz +mkdir /arrow_boost +pushd /boost_${BOOST_VERSION_UNDERSCORE} +./bootstrap.sh +./b2 -j${NCORES} tools/bcp +./dist/bin/bcp --namespace=arrow_boost --namespace-alias filesystem date_time system regex build algorithm locale format variant multiprecision/cpp_int /arrow_boost +popd + +pushd /arrow_boost +ls -l +./bootstrap.sh +./bjam -j${NCORES} dll-path="'\$ORIGIN/'" cxxflags='-std=c++11 -fPIC' cflags=-fPIC linkflags="-std=c++11" variant=release link=shared --prefix=/arrow_boost_dist --with-filesystem --with-date_time --with-system --with-regex install +popd +rm -rf boost_${BOOST_VERSION_UNDERSCORE}.tar.gz boost_${BOOST_VERSION_UNDERSCORE} arrow_boost +# Boost always install header-only parts but they also take up quite some space. +# We don't need them in array, so don't persist them in the docker layer. +# fusion 16.7 MiB +rm -r /arrow_boost_dist/include/boost/fusion +# spirit 8.2 MiB +rm -r /arrow_boost_dist/include/boost/spirit diff --git a/python/manylinux2010/scripts/build_brotli.sh b/python/manylinux2010/scripts/build_brotli.sh new file mode 100755 index 0000000000000..f26bf5947e38e --- /dev/null +++ b/python/manylinux2010/scripts/build_brotli.sh @@ -0,0 +1,35 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +export BROTLI_VERSION="0.6.0" +curl -sL "https://github.com/google/brotli/archive/v${BROTLI_VERSION}.tar.gz" -o brotli-${BROTLI_VERSION}.tar.gz +tar xf brotli-${BROTLI_VERSION}.tar.gz +pushd brotli-${BROTLI_VERSION} +mkdir build +pushd build +cmake -DCMAKE_BUILD_TYPE=release \ + "-DCMAKE_CXX_FLAGS=-fPIC" \ + "-DCMAKE_C_FLAGS=-fPIC" \ + -DCMAKE_INSTALL_PREFIX=/usr \ + -DBUILD_SHARED_LIBS=OFF \ + -GNinja \ + .. +ninja install +popd +popd +rm -rf brotli-${BROTLI_VERSION}.tar.gz brotli-${BROTLI_VERSION} diff --git a/python/manylinux2010/scripts/build_ccache.sh b/python/manylinux2010/scripts/build_ccache.sh new file mode 100755 index 0000000000000..c27f5089f01d4 --- /dev/null +++ b/python/manylinux2010/scripts/build_ccache.sh @@ -0,0 +1,29 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +curl -sL https://www.samba.org/ftp/ccache/ccache-3.3.4.tar.bz2 -o ccache-3.3.4.tar.bz2 +tar xf ccache-3.3.4.tar.bz2 +pushd ccache-3.3.4 +./configure --prefix=/usr +make -j5 +make install +popd +rm -rf ccache-3.3.4.tar.bz2 ccache-3.3.4 + +# Initialize the config directory, otherwise the build sometimes fails. +mkdir /root/.ccache diff --git a/python/manylinux2010/scripts/build_clang.sh b/python/manylinux2010/scripts/build_clang.sh new file mode 100755 index 0000000000000..45cd06862e3f1 --- /dev/null +++ b/python/manylinux2010/scripts/build_clang.sh @@ -0,0 +1,39 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source /multibuild/manylinux_utils.sh + +export LLVM_VERSION="7.0.1" +curl -sL http://releases.llvm.org/${LLVM_VERSION}/cfe-${LLVM_VERSION}.src.tar.xz -o cfe-${LLVM_VERSION}.src.tar.xz +unxz cfe-${LLVM_VERSION}.src.tar.xz +tar xf cfe-${LLVM_VERSION}.src.tar +pushd cfe-${LLVM_VERSION}.src +mkdir build +pushd build +cmake \ + -DCMAKE_BUILD_TYPE=Release \ + -DCLANG_INCLUDE_TESTS=OFF \ + -DCLANG_INCLUDE_DOCS=OFF \ + -DLLVM_INCLUDE_TESTS=OFF \ + -DLLVM_INCLUDE_DOCS=OFF \ + -GNinja \ + .. +ninja install +popd +popd +rm -rf cfe-${LLVM_VERSION}.src.tar.xz cfe-${LLVM_VERSION}.src.tar cfe-${LLVM_VERSION}.src diff --git a/python/manylinux2010/scripts/build_double_conversion.sh b/python/manylinux2010/scripts/build_double_conversion.sh new file mode 100755 index 0000000000000..0e73c702bee01 --- /dev/null +++ b/python/manylinux2010/scripts/build_double_conversion.sh @@ -0,0 +1,29 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +DC_VERSION=3.1.1 + +curl -sL https://github.com/google/double-conversion/archive/v${DC_VERSION}.tar.gz -o double-conversion-${DC_VERSION}.tar.gz +tar xf double-conversion-${DC_VERSION}.tar.gz +pushd double-conversion-3.1.1 + +cmake . -DCMAKE_INSTALL_PREFIX=/usr -DBUILD_SHARED_LIBS=OFF -DBUILD_TESTING=OFF -GNinja -DCMAKE_POSITION_INDEPENDENT_CODE=ON +ninja install + +popd +rm -rf double-conversion-${DC_VERSION}.tar.gz double-conversion-3.1.1 diff --git a/python/manylinux2010/scripts/build_flatbuffers.sh b/python/manylinux2010/scripts/build_flatbuffers.sh new file mode 100755 index 0000000000000..7aaaa60553a48 --- /dev/null +++ b/python/manylinux2010/scripts/build_flatbuffers.sh @@ -0,0 +1,32 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +export FLATBUFFERS_VERSION=1.10.0 +curl -sL https://github.com/google/flatbuffers/archive/v${FLATBUFFERS_VERSION}.tar.gz \ + -o flatbuffers-${FLATBUFFERS_VERSION}.tar.gz +tar xf flatbuffers-${FLATBUFFERS_VERSION}.tar.gz +pushd flatbuffers-${FLATBUFFERS_VERSION} +cmake \ + "-DCMAKE_CXX_FLAGS=-fPIC" \ + "-DCMAKE_INSTALL_PREFIX:PATH=/usr" \ + -DFLATBUFFERS_BUILD_TESTS=OFF \ + -DCMAKE_BUILD_TYPE=Release \ + -GNinja . +ninja install +popd +rm -rf flatbuffers-${FLATBUFFERS_VERSION}.tar.gz flatbuffers-${FLATBUFFERS_VERSION} diff --git a/python/manylinux2010/scripts/build_gflags.sh b/python/manylinux2010/scripts/build_gflags.sh new file mode 100755 index 0000000000000..7f40437a83f84 --- /dev/null +++ b/python/manylinux2010/scripts/build_gflags.sh @@ -0,0 +1,38 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +export GFLAGS_VERSION="2.2.1" +export CFLAGS="-fPIC" +export CXXFLAGS="-fPIC" + +curl -sL "https://github.com/gflags/gflags/archive/v${GFLAGS_VERSION}.tar.gz" -o gflags-${GFLAGS_VERSION}.tar.gz +tar xf gflags-${GFLAGS_VERSION}.tar.gz +pushd gflags-${GFLAGS_VERSION} + +cmake . \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/usr \ + -DINSTALL_HEADERS=on \ + -DBUILD_SHARED_LIBS=off \ + -DBUILD_STATIC_LIBS=on \ + -DBUILD_TESTING=off \ + -GNinja + +ninja install +popd +rm -rf gflags-${GFLAGS_VERSION}.tar.gz gflags-${GFLAGS_VERSION} diff --git a/python/manylinux2010/scripts/build_glog.sh b/python/manylinux2010/scripts/build_glog.sh new file mode 100755 index 0000000000000..5964e354c4652 --- /dev/null +++ b/python/manylinux2010/scripts/build_glog.sh @@ -0,0 +1,36 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +export GLOG_VERSION="0.3.5" +export CFLAGS="-fPIC" +export PREFIX="/usr" +curl -sL "https://github.com/google/glog/archive/v${GLOG_VERSION}.tar.gz" -o glog-${GLOG_VERSION}.tar.gz +tar xf glog-${GLOG_VERSION}.tar.gz +pushd glog-${GLOG_VERSION} + +cmake -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=${PREFIX} \ + -DBUILD_SHARED_LIBS=OFF \ + -DBUILD_TESTING=OFF \ + -DWITH_GFLAGS=OFF \ + -DCMAKE_CXX_FLAGS=${CFLAGS} \ + -GNinja . +ninja install +popd +rm -rf glog-${GLOG_VERSION}.tar.gz.tar.gz glog-${GLOG_VERSION} + diff --git a/python/manylinux2010/scripts/build_gtest.sh b/python/manylinux2010/scripts/build_gtest.sh new file mode 100755 index 0000000000000..723b59bddb7b6 --- /dev/null +++ b/python/manylinux2010/scripts/build_gtest.sh @@ -0,0 +1,39 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +GTEST_VERSION=1.8.1 + +curl -sL https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz -o googletest-release-${GTEST_VERSION}.tar.gz +tar xf googletest-release-${GTEST_VERSION}.tar.gz +ls -l +pushd googletest-release-${GTEST_VERSION} + +mkdir build_so +pushd build_so +cmake -DCMAKE_CXX_FLAGS='-fPIC' -Dgtest_force_shared_crt=ON -DBUILD_SHARED_LIBS=ON -DBUILD_GMOCK=ON -GNinja -DCMAKE_INSTALL_PREFIX=/usr .. +ninja install +popd + +mkdir build_a +pushd build_a +cmake -DCMAKE_CXX_FLAGS='-fPIC' -Dgtest_force_shared_crt=ON -DBUILD_SHARED_LIBS=OFF -DBUILD_GMOCK=ON -GNinja -DCMAKE_INSTALL_PREFIX=/usr .. +ninja install +popd + +popd +rm -rf googletest-release-${GTEST_VERSION}.tar.gz diff --git a/python/manylinux2010/scripts/build_llvm.sh b/python/manylinux2010/scripts/build_llvm.sh new file mode 100755 index 0000000000000..ccca2e99ddabe --- /dev/null +++ b/python/manylinux2010/scripts/build_llvm.sh @@ -0,0 +1,42 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source /multibuild/manylinux_utils.sh + +export LLVM_VERSION="7.0.1" +curl -sL http://releases.llvm.org/${LLVM_VERSION}/llvm-${LLVM_VERSION}.src.tar.xz -o llvm-${LLVM_VERSION}.src.tar.xz +unxz llvm-${LLVM_VERSION}.src.tar.xz +tar xf llvm-${LLVM_VERSION}.src.tar +pushd llvm-${LLVM_VERSION}.src +mkdir build +pushd build +cmake -DCMAKE_INSTALL_PREFIX=$PREFIX \ + -DCMAKE_BUILD_TYPE=Release -DLLVM_TARGETS_TO_BUILD=host \ + -DLLVM_INCLUDE_TESTS=OFF -DLLVM_INCLUDE_UTILS=OFF \ + -DLLVM_INCLUDE_DOCS=OFF -DLLVM_INCLUDE_EXAMPLES=OFF \ + -DLLVM_ENABLE_TERMINFO=OFF -DLLVM_ENABLE_ASSERTIONS=ON \ + -DLLVM_USE_INTEL_JITEVENTS=ON \ + -DLLVM_ENABLE_OCAMLDOC=OFF \ + -DLLVM_ENABLE_RTTI=ON \ + -DPYTHON_EXECUTABLE="$(cpython_path 2.7 32)/bin/python" \ + -GNinja \ + .. +ninja install +popd +popd +rm -rf llvm-${LLVM_VERSION}.src.tar.xz llvm-${LLVM_VERSION}.src.tar llvm-${LLVM_VERSION} diff --git a/python/manylinux2010/scripts/build_lz4.sh b/python/manylinux2010/scripts/build_lz4.sh new file mode 100755 index 0000000000000..75f791c0fd11e --- /dev/null +++ b/python/manylinux2010/scripts/build_lz4.sh @@ -0,0 +1,32 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +export LZ4_VERSION="1.8.3" +export PREFIX="/usr" +export CFLAGS="${CFLAGS} -O3 -fPIC" +export LDFLAGS="${LDFLAGS} -Wl,-rpath,${PREFIX}/lib -L${PREFIX}/lib" +curl -sL "https://github.com/lz4/lz4/archive/v${LZ4_VERSION}.tar.gz" -o lz4-${LZ4_VERSION}.tar.gz +tar xf lz4-${LZ4_VERSION}.tar.gz +pushd lz4-${LZ4_VERSION} + +make -j5 PREFIX=${PREFIX} +make install PREFIX=$PREFIX +popd +rm -rf lz4-${LZ4_VERSION}.tar.gz lz4-${LZ4_VERSION} +# We don't want to link against shared libs +rm -rf /usr/lib/liblz4.so* diff --git a/python/manylinux2010/scripts/build_openssl.sh b/python/manylinux2010/scripts/build_openssl.sh new file mode 100755 index 0000000000000..f4136f4aaa4d0 --- /dev/null +++ b/python/manylinux2010/scripts/build_openssl.sh @@ -0,0 +1,29 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +OPENSSL_VERSION="1.0.2q" +NCORES=$(($(grep -c ^processor /proc/cpuinfo) + 1)) + +wget --no-check-certificate https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz -O openssl-${OPENSSL_VERSION}.tar.gz +tar xf openssl-${OPENSSL_VERSION}.tar.gz +pushd openssl-${OPENSSL_VERSION} +./config -fpic shared --prefix=/usr +make -j${NCORES} +make install +popd +rm -rf openssl-${OPENSSL_VERSION}.tar.gz openssl-${OPENSSL_VERSION} diff --git a/python/manylinux2010/scripts/build_protobuf.sh b/python/manylinux2010/scripts/build_protobuf.sh new file mode 100755 index 0000000000000..c3ea8007fc052 --- /dev/null +++ b/python/manylinux2010/scripts/build_protobuf.sh @@ -0,0 +1,28 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +PROTOBUF_VERSION="2.6.0" + +curl -sL https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-${PROTOBUF_VERSION}.tar.gz -o protobuf-${PROTOBUF_VERSION}.tar.gz +tar xf protobuf-${PROTOBUF_VERSION}.tar.gz +pushd protobuf-${PROTOBUF_VERSION} +./configure --disable-shared --prefix=/usr "CXXFLAGS=-O2 -fPIC" +make -j5 +make install +popd +rm -rf protobuf-${PROTOBUF_VERSION}.tar.gz protobuf-${PROTOBUF_VERSION} diff --git a/python/manylinux2010/scripts/build_python.sh b/python/manylinux2010/scripts/build_python.sh new file mode 100755 index 0000000000000..d116d43f8e88f --- /dev/null +++ b/python/manylinux2010/scripts/build_python.sh @@ -0,0 +1,218 @@ +#!/bin/bash -e +# Copyright (c) 2016 manylinux +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# The following is taken from docker/build_scripts/build_env.sh, +# docker/build_scripts/build_utils.sh and +# docker/build_scripts/build.sh from the manylinux1 project +# (https://github.com/pypa/manylinux/). + +PYTHON_DOWNLOAD_URL=https://www.python.org/ftp/python +CPYTHON_VERSIONS="2.7.15 3.4.9 3.5.6 3.6.8 3.7.2" + +# openssl version to build, with expected sha256 hash of .tar.gz +# archive. +OPENSSL_ROOT=openssl-1.0.2q +OPENSSL_HASH=5744cfcbcec2b1b48629f7354203bc1e5e9b5466998bbccc5b5fcde3b18eb684 +OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source + +# Update to slightly newer, verified Git commit: +# https://github.com/NixOS/patchelf/commit/2a9cefd7d637d160d12dc7946393778fa8abbc58 +PATCHELF_VERSION=2a9cefd7d637d160d12dc7946393778fa8abbc58 +PATCHELF_HASH=12da4727f09be42ae0b54878e1b8e86d85cb7a5b595731cdc1a0a170c4873c6d + +CURL_ROOT=curl-7.61.1 +CURL_HASH=eaa812e9a871ea10dbe8e1d3f8f12a64a8e3e62aeab18cb23742e2f1727458ae +CURL_DOWNLOAD_URL=https://curl.haxx.se/download + +AUTOCONF_ROOT=autoconf-2.69 +AUTOCONF_HASH=954bd69b391edc12d6a4a51a2dd1476543da5c6bbf05a95b59dc0dd6fd4c2969 +AUTOCONF_DOWNLOAD_URL=http://ftp.gnu.org/gnu/autoconf +AUTOMAKE_ROOT=automake-1.16.1 +AUTOMAKE_HASH=608a97523f97db32f1f5d5615c98ca69326ced2054c9f82e65bade7fc4c9dea8 +AUTOMAKE_DOWNLOAD_URL=http://ftp.gnu.org/gnu/automake +LIBTOOL_ROOT=libtool-2.4.6 +LIBTOOL_HASH=e3bd4d5d3d025a36c21dd6af7ea818a2afcd4dfc1ea5a17b39d7854bcd0c06e3 +LIBTOOL_DOWNLOAD_URL=http://ftp.gnu.org/gnu/libtool + +SQLITE_AUTOCONF_VERSION=sqlite-autoconf-3230100 +SQLITE_AUTOCONF_HASH=92842b283e5e744eff5da29ed3c69391de7368fccc4d0ee6bf62490ce555ef25 +SQLITE_AUTOCONF_DOWNLOAD_URL=https://www.sqlite.org/2018 + +GIT_ROOT=2.19.1 +GIT_HASH=ba2fed9d02e424b735e035c4f2b0bdb168ef0df7e35156b5051d900dc7247787 +GIT_DOWNLOAD_URL=https://github.com/git/git/archive + +GET_PIP_URL=https://bootstrap.pypa.io/get-pip.py +EPEL_RPM_HASH=0dcc89f9bf67a2a515bad64569b7a9615edc5e018f676a578d5fd0f17d3c81d4 +DEVTOOLS_HASH=a8ebeb4bed624700f727179e6ef771dafe47651131a00a78b342251415646acc + +function check_var { + if [ -z "$1" ]; then + echo "required variable not defined" + exit 1 + fi +} + +function lex_pyver { + # Echoes Python version string padded with zeros + # Thus: + # 3.2.1 -> 003002001 + # 3 -> 003000000 + echo $1 | awk -F "." '{printf "%03d%03d%03d", $1, $2, $3}' +} + +function pyver_dist_dir { + # Echoes the dist directory name of given pyver, removing alpha/beta prerelease + # Thus: + # 3.2.1 -> 3.2.1 + # 3.7.0b4 -> 3.7.0 + echo $1 | awk -F "." '{printf "%d.%d.%d", $1, $2, $3}' +} + +function do_cpython_build { + local py_ver=$1 + check_var $py_ver + local ucs_setting=$2 + check_var $ucs_setting + tar -xzf Python-$py_ver.tgz + pushd Python-$py_ver + if [ "$ucs_setting" = "none" ]; then + unicode_flags="" + dir_suffix="" + else + local unicode_flags="--enable-unicode=$ucs_setting" + local dir_suffix="-$ucs_setting" + fi + local prefix="/opt/_internal/cpython-${py_ver}${dir_suffix}" + mkdir -p ${prefix}/lib + ./configure --prefix=${prefix} --disable-shared $unicode_flags > /dev/null + make -j2 > /dev/null + make install > /dev/null + popd + rm -rf Python-$py_ver + # Some python's install as bin/python3. Make them available as + # bin/python. + if [ -e ${prefix}/bin/python3 ]; then + ln -s python3 ${prefix}/bin/python + fi + # --force-reinstall is to work around: + # https://github.com/pypa/pip/issues/5220 + # https://github.com/pypa/get-pip/issues/19 + ${prefix}/bin/python get-pip.py --force-reinstall + if [ -e ${prefix}/bin/pip3 ] && [ ! -e ${prefix}/bin/pip ]; then + ln -s pip3 ${prefix}/bin/pip + fi + # Since we fall back on a canned copy of get-pip.py, we might not have + # the latest pip and friends. Upgrade them to make sure. + ${prefix}/bin/pip install -U --require-hashes -r ${MY_DIR}/requirements.txt + local abi_tag=$(${prefix}/bin/python ${MY_DIR}/python-tag-abi-tag.py) + ln -s ${prefix} /opt/python/${abi_tag} +} + + +function build_cpython { + local py_ver=$1 + check_var $py_ver + check_var $PYTHON_DOWNLOAD_URL + local py_dist_dir=$(pyver_dist_dir $py_ver) + curl -fsSLO $PYTHON_DOWNLOAD_URL/$py_dist_dir/Python-$py_ver.tgz + curl -fsSLO $PYTHON_DOWNLOAD_URL/$py_dist_dir/Python-$py_ver.tgz.asc + if [ $(lex_pyver $py_ver) -lt $(lex_pyver 3.3) ]; then + do_cpython_build $py_ver ucs2 + do_cpython_build $py_ver ucs4 + else + do_cpython_build $py_ver none + fi + rm -f Python-$py_ver.tgz + rm -f Python-$py_ver.tgz.asc +} + + +function build_cpythons { + check_var $GET_PIP_URL + curl -fsSLO $GET_PIP_URL + for py_ver in $@; do + build_cpython $py_ver + done + rm -f get-pip.py +} + +function do_openssl_build { + ./config no-ssl2 no-shared -fPIC --prefix=/usr/local/ssl > /dev/null + make > /dev/null + make install_sw > /dev/null +} + + +function check_required_source { + local file=$1 + check_var ${file} + if [ ! -f $file ]; then + echo "Required source archive must be prefetched to docker/sources/ with prefetch.sh: $file" + return 1 + fi +} + + +function fetch_source { + # This is called both inside and outside the build context (e.g. in Travis) to prefetch + # source tarballs, where curl exists (and works) + local file=$1 + check_var ${file} + local url=$2 + check_var ${url} + if [ -f ${file} ]; then + echo "${file} exists, skipping fetch" + else + curl -fsSL -o ${file} ${url}/${file} + fi +} + + +function check_sha256sum { + local fname=$1 + check_var ${fname} + local sha256=$2 + check_var ${sha256} + + echo "${sha256} ${fname}" > ${fname}.sha256 + sha256sum -c ${fname}.sha256 + rm -f ${fname}.sha256 +} + + +function build_openssl { + local openssl_fname=$1 + check_var ${openssl_fname} + local openssl_sha256=$2 + check_var ${openssl_sha256} + # Can't use curl here because we don't have it yet, OpenSSL must be prefetched + fetch_source ${openssl_fname}.tar.gz ${OPENSSL_DOWNLOAD_URL} + check_sha256sum ${openssl_fname}.tar.gz ${openssl_sha256} + tar -xzf ${openssl_fname}.tar.gz + (cd ${openssl_fname} && do_openssl_build) + rm -rf ${openssl_fname} ${openssl_fname}.tar.gz +} + +build_openssl $OPENSSL_ROOT $OPENSSL_HASH + +mkdir -p /opt/python +build_cpythons $CPYTHON_VERSIONS diff --git a/python/manylinux2010/scripts/build_rapidjson.sh b/python/manylinux2010/scripts/build_rapidjson.sh new file mode 100755 index 0000000000000..3b2ee99c422d0 --- /dev/null +++ b/python/manylinux2010/scripts/build_rapidjson.sh @@ -0,0 +1,37 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +export RAPIDJSON_VERSION="1.1.0" + +curl -sL "https://github.com/miloyip/rapidjson/archive/v${RAPIDJSON_VERSION}.tar.gz" -o rapidjson-${RAPIDJSON_VERSION}.tar.gz +tar xf rapidjson-${RAPIDJSON_VERSION}.tar.gz +pushd rapidjson-${RAPIDJSON_VERSION} +mkdir build +pushd build +cmake -GNinja \ + -DRAPIDJSON_HAS_STDSTRING=ON \ + -DCMAKE_INSTALL_PREFIX=/usr \ + -DRAPIDJSON_BUILD_TESTS=OFF \ + -DRAPIDJSON_BUILD_EXAMPLES=OFF \ + -DRAPIDJSON_BUILD_DOC=OFF \ + -DCMAKE_BUILD_TYPE=release \ + .. +ninja install +popd +popd +rm -rf rapidjson-${RAPIDJSON_VERSION}.tar.gz rapidjson-${RAPIDJSON_VERSION} diff --git a/python/manylinux2010/scripts/build_re2.sh b/python/manylinux2010/scripts/build_re2.sh new file mode 100755 index 0000000000000..a0fe9e5680265 --- /dev/null +++ b/python/manylinux2010/scripts/build_re2.sh @@ -0,0 +1,32 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +export RE2_VERSION="2019-01-01" + +curl -sL "http://github.com/google/re2/archive/${RE2_VERSION}.tar.gz" -o re2-${RE2_VERSION}.tar.gz +tar xf re2-${RE2_VERSION}.tar.gz +pushd re2-${RE2_VERSION} + +export CXXFLAGS="-fPIC ${CXXFLAGS}" +export CFLAGS="-fPIC ${CFLAGS}" + +# Build shared libraries +make prefix=/usr install + +popd +rm -rf re2-${RE2_VERSION}.tar.gz re2-${RE2_VERSION} /usr/lib/libre2.so* diff --git a/python/manylinux2010/scripts/build_snappy.sh b/python/manylinux2010/scripts/build_snappy.sh new file mode 100755 index 0000000000000..a749897ac0cab --- /dev/null +++ b/python/manylinux2010/scripts/build_snappy.sh @@ -0,0 +1,26 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +export SNAPPY_VERSION="1.1.7" +curl -sL "https://github.com/google/snappy/archive/${SNAPPY_VERSION}.tar.gz" -o snappy-${SNAPPY_VERSION}.tar.gz +tar xf snappy-${SNAPPY_VERSION}.tar.gz +pushd snappy-${SNAPPY_VERSION} +CXXFLAGS='-DNDEBUG -O2' cmake -GNinja -DCMAKE_INSTALL_PREFIX=/usr -DBUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=1 -DSNAPPY_BUILD_TESTS=OFF . +ninja install +popd +rm -rf snappy-${SNAPPY_VERSION}.tar.gz snappy-${SNAPPY_VERSION} diff --git a/python/manylinux2010/scripts/build_thrift.sh b/python/manylinux2010/scripts/build_thrift.sh new file mode 100755 index 0000000000000..dca4ad4e077a2 --- /dev/null +++ b/python/manylinux2010/scripts/build_thrift.sh @@ -0,0 +1,45 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +export THRIFT_VERSION=0.12.0 +wget http://archive.apache.org/dist/thrift/${THRIFT_VERSION}/thrift-${THRIFT_VERSION}.tar.gz +tar xf thrift-${THRIFT_VERSION}.tar.gz +pushd thrift-${THRIFT_VERSION} +mkdir build-tmp +pushd build-tmp +cmake -DCMAKE_BUILD_TYPE=release \ + "-DCMAKE_CXX_FLAGS=-fPIC" \ + "-DCMAKE_C_FLAGS=-fPIC" \ + "-DCMAKE_INSTALL_PREFIX=/usr" \ + "-DCMAKE_INSTALL_RPATH=/usr/lib" \ + -DBUILD_EXAMPLES=OFF \ + -DBUILD_TESTING=OFF \ + -DWITH_QT4=OFF \ + -DWITH_C_GLIB=OFF \ + -DWITH_JAVA=OFF \ + -DWITH_PYTHON=OFF \ + -DWITH_CPP=ON \ + -DWITH_STATIC_LIB=ON \ + -DWITH_SHARED_LIB=OFF \ + -DBoost_NAMESPACE=arrow_boost \ + -DBOOST_ROOT=/arrow_boost_dist \ + -GNinja .. +ninja install +popd +popd +rm -rf thrift-${THRIFT_VERSION}.tar.gz thrift-${THRIFT_VERSION} diff --git a/python/manylinux2010/scripts/build_virtualenvs.sh b/python/manylinux2010/scripts/build_virtualenvs.sh new file mode 100755 index 0000000000000..a737e2f6ef968 --- /dev/null +++ b/python/manylinux2010/scripts/build_virtualenvs.sh @@ -0,0 +1,70 @@ +#!/bin/bash -e +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Build upon the scripts in https://github.com/matthew-brett/manylinux-builds +# * Copyright (c) 2013-2016, Matt Terry and Matthew Brett (BSD 2-clause) + +PYTHON_VERSIONS="${PYTHON_VERSIONS:-2.7,16 2.7,32 3.5,16 3.6,16, 3.7,16}" + +source /multibuild/manylinux_utils.sh + +for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do + IFS="," + set -- $PYTHON_TUPLE; + PYTHON=$1 + U_WIDTH=$2 + PYTHON_INTERPRETER="$(cpython_path $PYTHON ${U_WIDTH})/bin/python" + PIP="$(cpython_path $PYTHON ${U_WIDTH})/bin/pip" + PATH="$PATH:$(cpython_path $PYTHON ${U_WIDTH})" + + echo "=== (${PYTHON}, ${U_WIDTH}) Installing build dependencies ===" + $PIP install "numpy==1.14.5" "cython==0.29.3" "virtualenv==16.3.0" + # Pandas requires numpy and cython + $PIP install "pandas==0.24.0" + + # TensorFlow is not supported for Python 2.7 with unicode width 16 or with Python 3.7 + if [ $PYTHON != "2.7" ] || [ $U_WIDTH = "32" ]; then + if [ $PYTHON != "3.7" ]; then + $PIP install "tensorflow==1.11.0" "Keras-Preprocessing==1.0.5" + fi + fi + + + echo "=== (${PYTHON}, ${U_WIDTH}) Preparing virtualenv for tests ===" + "$(cpython_path $PYTHON ${U_WIDTH})/bin/virtualenv" -p ${PYTHON_INTERPRETER} --no-download /venv-test-${PYTHON}-${U_WIDTH} + source /venv-test-${PYTHON}-${U_WIDTH}/bin/activate + pip install pytest hypothesis 'numpy==1.14.5' 'pandas==0.24.0' + deactivate +done + +# Remove debug symbols from libraries that were installed via wheel. +find /venv-test-*/lib/*/site-packages/pandas -name '*.so' -exec strip '{}' ';' +find /venv-test-*/lib/*/site-packages/numpy -name '*.so' -exec strip '{}' ';' +find /opt/_internal/cpython-*/lib/*/site-packages/pandas -name '*.so' -exec strip '{}' ';' +# Only Python 3.6+ packages are stripable as they are built inside of the image +find /opt/_internal/cpython-3.6.*/lib/python3.6/site-packages/numpy -name '*.so' -exec strip '{}' ';' +find /opt/_internal/cpython-3.7.*/lib/python3.7/site-packages/numpy -name '*.so' -exec strip '{}' ';' +find /opt/_internal/*/lib/*/site-packages/Cython -name '*.so' -exec strip '{}' ';' + +# Remove pip cache again. It's useful during the virtualenv creation but we +# don't want it persisted in the docker layer, ~264MiB +rm -rf /root/.cache +# Remove pandas' tests module as it includes a lot of data, ~27MiB per Python +# venv, i.e. 216MiB in total +rm -rf /opt/_internal/*/lib/*/site-packages/pandas/tests +rm -rf /venv-test-*/lib/*/site-packages/pandas/tests diff --git a/python/manylinux2010/scripts/build_zlib.sh b/python/manylinux2010/scripts/build_zlib.sh new file mode 100755 index 0000000000000..182a42ff5f55f --- /dev/null +++ b/python/manylinux2010/scripts/build_zlib.sh @@ -0,0 +1,26 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +curl -sL https://zlib.net/zlib-1.2.11.tar.gz -o /zlib-1.2.11.tar.gz +tar xf zlib-1.2.11.tar.gz +pushd zlib-1.2.11 +./configure +make -j4 +make install +popd +rm -rf zlib-1.2.11.tar.gz zlib-1.2.11 diff --git a/python/manylinux2010/scripts/build_zstd.sh b/python/manylinux2010/scripts/build_zstd.sh new file mode 100755 index 0000000000000..1add13faa61fe --- /dev/null +++ b/python/manylinux2010/scripts/build_zstd.sh @@ -0,0 +1,32 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +export ZSTD_VERSION="1.3.8" + +curl -sL "https://github.com/facebook/zstd/archive/v${ZSTD_VERSION}.tar.gz" -o zstd-${ZSTD_VERSION}.tar.gz +tar xf zstd-${ZSTD_VERSION}.tar.gz +pushd zstd-${ZSTD_VERSION} +mkdir build_cmake +pushd build_cmake + +cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr ../build/cmake -DZSTD_BUILD_STATIC=ON -DZSTD_BUILD_SHARED=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=1 +ninja install + +popd +popd +rm -rf zstd-${ZSTD_VERSION}.tar.gz zstd-${ZSTD_VERSION} diff --git a/python/manylinux2010/scripts/check_arrow_visibility.sh b/python/manylinux2010/scripts/check_arrow_visibility.sh new file mode 100755 index 0000000000000..ed464e0779a0e --- /dev/null +++ b/python/manylinux2010/scripts/check_arrow_visibility.sh @@ -0,0 +1,35 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +nm --demangle --dynamic /arrow-dist/lib/libarrow.so > nm_arrow.log + +# Filter out Arrow symbols and see if anything remains. +# '_init' and '_fini' symbols may or not be present, we don't care. +# (note we must ignore the grep exit status when no match is found) +grep ' T ' nm_arrow.log | grep -v -E '(arrow|\b_init\b|\b_fini\b)' | cat - > visible_symbols.log + +if [[ -f visible_symbols.log && `cat visible_symbols.log | wc -l` -eq 0 ]] +then + exit 0 +fi + +echo "== Unexpected symbols exported by libarrow.so ==" +cat visible_symbols.log +echo "================================================" + +exit 1 diff --git a/python/manylinux2010/scripts/install_cmake.sh b/python/manylinux2010/scripts/install_cmake.sh new file mode 100755 index 0000000000000..90e52c15143ec --- /dev/null +++ b/python/manylinux2010/scripts/install_cmake.sh @@ -0,0 +1,23 @@ +#!/bin/bash -e +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +rm /usr/bin/cmake +/opt/python/cp35-cp35m/bin/pip install cmake ninja +ln -s /opt/python/cp35-cp35m/bin/cmake /usr/bin/cmake +ln -s /opt/python/cp35-cp35m/bin/ninja /usr/bin/ninja +strip /opt/_internal/cpython-3.*/lib/python3.5/site-packages/cmake/data/bin/* diff --git a/python/manylinux2010/scripts/python-tag-abi-tag.py b/python/manylinux2010/scripts/python-tag-abi-tag.py new file mode 100644 index 0000000000000..212ab545c5fc4 --- /dev/null +++ b/python/manylinux2010/scripts/python-tag-abi-tag.py @@ -0,0 +1,30 @@ +# Copyright (c) 2016 manylinux +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# The following is taken from docker/build_scripts/python-tag-abi-tag.py +# from the manylinux1 project (https://github.com/pypa/manylinux/). + +# Utility script to print the python tag + the abi tag for a Python +# See PEP 425 for exactly what these are, but an example would be: +# cp27-cp27mu + +from wheel.pep425tags import get_abbr_impl, get_impl_ver, get_abi_tag + +print("{0}{1}-{2}".format(get_abbr_impl(), get_impl_ver(), get_abi_tag())) diff --git a/python/manylinux2010/scripts/requirements.txt b/python/manylinux2010/scripts/requirements.txt new file mode 100644 index 0000000000000..38a32dfe3900d --- /dev/null +++ b/python/manylinux2010/scripts/requirements.txt @@ -0,0 +1,34 @@ +# Copyright (c) 2016 manylinux +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# The following is taken from docker/build_scripts/requirements.txt +# from the manylinux1 project (https://github.com/pypa/manylinux/). + +# pip requirements for all cpythons +# NOTE: pip has GPG signatures; could download and verify independently. +pip==19.0.3 \ + --hash=sha256:6e6f197a1abfb45118dbb878b5c859a0edbdd33fd250100bc015b67fded4b9f2 \ + --hash=sha256:bd812612bbd8ba84159d9ddc0266b7fbce712fc9bc98c82dee5750546ec8ec64 +wheel==0.31.1 \ + --hash=sha256:80044e51ec5bbf6c894ba0bc48d26a8c20a9ba629f4ca19ea26ecfcf87685f5f \ + --hash=sha256:0a2e54558a0628f2145d2fc822137e322412115173e8a2ddbe1c9024338ae83c +setuptools==40.7.3 \ + --hash=sha256:4f4acaf06d617dccfd3fbbc9fbd83cf4749759a1fa2bdf589206a3278e0d537a \ + --hash=sha256:702fdd31cb10a65a94beba1a7d89219a58d2587a349e0a1b7827b133e99ca430