From 8fa9e373fa2429b560ba754f53293ef8fbee5f76 Mon Sep 17 00:00:00 2001 From: Matthieu Darbois Date: Sat, 6 Feb 2021 20:34:22 +0100 Subject: [PATCH] [manylinux2014] Rework scripts to be more docker cache friendly (#879) This won't change travis-ci build time (at least for now) but allows maintainers/contributors to benefit from docker caching for local developments (faster builds at the expense of higher disk usage) Round 1 Use docker multi-stage build to build the manylinux image in 3 steps: 1. runtime_base: this image has all the necessary bits that are common to the build_base image and the manylinux image. This includes system packages required for runtime, the gcc toolchain & some basic tools. 2. build_base: this image builds all the remaining tools required for the manylinux image. Any tool requiring specific system development package not required in the manylinux image shall be built from this image. For tools that used to be installed in /usr/local, they are staged in /manylinux-rootfs for an easy copy in the third stage. 3. manylinux: This image uses the runtime_base image as a base image. Tools built in the 2nd step are copied directly in the final filesystem. A finalization script is then run to install the remaining python tools/dependencies and run checks. --- docker/Dockerfile | 34 +++- docker/build_scripts/build.sh | 191 ++---------------- docker/build_scripts/build_env.sh | 16 -- docker/build_scripts/build_env_runtime.sh | 17 ++ docker/build_scripts/build_utils.sh | 18 +- docker/build_scripts/finalize.sh | 69 +++++++ .../build_scripts/install-runtime-packages.sh | 141 +++++++++++++ 7 files changed, 277 insertions(+), 209 deletions(-) create mode 100644 docker/build_scripts/build_env_runtime.sh create mode 100755 docker/build_scripts/finalize.sh create mode 100755 docker/build_scripts/install-runtime-packages.sh diff --git a/docker/Dockerfile b/docker/Dockerfile index f9c10b0c..24e46745 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -2,11 +2,11 @@ ARG BASEIMAGE=amd64/centos:7 ARG POLICY=manylinux2014 ARG PLATFORM=x86_64 -ARG DEVTOOLSET_ROOTPATH= -ARG LD_LIBRARY_PATH_ARG= -ARG PREPEND_PATH= +ARG DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-9/root +ARG LD_LIBRARY_PATH_ARG=${DEVTOOLSET_ROOTPATH}/usr/lib64:${DEVTOOLSET_ROOTPATH}/usr/lib:${DEVTOOLSET_ROOTPATH}/usr/lib64/dyninst:${DEVTOOLSET_ROOTPATH}/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib +ARG PREPEND_PATH=${DEVTOOLSET_ROOTPATH}/usr/bin: -FROM $BASEIMAGE +FROM $BASEIMAGE AS runtime_base ARG POLICY ARG PLATFORM ARG DEVTOOLSET_ROOTPATH @@ -22,14 +22,36 @@ ENV PATH=${PREPEND_PATH}${PATH} ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig # setup entrypoint, this will wrap commands with `linux32` with i686 images -COPY build_scripts/install-entrypoint.sh /build_scripts/install-entrypoint.sh +COPY build_scripts/install-entrypoint.sh /build_scripts/ RUN bash /build_scripts/install-entrypoint.sh && rm -rf build_scripts COPY manylinux-entrypoint /usr/local/bin/manylinux-entrypoint ENTRYPOINT ["manylinux-entrypoint"] -COPY build_scripts /build_scripts + +COPY build_scripts/install-runtime-packages.sh /build_scripts/ +COPY build_scripts/build_utils.sh /build_scripts/ +COPY build_scripts/build_env_runtime.sh /build_scripts/ +RUN manylinux-entrypoint /build_scripts/install-runtime-packages.sh && rm -rf /build_scripts + +FROM runtime_base AS build_base +COPY build_scripts/*pubkey*.txt /build_scripts/ +COPY build_scripts/build_utils.sh /build_scripts/ +COPY build_scripts/build.sh /build_scripts/ +COPY build_scripts/build_env.sh /build_scripts/ RUN manylinux-entrypoint bash build_scripts/build.sh && rm -rf build_scripts +FROM runtime_base +COPY --from=build_base /manylinux-rootfs / +COPY --from=build_base /opt/_internal /opt/_internal/ +COPY build_scripts/build_utils.sh /build_scripts/build_utils.sh +COPY build_scripts/finalize.sh /build_scripts/finalize.sh +COPY build_scripts/python-tag-abi-tag.py /build_scripts/python-tag-abi-tag.py +COPY build_scripts/ssl-check.py /build_scripts/ssl-check.py +COPY build_scripts/manylinux-check.py /build_scripts/manylinux-check.py +COPY build_scripts/requirements.txt /build_scripts/requirements.txt +COPY build_scripts/requirements-tools.txt /build_scripts/requirements-tools.txt +RUN manylinux-entrypoint /build_scripts/finalize.sh && rm -rf /build_scripts + ENV SSL_CERT_FILE=/opt/_internal/certs.pem CMD ["/bin/bash"] diff --git a/docker/build_scripts/build.sh b/docker/build_scripts/build.sh index 615049fc..ec4677e4 100755 --- a/docker/build_scripts/build.sh +++ b/docker/build_scripts/build.sh @@ -6,119 +6,40 @@ set -ex # Set build environment variables MY_DIR=$(dirname "${BASH_SOURCE[0]}") -. $MY_DIR/build_env.sh - -# Dependencies for compiling Python that we want to remove from -# the final image after compiling Python -PYTHON_COMPILE_DEPS="zlib-devel bzip2-devel expat-devel ncurses-devel readline-devel tk-devel gdbm-devel libdb-devel libpcap-devel xz-devel openssl-devel keyutils-libs-devel krb5-devel libcom_err-devel libidn-devel curl-devel perl-devel" - -# Libraries that are allowed as part of the manylinux2014 profile -# Extract from PEP: https://www.python.org/dev/peps/pep-0599/#the-manylinux2014-policy -# On RPM-based systems, they are provided by these packages: -# Package: Libraries -# glib2: libglib-2.0.so.0, libgthread-2.0.so.0, libgobject-2.0.so.0 -# glibc: libresolv.so.2, libutil.so.1, libnsl.so.1, librt.so.1, libpthread.so.0, libdl.so.2, libm.so.6, libc.so.6 -# libICE: libICE.so.6 -# libX11: libX11.so.6 -# libXext: libXext.so.6 -# libXrender: libXrender.so.1 -# libgcc: libgcc_s.so.1 -# libstdc++: libstdc++.so.6 -# mesa: libGL.so.1 -# -# PEP is missing the package for libSM.so.6 for RPM based system -# Install development packages (except for libgcc which is provided by gcc install) -MANYLINUX_DEPS="glibc-devel libstdc++-devel glib2-devel libX11-devel libXext-devel libXrender-devel mesa-libGL-devel libICE-devel libSM-devel" - -CMAKE_DEPS="openssl-devel zlib-devel libcurl-devel" +source $MY_DIR/build_env.sh # Get build utilities source $MY_DIR/build_utils.sh -# See https://unix.stackexchange.com/questions/41784/can-yum-express-a-preference-for-x86-64-over-i386-packages -echo "multilib_policy=best" >> /etc/yum.conf -# Error out if requested packages do not exist -echo "skip_missing_names_on_install=False" >> /etc/yum.conf -# Make sure that locale will not be removed -sed -i '/^override_install_langs=/d' /etc/yum.conf - -# https://hub.docker.com/_/centos/ -# "Additionally, images with minor version tags that correspond to install -# media are also offered. These images DO NOT recieve updates as they are -# intended to match installation iso contents. If you choose to use these -# images it is highly recommended that you include RUN yum -y update && yum -# clean all in your Dockerfile, or otherwise address any potential security -# concerns." -# Decided not to clean at this point: https://github.com/pypa/manylinux/pull/129 -yum -y update -yum -y install yum-utils curl -yum-config-manager --enable extras - -if ! which localedef &> /dev/null; then - # somebody messed up glibc-common package to squeeze image size, reinstall the package - yum -y reinstall glibc-common -fi - -# upgrading glibc-common can end with removal on en_US.UTF-8 locale -localedef -i en_US -f UTF-8 en_US.UTF-8 - -TOOLCHAIN_DEPS="devtoolset-9-binutils devtoolset-9-gcc devtoolset-9-gcc-c++ devtoolset-9-gcc-gfortran" -if [ "${AUDITWHEEL_ARCH}" == "x86_64" ]; then - # Software collection (for devtoolset-9) - yum -y install centos-release-scl-rh - # EPEL support (for yasm) - yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm - YASM=yasm -elif [ "${AUDITWHEEL_ARCH}" == "aarch64" ] || [ "${AUDITWHEEL_ARCH}" == "ppc64le" ] || [ "${AUDITWHEEL_ARCH}" == "s390x" ]; then - # Software collection (for devtoolset-9) - yum -y install centos-release-scl-rh -elif [ "${AUDITWHEEL_ARCH}" == "i686" ]; then - # No yasm on i686 - # Install mayeut/devtoolset-9 repo to get devtoolset-9 - curl -fsSLo /etc/yum.repos.d/mayeut-devtoolset-9.repo https://copr.fedorainfracloud.org/coprs/mayeut/devtoolset-9/repo/custom-1/mayeut-devtoolset-9-custom-1.repo -fi +# Dependencies for compiling Python that we want to remove from +# the final image after compiling Python +PYTHON_COMPILE_DEPS="zlib-devel bzip2-devel expat-devel ncurses-devel readline-devel tk-devel gdbm-devel libdb-devel libpcap-devel xz-devel openssl-devel keyutils-libs-devel krb5-devel libcom_err-devel libidn-devel curl-devel perl-devel libffi-devel kernel-devel" +CMAKE_DEPS="openssl-devel zlib-devel libcurl-devel" # Development tools and libraries -yum -y install \ - autoconf \ - automake \ - bison \ - bzip2 \ - ${TOOLCHAIN_DEPS} \ - diffutils \ - gettext \ - file \ - kernel-devel \ - libffi-devel \ - make \ - patch \ - unzip \ - which \ - ${YASM} \ - ${PYTHON_COMPILE_DEPS} \ - ${CMAKE_DEPS} +yum -y install ${PYTHON_COMPILE_DEPS} ${CMAKE_DEPS} # Install git build_git $GIT_ROOT $GIT_HASH -git version - -# Install newest automake -build_automake $AUTOMAKE_ROOT $AUTOMAKE_HASH -automake --version - -# Install newest libtool -build_libtool $LIBTOOL_ROOT $LIBTOOL_HASH -libtool --version +/manylinux-rootfs/usr/local/bin/git version # Install a more recent SQLite3 curl -fsSLO $SQLITE_AUTOCONF_DOWNLOAD_URL/$SQLITE_AUTOCONF_ROOT.tar.gz check_sha256sum $SQLITE_AUTOCONF_ROOT.tar.gz $SQLITE_AUTOCONF_HASH tar xfz $SQLITE_AUTOCONF_ROOT.tar.gz cd $SQLITE_AUTOCONF_ROOT -do_standard_install +DESTDIR=/sqlite3 do_standard_install cd .. rm -rf $SQLITE_AUTOCONF_ROOT* -rm /usr/local/lib/libsqlite3.a +rm /sqlite3/usr/local/lib/libsqlite3.a +# Install for build +cp -rf /sqlite3/* / +# Clean-up for runtime +rm -rf /sqlite3/usr/local/bin /sqlite3/usr/local/include /sqlite3/usr/local/lib/pkg-config /sqlite3/usr/local/share +# Install for runtime +cp -rf /sqlite3/* /manylinux-rootfs/ +# clean-up +rm -rf /sqlite3 # Install a recent version of cmake3 curl -L -O $CMAKE_DOWNLOAD_URL/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}.tar.gz @@ -127,63 +48,16 @@ tar -xzf cmake-${CMAKE_VERSION}.tar.gz cd cmake-${CMAKE_VERSION} ./bootstrap --system-curl --parallel=$(nproc) make -j$(nproc) -make install +make install DESTDIR=/manylinux-rootfs cd .. -rm -rf cmake-${CMAKE_VERSION} - -# Install libcrypt.so.1 and libcrypt.so.2 -build_libxcrypt "$LIBXCRYPT_DOWNLOAD_URL" "$LIBXCRYPT_VERSION" "$LIBXCRYPT_HASH" +rm -rf cmake-${CMAKE_VERSION}* # Compile the latest Python releases. # (In order to have a proper SSL module, Python is compiled # against a recent openssl [see env vars above], which is linked # statically. -mkdir -p /opt/python build_cpythons $CPYTHON_VERSIONS -# Create venv for auditwheel & certifi -TOOLS_PATH=/opt/_internal/tools -/opt/python/cp37-cp37m/bin/python -m venv $TOOLS_PATH -source $TOOLS_PATH/bin/activate - -# Install default packages -pip install -U --require-hashes -r $MY_DIR/requirements.txt -# Install certifi and auditwheel -pip install -U --require-hashes -r $MY_DIR/requirements-tools.txt - -# Make auditwheel available in PATH -ln -s $TOOLS_PATH/bin/auditwheel /usr/local/bin/auditwheel - -# Our openssl doesn't know how to find the system CA trust store -# (https://github.com/pypa/manylinux/issues/53) -# And it's not clear how up-to-date that is anyway -# So let's just use the same one pip and everyone uses -ln -s $(python -c 'import certifi; print(certifi.where())') /opt/_internal/certs.pem -# If you modify this line you also have to modify the versions in the Dockerfiles: -export SSL_CERT_FILE=/opt/_internal/certs.pem - -# Deactivate the tools virtual environment -deactivate - -# Install patchelf (latest with unreleased bug fixes) and apply our patches -build_patchelf $PATCHELF_VERSION $PATCHELF_HASH - -# Clean up development headers and other unnecessary stuff for -# final image -yum -y erase \ - avahi \ - bitstream-vera-fonts \ - freetype \ - gettext \ - gtk2 \ - hicolor-icon-theme \ - libX11 \ - wireless-tools \ - ${PYTHON_COMPILE_DEPS} > /dev/null 2>&1 -yum -y install ${MANYLINUX_DEPS} -yum -y clean all > /dev/null 2>&1 -yum list installed - # we don't need libpython*.a, and they're many megabytes find /opt/_internal -name '*.a' -print0 | xargs -0 rm -f @@ -191,37 +65,12 @@ find /opt/_internal -name '*.a' -print0 | xargs -0 rm -f # *everything*, including non-ELF files: find /opt/_internal -type f -print0 \ | xargs -0 -n1 strip --strip-unneeded 2>/dev/null || true -find /usr/local -type f -print0 \ +find /manylinux-rootfs -type f -print0 \ | xargs -0 -n1 strip --strip-unneeded 2>/dev/null || true -for PYTHON in /opt/python/*/bin/python; do - # Smoke test to make sure that our Pythons work, and do indeed detect as - # being manylinux compatible: - $PYTHON $MY_DIR/manylinux-check.py - # Make sure that SSL cert checking works - $PYTHON $MY_DIR/ssl-check.py -done - # We do not need the Python test suites, or indeed the precompiled .pyc and # .pyo files. Partially cribbed from: # https://github.com/docker-library/python/blob/master/3.4/slim/Dockerfile find /opt/_internal -depth \ \( -type d -a -name test -o -name tests \) \ -o \( -type f -a -name '*.pyc' -o -name '*.pyo' \) | xargs rm -rf - -# Fix libc headers to remain compatible with C99 compilers. -find /usr/include/ -type f -exec sed -i 's/\bextern _*inline_*\b/extern __inline __attribute__ ((__gnu_inline__))/g' {} + - -if [ "${DEVTOOLSET_ROOTPATH:-}" != "" ]; then - # remove useless things that have been installed by devtoolset - rm -rf $DEVTOOLSET_ROOTPATH/usr/share/man - find $DEVTOOLSET_ROOTPATH/usr/share/locale -mindepth 1 -maxdepth 1 -not \( -name 'en*' -or -name 'locale.alias' \) | xargs rm -rf -fi -rm -rf /usr/share/backgrounds -# if we updated glibc, we need to strip locales again... -localedef --list-archive | grep -v -i ^en_US.utf8 | xargs localedef --delete-from-archive -mv -f /usr/lib/locale/locale-archive /usr/lib/locale/locale-archive.tmpl -build-locale-archive -find /usr/share/locale -mindepth 1 -maxdepth 1 -not \( -name 'en*' -or -name 'locale.alias' \) | xargs rm -rf -find /usr/local/share/locale -mindepth 1 -maxdepth 1 -not \( -name 'en*' -or -name 'locale.alias' \) | xargs rm -rf -rm -rf /usr/local/share/man diff --git a/docker/build_scripts/build_env.sh b/docker/build_scripts/build_env.sh index 3daa21d7..83db5a65 100644 --- a/docker/build_scripts/build_env.sh +++ b/docker/build_scripts/build_env.sh @@ -4,18 +4,6 @@ PYTHON_DOWNLOAD_URL=https://www.python.org/ftp/python # of the form .. or ..rc CPYTHON_VERSIONS="3.5.10 3.6.12 3.7.9 3.8.7 3.9.1" -PATCHELF_VERSION=0.12 -PATCHELF_HASH=3dca33fb862213b3541350e1da262249959595903f559eae0fbc68966e9c3f56 -PATCHELF_DOWNLOAD_URL=https://github.com/NixOS/patchelf/archive - -AUTOMAKE_ROOT=automake-1.16.2 -AUTOMAKE_HASH=b2f361094b410b4acbf4efba7337bdb786335ca09eb2518635a09fb7319ca5c1 -AUTOMAKE_DOWNLOAD_URL=http://ftp.gnu.org/gnu/automake - -LIBTOOL_ROOT=libtool-2.4.6 -LIBTOOL_HASH=e3bd4d5d3d025a36c21dd6af7ea818a2afcd4dfc1ea5a17b39d7854bcd0c06e3 -LIBTOOL_DOWNLOAD_URL=http://ftp.gnu.org/gnu/libtool - SQLITE_AUTOCONF_ROOT=sqlite-autoconf-3340000 SQLITE_AUTOCONF_HASH=bf6db7fae37d51754737747aaaf413b4d6b3b5fbacd52bdb2d0d6e5b2edd9aee SQLITE_AUTOCONF_DOWNLOAD_URL=https://www.sqlite.org/2020 @@ -24,10 +12,6 @@ CMAKE_VERSION=3.18.3 CMAKE_HASH=2c89f4e30af4914fd6fb5d00f863629812ada848eee4e2d29ec7e456d7fa32e5 CMAKE_DOWNLOAD_URL=https://github.com/Kitware/CMake/releases/download -LIBXCRYPT_VERSION=4.4.17 -LIBXCRYPT_HASH=7665168d0409574a03f7b484682e68334764c29c21ca5df438955a381384ca07 -LIBXCRYPT_DOWNLOAD_URL=https://github.com/besser82/libxcrypt/archive - GIT_ROOT=git-2.30.0 GIT_HASH=d24c4fa2a658318c2e66e25ab67cc30038a35696d2d39e6b12ceccf024de1e5e GIT_DOWNLOAD_URL=https://www.kernel.org/pub/software/scm/git diff --git a/docker/build_scripts/build_env_runtime.sh b/docker/build_scripts/build_env_runtime.sh new file mode 100644 index 00000000..2a1ba389 --- /dev/null +++ b/docker/build_scripts/build_env_runtime.sh @@ -0,0 +1,17 @@ +# source me + +PATCHELF_VERSION=0.12 +PATCHELF_HASH=3dca33fb862213b3541350e1da262249959595903f559eae0fbc68966e9c3f56 +PATCHELF_DOWNLOAD_URL=https://github.com/NixOS/patchelf/archive + +AUTOMAKE_ROOT=automake-1.16.2 +AUTOMAKE_HASH=b2f361094b410b4acbf4efba7337bdb786335ca09eb2518635a09fb7319ca5c1 +AUTOMAKE_DOWNLOAD_URL=http://ftp.gnu.org/gnu/automake + +LIBTOOL_ROOT=libtool-2.4.6 +LIBTOOL_HASH=e3bd4d5d3d025a36c21dd6af7ea818a2afcd4dfc1ea5a17b39d7854bcd0c06e3 +LIBTOOL_DOWNLOAD_URL=http://ftp.gnu.org/gnu/libtool + +LIBXCRYPT_VERSION=4.4.17 +LIBXCRYPT_HASH=7665168d0409574a03f7b484682e68334764c29c21ca5df438955a381384ca07 +LIBXCRYPT_DOWNLOAD_URL=https://github.com/besser82/libxcrypt/archive diff --git a/docker/build_scripts/build_utils.sh b/docker/build_scripts/build_utils.sh index 6c780f7a..4a3e5baa 100755 --- a/docker/build_scripts/build_utils.sh +++ b/docker/build_scripts/build_utils.sh @@ -35,25 +35,11 @@ function do_cpython_build { pushd Python-$py_ver local prefix="/opt/_internal/cpython-${py_ver}" mkdir -p ${prefix}/lib - ./configure --prefix=${prefix} --disable-shared > /dev/null + ./configure --prefix=${prefix} --disable-shared --with-ensurepip=no > /dev/null make -j$(nproc) > /dev/null make -j$(nproc) install > /dev/null popd rm -rf Python-$py_ver - # Some python's install as bin/python3. Make them available as - # bin/python. - if [ -e ${prefix}/bin/python3 ] && [ ! -e ${prefix}/bin/python ]; then - ln -s python3 ${prefix}/bin/python - fi - ${prefix}/bin/python -m ensurepip - if [ -e ${prefix}/bin/pip3 ] && [ ! -e ${prefix}/bin/pip ]; then - ln -s pip3 ${prefix}/bin/pip - fi - # Since we fall back on a canned copy of pip, we might not have - # the latest pip and friends. Upgrade them to make sure. - ${prefix}/bin/pip install -U --require-hashes -r ${MY_DIR}/requirements.txt - local abi_tag=$(${prefix}/bin/python ${MY_DIR}/python-tag-abi-tag.py) - ln -s ${prefix} /opt/python/${abi_tag} } @@ -121,7 +107,7 @@ function build_git { fetch_source ${git_fname}.tar.gz ${GIT_DOWNLOAD_URL} check_sha256sum ${git_fname}.tar.gz ${git_sha256} tar -xzf ${git_fname}.tar.gz - (cd ${git_fname} && make -j$(nproc) install prefix=/usr/local NO_GETTEXT=1 NO_TCLTK=1 > /dev/null) + (cd ${git_fname} && make -j$(nproc) install prefix=/usr/local NO_GETTEXT=1 NO_TCLTK=1 DESTDIR=/manylinux-rootfs > /dev/null) rm -rf ${git_fname} ${git_fname}.tar.gz } diff --git a/docker/build_scripts/finalize.sh b/docker/build_scripts/finalize.sh new file mode 100755 index 00000000..7d6d2273 --- /dev/null +++ b/docker/build_scripts/finalize.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +# Stop at any error, show all commands +set -exuo pipefail + +# Get script directory +MY_DIR=$(dirname "${BASH_SOURCE[0]}") + +# Get build utilities +source $MY_DIR/build_utils.sh + +mkdir /opt/python +for PREFIX in $(find /opt/_internal/ -mindepth 1 -maxdepth 1 -name 'cpython*'); do + # Some python's install as bin/python3. Make them available as + # bin/python. + if [ -e ${PREFIX}/bin/python3 ] && [ ! -e ${PREFIX}/bin/python ]; then + ln -s python3 ${PREFIX}/bin/python + fi + ${PREFIX}/bin/python -m ensurepip + if [ -e ${PREFIX}/bin/pip3 ] && [ ! -e ${PREFIX}/bin/pip ]; then + ln -s pip3 ${PREFIX}/bin/pip + fi + # Since we fall back on a canned copy of pip, we might not have + # the latest pip and friends. Upgrade them to make sure. + ${PREFIX}/bin/pip install -U --require-hashes -r ${MY_DIR}/requirements.txt + # Create a symlink to PREFIX using the ABI_TAG in /opt/python/ + ABI_TAG=$(${PREFIX}/bin/python ${MY_DIR}/python-tag-abi-tag.py) + ln -s ${PREFIX} /opt/python/${ABI_TAG} +done + +# Create venv for auditwheel & certifi +TOOLS_PATH=/opt/_internal/tools +/opt/python/cp37-cp37m/bin/python -m venv $TOOLS_PATH +source $TOOLS_PATH/bin/activate + +# Install default packages +pip install -U --require-hashes -r $MY_DIR/requirements.txt +# Install certifi and auditwheel +pip install -U --require-hashes -r $MY_DIR/requirements-tools.txt + +# Make auditwheel available in PATH +ln -s $TOOLS_PATH/bin/auditwheel /usr/local/bin/auditwheel + +# Our openssl doesn't know how to find the system CA trust store +# (https://github.com/pypa/manylinux/issues/53) +# And it's not clear how up-to-date that is anyway +# So let's just use the same one pip and everyone uses +ln -s $(python -c 'import certifi; print(certifi.where())') /opt/_internal/certs.pem +# If you modify this line you also have to modify the versions in the Dockerfiles: +export SSL_CERT_FILE=/opt/_internal/certs.pem + +# Deactivate the tools virtual environment +deactivate + + +for PYTHON in /opt/python/*/bin/python; do + # Smoke test to make sure that our Pythons work, and do indeed detect as + # being manylinux compatible: + $PYTHON $MY_DIR/manylinux-check.py + # Make sure that SSL cert checking works + $PYTHON $MY_DIR/ssl-check.py +done + +# We do not need the precompiled .pyc and .pyo files. Partially cribbed from: +# https://github.com/docker-library/python/blob/master/3.4/slim/Dockerfile +find /opt/_internal -type f -a \( -name '*.pyc' -o -name '*.pyo' \) | xargs rm -rf + +# remove cache +rm -rf /root/.cache diff --git a/docker/build_scripts/install-runtime-packages.sh b/docker/build_scripts/install-runtime-packages.sh new file mode 100755 index 00000000..924ed3ce --- /dev/null +++ b/docker/build_scripts/install-runtime-packages.sh @@ -0,0 +1,141 @@ +#!/bin/bash +# Install packages that will be needed at runtime + +# Stop at any error, show all commands +set -exuo pipefail + +# Set build environment variables +MY_DIR=$(dirname "${BASH_SOURCE[0]}") +source $MY_DIR/build_env_runtime.sh + +# Get build utilities +source $MY_DIR/build_utils.sh + +# Libraries that are allowed as part of the manylinux2014 profile +# Extract from PEP: https://www.python.org/dev/peps/pep-0599/#the-manylinux2014-policy +# On RPM-based systems, they are provided by these packages: +# Package: Libraries +# glib2: libglib-2.0.so.0, libgthread-2.0.so.0, libgobject-2.0.so.0 +# glibc: libresolv.so.2, libutil.so.1, libnsl.so.1, librt.so.1, libpthread.so.0, libdl.so.2, libm.so.6, libc.so.6 +# libICE: libICE.so.6 +# libX11: libX11.so.6 +# libXext: libXext.so.6 +# libXrender: libXrender.so.1 +# libgcc: libgcc_s.so.1 +# libstdc++: libstdc++.so.6 +# mesa: libGL.so.1 +# +# PEP is missing the package for libSM.so.6 for RPM based system + +# MANYLINUX_DEPS: Install development packages (except for libgcc which is provided by gcc install) +# RUNTIME_DEPS: Runtime dependencies. c.f. build.sh +if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then + MANYLINUX_DEPS="glibc-devel libstdc++-devel glib2-devel libX11-devel libXext-devel libXrender-devel mesa-libGL-devel libICE-devel libSM-devel" + RUNTIME_DEPS="zlib bzip2 expat ncurses readline tk gdbm libdb libpcap xz openssl keyutils-libs libkadm5 libcom_err libidn libcurl uuid libffi" +else + echo "Unsupported policy: '${AUDITWHEEL_POLICY}'" + exit 1 +fi + +BASETOOLS="autoconf automake bison bzip2 diffutils file make patch unzip" +if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then + PACKAGE_MANAGER=yum + BASETOOLS="${BASETOOLS} which" + # See https://unix.stackexchange.com/questions/41784/can-yum-express-a-preference-for-x86-64-over-i386-packages + echo "multilib_policy=best" >> /etc/yum.conf + # Error out if requested packages do not exist + echo "skip_missing_names_on_install=False" >> /etc/yum.conf + # Make sure that locale will not be removed + sed -i '/^override_install_langs=/d' /etc/yum.conf + yum -y update + yum -y install yum-utils curl + yum-config-manager --enable extras + TOOLCHAIN_DEPS="devtoolset-9-binutils devtoolset-9-gcc devtoolset-9-gcc-c++ devtoolset-9-gcc-gfortran" + if [ "${AUDITWHEEL_ARCH}" == "x86_64" ]; then + # Software collection (for devtoolset-9) + yum -y install centos-release-scl-rh + # EPEL support (for yasm) + yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm + TOOLCHAIN_DEPS="${TOOLCHAIN_DEPS} yasm" + elif [ "${AUDITWHEEL_ARCH}" == "aarch64" ] || [ "${AUDITWHEEL_ARCH}" == "ppc64le" ] || [ "${AUDITWHEEL_ARCH}" == "s390x" ]; then + # Software collection (for devtoolset-9) + yum -y install centos-release-scl-rh + elif [ "${AUDITWHEEL_ARCH}" == "i686" ]; then + # No yasm on i686 + # Install mayeut/devtoolset-9 repo to get devtoolset-9 + curl -fsSLo /etc/yum.repos.d/mayeut-devtoolset-9.repo https://copr.fedorainfracloud.org/coprs/mayeut/devtoolset-9/repo/custom-1/mayeut-devtoolset-9-custom-1.repo + fi +else + echo "Unsupported policy: '${AUDITWHEEL_POLICY}'" + exit 1 +fi + +if ! which localedef &> /dev/null; then + # somebody messed up glibc-common package to squeeze image size, reinstall the package + if [ ${PACKAGE_MANAGER} == yum ]; then + yum -y reinstall glibc-common + else + echo "Not implemented" + exit 1 + fi +fi + +# upgrading glibc-common can end with removal on en_US.UTF-8 locale +localedef -i en_US -f UTF-8 en_US.UTF-8 + +if [ ${PACKAGE_MANAGER} == yum ]; then + yum -y install ${BASETOOLS} ${TOOLCHAIN_DEPS} ${MANYLINUX_DEPS} ${RUNTIME_DEPS} + yum clean all + rm -rf /var/cache/yum +else + echo "Not implemented" + exit 1 +fi + + +# Fix libc headers to remain compatible with C99 compilers. +find /usr/include/ -type f -exec sed -i 's/\bextern _*inline_*\b/extern __inline __attribute__ ((__gnu_inline__))/g' {} + + +if [ "${DEVTOOLSET_ROOTPATH:-}" != "" ]; then + # remove useless things that have been installed by devtoolset + rm -rf $DEVTOOLSET_ROOTPATH/usr/share/man + find $DEVTOOLSET_ROOTPATH/usr/share/locale -mindepth 1 -maxdepth 1 -not \( -name 'en*' -or -name 'locale.alias' \) | xargs rm -rf +fi + +rm -rf /usr/share/backgrounds + +# if we updated glibc, we need to strip locales again... +if localedef --list-archive | grep -sq -v -i ^en_US.utf8; then + localedef --list-archive | grep -v -i ^en_US.utf8 | xargs localedef --delete-from-archive +fi +if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then + mv -f /usr/lib/locale/locale-archive /usr/lib/locale/locale-archive.tmpl + build-locale-archive +fi + +find /usr/share/locale -mindepth 1 -maxdepth 1 -not \( -name 'en*' -or -name 'locale.alias' \) | xargs rm -rf +if [ -d /usr/local/share/locale ]; then + find /usr/local/share/locale -mindepth 1 -maxdepth 1 -not \( -name 'en*' -or -name 'locale.alias' \) | xargs rm -rf +fi +if [ -d /usr/local/share/man ]; then + rm -rf /usr/local/share/man +fi + + +# Install newest automake +build_automake $AUTOMAKE_ROOT $AUTOMAKE_HASH +automake --version + +# Install newest libtool +build_libtool $LIBTOOL_ROOT $LIBTOOL_HASH +libtool --version + +# Install patchelf (latest with unreleased bug fixes) and apply our patches +build_patchelf $PATCHELF_VERSION $PATCHELF_HASH + +# Install libcrypt.so.1 and libcrypt.so.2 +build_libxcrypt "$LIBXCRYPT_DOWNLOAD_URL" "$LIBXCRYPT_VERSION" "$LIBXCRYPT_HASH" + +# Strip what we can -- and ignore errors, because this just attempts to strip +# *everything*, including non-ELF files: +find /usr/local -type f -print0 | xargs -0 -n1 strip --strip-unneeded 2>/dev/null || true