diff --git a/docker/Dockerfile b/docker/Dockerfile index f9c10b0c..24e46745 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -2,11 +2,11 @@ ARG BASEIMAGE=amd64/centos:7 ARG POLICY=manylinux2014 ARG PLATFORM=x86_64 -ARG DEVTOOLSET_ROOTPATH= -ARG LD_LIBRARY_PATH_ARG= -ARG PREPEND_PATH= +ARG DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-9/root +ARG LD_LIBRARY_PATH_ARG=${DEVTOOLSET_ROOTPATH}/usr/lib64:${DEVTOOLSET_ROOTPATH}/usr/lib:${DEVTOOLSET_ROOTPATH}/usr/lib64/dyninst:${DEVTOOLSET_ROOTPATH}/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib +ARG PREPEND_PATH=${DEVTOOLSET_ROOTPATH}/usr/bin: -FROM $BASEIMAGE +FROM $BASEIMAGE AS runtime_base ARG POLICY ARG PLATFORM ARG DEVTOOLSET_ROOTPATH @@ -22,14 +22,36 @@ ENV PATH=${PREPEND_PATH}${PATH} ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig # setup entrypoint, this will wrap commands with `linux32` with i686 images -COPY build_scripts/install-entrypoint.sh /build_scripts/install-entrypoint.sh +COPY build_scripts/install-entrypoint.sh /build_scripts/ RUN bash /build_scripts/install-entrypoint.sh && rm -rf build_scripts COPY manylinux-entrypoint /usr/local/bin/manylinux-entrypoint ENTRYPOINT ["manylinux-entrypoint"] -COPY build_scripts /build_scripts + +COPY build_scripts/install-runtime-packages.sh /build_scripts/ +COPY build_scripts/build_utils.sh /build_scripts/ +COPY build_scripts/build_env_runtime.sh /build_scripts/ +RUN manylinux-entrypoint /build_scripts/install-runtime-packages.sh && rm -rf /build_scripts + +FROM runtime_base AS build_base +COPY build_scripts/*pubkey*.txt /build_scripts/ +COPY build_scripts/build_utils.sh /build_scripts/ +COPY build_scripts/build.sh /build_scripts/ +COPY build_scripts/build_env.sh /build_scripts/ RUN manylinux-entrypoint bash build_scripts/build.sh && rm -rf build_scripts +FROM runtime_base +COPY --from=build_base /manylinux-rootfs / +COPY --from=build_base /opt/_internal /opt/_internal/ +COPY build_scripts/build_utils.sh /build_scripts/build_utils.sh +COPY build_scripts/finalize.sh /build_scripts/finalize.sh +COPY build_scripts/python-tag-abi-tag.py /build_scripts/python-tag-abi-tag.py +COPY build_scripts/ssl-check.py /build_scripts/ssl-check.py +COPY build_scripts/manylinux-check.py /build_scripts/manylinux-check.py +COPY build_scripts/requirements.txt /build_scripts/requirements.txt +COPY build_scripts/requirements-tools.txt /build_scripts/requirements-tools.txt +RUN manylinux-entrypoint /build_scripts/finalize.sh && rm -rf /build_scripts + ENV SSL_CERT_FILE=/opt/_internal/certs.pem CMD ["/bin/bash"] diff --git a/docker/build_scripts/build.sh b/docker/build_scripts/build.sh index 15d727ff..ede3473b 100755 --- a/docker/build_scripts/build.sh +++ b/docker/build_scripts/build.sh @@ -6,119 +6,40 @@ set -ex # Set build environment variables MY_DIR=$(dirname "${BASH_SOURCE[0]}") -. $MY_DIR/build_env.sh - -# Dependencies for compiling Python that we want to remove from -# the final image after compiling Python -PYTHON_COMPILE_DEPS="zlib-devel bzip2-devel expat-devel ncurses-devel readline-devel tk-devel gdbm-devel libdb-devel libpcap-devel xz-devel openssl-devel keyutils-libs-devel krb5-devel libcom_err-devel libidn-devel curl-devel perl-devel" - -# Libraries that are allowed as part of the manylinux2014 profile -# Extract from PEP: https://www.python.org/dev/peps/pep-0599/#the-manylinux2014-policy -# On RPM-based systems, they are provided by these packages: -# Package: Libraries -# glib2: libglib-2.0.so.0, libgthread-2.0.so.0, libgobject-2.0.so.0 -# glibc: libresolv.so.2, libutil.so.1, libnsl.so.1, librt.so.1, libpthread.so.0, libdl.so.2, libm.so.6, libc.so.6 -# libICE: libICE.so.6 -# libX11: libX11.so.6 -# libXext: libXext.so.6 -# libXrender: libXrender.so.1 -# libgcc: libgcc_s.so.1 -# libstdc++: libstdc++.so.6 -# mesa: libGL.so.1 -# -# PEP is missing the package for libSM.so.6 for RPM based system -# Install development packages (except for libgcc which is provided by gcc install) -MANYLINUX_DEPS="glibc-devel libstdc++-devel glib2-devel libX11-devel libXext-devel libXrender-devel mesa-libGL-devel libICE-devel libSM-devel" - -CMAKE_DEPS="openssl-devel zlib-devel libcurl-devel" +source $MY_DIR/build_env.sh # Get build utilities source $MY_DIR/build_utils.sh -# See https://unix.stackexchange.com/questions/41784/can-yum-express-a-preference-for-x86-64-over-i386-packages -echo "multilib_policy=best" >> /etc/yum.conf -# Error out if requested packages do not exist -echo "skip_missing_names_on_install=False" >> /etc/yum.conf -# Make sure that locale will not be removed -sed -i '/^override_install_langs=/d' /etc/yum.conf - -# https://hub.docker.com/_/centos/ -# "Additionally, images with minor version tags that correspond to install -# media are also offered. These images DO NOT recieve updates as they are -# intended to match installation iso contents. If you choose to use these -# images it is highly recommended that you include RUN yum -y update && yum -# clean all in your Dockerfile, or otherwise address any potential security -# concerns." -# Decided not to clean at this point: https://github.com/pypa/manylinux/pull/129 -yum -y update -yum -y install yum-utils curl -yum-config-manager --enable extras - -if ! which localedef &> /dev/null; then - # somebody messed up glibc-common package to squeeze image size, reinstall the package - yum -y reinstall glibc-common -fi - -# upgrading glibc-common can end with removal on en_US.UTF-8 locale -localedef -i en_US -f UTF-8 en_US.UTF-8 - -TOOLCHAIN_DEPS="devtoolset-9-binutils devtoolset-9-gcc devtoolset-9-gcc-c++ devtoolset-9-gcc-gfortran" -if [ "${AUDITWHEEL_ARCH}" == "x86_64" ]; then - # Software collection (for devtoolset-9) - yum -y install centos-release-scl-rh - # EPEL support (for yasm) - yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm - YASM=yasm -elif [ "${AUDITWHEEL_ARCH}" == "aarch64" ] || [ "${AUDITWHEEL_ARCH}" == "ppc64le" ] || [ "${AUDITWHEEL_ARCH}" == "s390x" ]; then - # Software collection (for devtoolset-9) - yum -y install centos-release-scl-rh -elif [ "${AUDITWHEEL_ARCH}" == "i686" ]; then - # No yasm on i686 - # Install mayeut/devtoolset-9 repo to get devtoolset-9 - curl -fsSLo /etc/yum.repos.d/mayeut-devtoolset-9.repo https://copr.fedorainfracloud.org/coprs/mayeut/devtoolset-9/repo/custom-1/mayeut-devtoolset-9-custom-1.repo -fi +# Dependencies for compiling Python that we want to remove from +# the final image after compiling Python +PYTHON_COMPILE_DEPS="zlib-devel bzip2-devel expat-devel ncurses-devel readline-devel tk-devel gdbm-devel libdb-devel libpcap-devel xz-devel openssl-devel keyutils-libs-devel krb5-devel libcom_err-devel libidn-devel curl-devel perl-devel libffi-devel kernel-devel" +CMAKE_DEPS="openssl-devel zlib-devel libcurl-devel" # Development tools and libraries -yum -y install \ - autoconf \ - automake \ - bison \ - bzip2 \ - ${TOOLCHAIN_DEPS} \ - diffutils \ - gettext \ - file \ - kernel-devel \ - libffi-devel \ - make \ - patch \ - unzip \ - which \ - ${YASM} \ - ${PYTHON_COMPILE_DEPS} \ - ${CMAKE_DEPS} +yum -y install ${PYTHON_COMPILE_DEPS} ${CMAKE_DEPS} # Install git build_git $GIT_ROOT $GIT_HASH -git version - -# Install newest automake -build_automake $AUTOMAKE_ROOT $AUTOMAKE_HASH -automake --version - -# Install newest libtool -build_libtool $LIBTOOL_ROOT $LIBTOOL_HASH -libtool --version +/manylinux-rootfs/usr/local/bin/git version # Install a more recent SQLite3 curl -fsSLO $SQLITE_AUTOCONF_DOWNLOAD_URL/$SQLITE_AUTOCONF_VERSION.tar.gz check_sha256sum $SQLITE_AUTOCONF_VERSION.tar.gz $SQLITE_AUTOCONF_HASH tar xfz $SQLITE_AUTOCONF_VERSION.tar.gz cd $SQLITE_AUTOCONF_VERSION -do_standard_install +DESTDIR=/sqlite3 do_standard_install cd .. rm -rf $SQLITE_AUTOCONF_VERSION* -rm /usr/local/lib/libsqlite3.a +rm /sqlite3/usr/local/lib/libsqlite3.a +# Install for build +cp -rf /sqlite3/* / +# Clean-up for runtime +rm -rf /sqlite3/usr/local/bin /sqlite3/usr/local/include /sqlite3/usr/local/lib/pkg-config /sqlite3/usr/local/share +# Install for runtime +cp -rf /sqlite3/* /manylinux-rootfs/ +# clean-up +rm -rf /sqlite3 # Install a recent version of cmake3 curl -L -O $CMAKE_DOWNLOAD_URL/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}.tar.gz @@ -127,63 +48,16 @@ tar -xzf cmake-${CMAKE_VERSION}.tar.gz cd cmake-${CMAKE_VERSION} ./bootstrap --system-curl --parallel=$(nproc) make -j$(nproc) -make install +make install DESTDIR=/manylinux-rootfs cd .. -rm -rf cmake-${CMAKE_VERSION} - -# Install libcrypt.so.1 and libcrypt.so.2 -build_libxcrypt "$LIBXCRYPT_DOWNLOAD_URL" "$LIBXCRYPT_VERSION" "$LIBXCRYPT_HASH" +rm -rf cmake-${CMAKE_VERSION}* # Compile the latest Python releases. # (In order to have a proper SSL module, Python is compiled # against a recent openssl [see env vars above], which is linked # statically. -mkdir -p /opt/python build_cpythons $CPYTHON_VERSIONS -# Create venv for auditwheel & certifi -TOOLS_PATH=/opt/_internal/tools -/opt/python/cp37-cp37m/bin/python -m venv $TOOLS_PATH -source $TOOLS_PATH/bin/activate - -# Install default packages -pip install -U --require-hashes -r $MY_DIR/requirements.txt -# Install certifi and auditwheel -pip install -U --require-hashes -r $MY_DIR/requirements-tools.txt - -# Make auditwheel available in PATH -ln -s $TOOLS_PATH/bin/auditwheel /usr/local/bin/auditwheel - -# Our openssl doesn't know how to find the system CA trust store -# (https://github.com/pypa/manylinux/issues/53) -# And it's not clear how up-to-date that is anyway -# So let's just use the same one pip and everyone uses -ln -s $(python -c 'import certifi; print(certifi.where())') /opt/_internal/certs.pem -# If you modify this line you also have to modify the versions in the Dockerfiles: -export SSL_CERT_FILE=/opt/_internal/certs.pem - -# Deactivate the tools virtual environment -deactivate - -# Install patchelf (latest with unreleased bug fixes) and apply our patches -build_patchelf $PATCHELF_VERSION $PATCHELF_HASH - -# Clean up development headers and other unnecessary stuff for -# final image -yum -y erase \ - avahi \ - bitstream-vera-fonts \ - freetype \ - gettext \ - gtk2 \ - hicolor-icon-theme \ - libX11 \ - wireless-tools \ - ${PYTHON_COMPILE_DEPS} > /dev/null 2>&1 -yum -y install ${MANYLINUX_DEPS} -yum -y clean all > /dev/null 2>&1 -yum list installed - # we don't need libpython*.a, and they're many megabytes find /opt/_internal -name '*.a' -print0 | xargs -0 rm -f @@ -191,37 +65,12 @@ find /opt/_internal -name '*.a' -print0 | xargs -0 rm -f # *everything*, including non-ELF files: find /opt/_internal -type f -print0 \ | xargs -0 -n1 strip --strip-unneeded 2>/dev/null || true -find /usr/local -type f -print0 \ +find /manylinux-rootfs -type f -print0 \ | xargs -0 -n1 strip --strip-unneeded 2>/dev/null || true -for PYTHON in /opt/python/*/bin/python; do - # Smoke test to make sure that our Pythons work, and do indeed detect as - # being manylinux compatible: - $PYTHON $MY_DIR/manylinux-check.py - # Make sure that SSL cert checking works - $PYTHON $MY_DIR/ssl-check.py -done - # We do not need the Python test suites, or indeed the precompiled .pyc and # .pyo files. Partially cribbed from: # https://github.com/docker-library/python/blob/master/3.4/slim/Dockerfile find /opt/_internal -depth \ \( -type d -a -name test -o -name tests \) \ -o \( -type f -a -name '*.pyc' -o -name '*.pyo' \) | xargs rm -rf - -# Fix libc headers to remain compatible with C99 compilers. -find /usr/include/ -type f -exec sed -i 's/\bextern _*inline_*\b/extern __inline __attribute__ ((__gnu_inline__))/g' {} + - -if [ "${DEVTOOLSET_ROOTPATH:-}" != "" ]; then - # remove useless things that have been installed by devtoolset - rm -rf $DEVTOOLSET_ROOTPATH/usr/share/man - find $DEVTOOLSET_ROOTPATH/usr/share/locale -mindepth 1 -maxdepth 1 -not \( -name 'en*' -or -name 'locale.alias' \) | xargs rm -rf -fi -rm -rf /usr/share/backgrounds -# if we updated glibc, we need to strip locales again... -localedef --list-archive | grep -v -i ^en_US.utf8 | xargs localedef --delete-from-archive -mv -f /usr/lib/locale/locale-archive /usr/lib/locale/locale-archive.tmpl -build-locale-archive -find /usr/share/locale -mindepth 1 -maxdepth 1 -not \( -name 'en*' -or -name 'locale.alias' \) | xargs rm -rf -find /usr/local/share/locale -mindepth 1 -maxdepth 1 -not \( -name 'en*' -or -name 'locale.alias' \) | xargs rm -rf -rm -rf /usr/local/share/man diff --git a/docker/build_scripts/build_env.sh b/docker/build_scripts/build_env.sh index c172ca31..6bbdb454 100644 --- a/docker/build_scripts/build_env.sh +++ b/docker/build_scripts/build_env.sh @@ -4,17 +4,6 @@ PYTHON_DOWNLOAD_URL=https://www.python.org/ftp/python # of the form .. or ..rc CPYTHON_VERSIONS="3.5.10 3.6.12 3.7.9 3.8.6 3.9.1" -PATCHELF_VERSION=0.12 -PATCHELF_HASH=3dca33fb862213b3541350e1da262249959595903f559eae0fbc68966e9c3f56 - -AUTOMAKE_ROOT=automake-1.16.2 -AUTOMAKE_HASH=b2f361094b410b4acbf4efba7337bdb786335ca09eb2518635a09fb7319ca5c1 -AUTOMAKE_DOWNLOAD_URL=http://ftp.gnu.org/gnu/automake - -LIBTOOL_ROOT=libtool-2.4.6 -LIBTOOL_HASH=e3bd4d5d3d025a36c21dd6af7ea818a2afcd4dfc1ea5a17b39d7854bcd0c06e3 -LIBTOOL_DOWNLOAD_URL=http://ftp.gnu.org/gnu/libtool - SQLITE_AUTOCONF_VERSION=sqlite-autoconf-3330000 SQLITE_AUTOCONF_HASH=106a2c48c7f75a298a7557bcc0d5f4f454e5b43811cc738b7ca294d6956bbb15 SQLITE_AUTOCONF_DOWNLOAD_URL=https://www.sqlite.org/2020 @@ -23,10 +12,6 @@ CMAKE_VERSION=3.18.3 CMAKE_HASH=2c89f4e30af4914fd6fb5d00f863629812ada848eee4e2d29ec7e456d7fa32e5 CMAKE_DOWNLOAD_URL=https://github.com/Kitware/CMake/releases/download -LIBXCRYPT_VERSION=4.4.17 -LIBXCRYPT_DOWNLOAD_URL=https://codeload.github.com/besser82/libxcrypt/tar.gz -LIBXCRYPT_HASH=7665168d0409574a03f7b484682e68334764c29c21ca5df438955a381384ca07 - GIT_ROOT=git-2.29.1 GIT_HASH=728845a66103d8d1572a656123c2c09d7aa4c0ab8f4c3dc3911e14e18c85ee67 GIT_DOWNLOAD_URL=https://www.kernel.org/pub/software/scm/git diff --git a/docker/build_scripts/build_env_runtime.sh b/docker/build_scripts/build_env_runtime.sh new file mode 100644 index 00000000..aa80c1a3 --- /dev/null +++ b/docker/build_scripts/build_env_runtime.sh @@ -0,0 +1,16 @@ +# source me + +PATCHELF_VERSION=0.12 +PATCHELF_HASH=3dca33fb862213b3541350e1da262249959595903f559eae0fbc68966e9c3f56 + +AUTOMAKE_ROOT=automake-1.16.2 +AUTOMAKE_HASH=b2f361094b410b4acbf4efba7337bdb786335ca09eb2518635a09fb7319ca5c1 +AUTOMAKE_DOWNLOAD_URL=http://ftp.gnu.org/gnu/automake + +LIBTOOL_ROOT=libtool-2.4.6 +LIBTOOL_HASH=e3bd4d5d3d025a36c21dd6af7ea818a2afcd4dfc1ea5a17b39d7854bcd0c06e3 +LIBTOOL_DOWNLOAD_URL=http://ftp.gnu.org/gnu/libtool + +LIBXCRYPT_VERSION=4.4.17 +LIBXCRYPT_DOWNLOAD_URL=https://codeload.github.com/besser82/libxcrypt/tar.gz +LIBXCRYPT_HASH=7665168d0409574a03f7b484682e68334764c29c21ca5df438955a381384ca07 diff --git a/docker/build_scripts/build_utils.sh b/docker/build_scripts/build_utils.sh index 5a36434e..2eda56f5 100755 --- a/docker/build_scripts/build_utils.sh +++ b/docker/build_scripts/build_utils.sh @@ -35,25 +35,11 @@ function do_cpython_build { pushd Python-$py_ver local prefix="/opt/_internal/cpython-${py_ver}" mkdir -p ${prefix}/lib - ./configure --prefix=${prefix} --disable-shared > /dev/null + ./configure --prefix=${prefix} --disable-shared --with-ensurepip=no > /dev/null make -j$(nproc) > /dev/null make -j$(nproc) install > /dev/null popd rm -rf Python-$py_ver - # Some python's install as bin/python3. Make them available as - # bin/python. - if [ -e ${prefix}/bin/python3 ] && [ ! -e ${prefix}/bin/python ]; then - ln -s python3 ${prefix}/bin/python - fi - ${prefix}/bin/python -m ensurepip - if [ -e ${prefix}/bin/pip3 ] && [ ! -e ${prefix}/bin/pip ]; then - ln -s pip3 ${prefix}/bin/pip - fi - # Since we fall back on a canned copy of pip, we might not have - # the latest pip and friends. Upgrade them to make sure. - ${prefix}/bin/pip install -U --require-hashes -r ${MY_DIR}/requirements.txt - local abi_tag=$(${prefix}/bin/python ${MY_DIR}/python-tag-abi-tag.py) - ln -s ${prefix} /opt/python/${abi_tag} } @@ -121,7 +107,7 @@ function build_git { fetch_source ${git_fname}.tar.gz ${GIT_DOWNLOAD_URL} check_sha256sum ${git_fname}.tar.gz ${git_sha256} tar -xzf ${git_fname}.tar.gz - (cd ${git_fname} && make -j$(nproc) install prefix=/usr/local NO_GETTEXT=1 NO_TCLTK=1 > /dev/null) + (cd ${git_fname} && make -j$(nproc) install prefix=/usr/local NO_GETTEXT=1 NO_TCLTK=1 DESTDIR=/manylinux-rootfs > /dev/null) rm -rf ${git_fname} ${git_fname}.tar.gz } diff --git a/docker/build_scripts/finalize.sh b/docker/build_scripts/finalize.sh new file mode 100755 index 00000000..7d6d2273 --- /dev/null +++ b/docker/build_scripts/finalize.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +# Stop at any error, show all commands +set -exuo pipefail + +# Get script directory +MY_DIR=$(dirname "${BASH_SOURCE[0]}") + +# Get build utilities +source $MY_DIR/build_utils.sh + +mkdir /opt/python +for PREFIX in $(find /opt/_internal/ -mindepth 1 -maxdepth 1 -name 'cpython*'); do + # Some python's install as bin/python3. Make them available as + # bin/python. + if [ -e ${PREFIX}/bin/python3 ] && [ ! -e ${PREFIX}/bin/python ]; then + ln -s python3 ${PREFIX}/bin/python + fi + ${PREFIX}/bin/python -m ensurepip + if [ -e ${PREFIX}/bin/pip3 ] && [ ! -e ${PREFIX}/bin/pip ]; then + ln -s pip3 ${PREFIX}/bin/pip + fi + # Since we fall back on a canned copy of pip, we might not have + # the latest pip and friends. Upgrade them to make sure. + ${PREFIX}/bin/pip install -U --require-hashes -r ${MY_DIR}/requirements.txt + # Create a symlink to PREFIX using the ABI_TAG in /opt/python/ + ABI_TAG=$(${PREFIX}/bin/python ${MY_DIR}/python-tag-abi-tag.py) + ln -s ${PREFIX} /opt/python/${ABI_TAG} +done + +# Create venv for auditwheel & certifi +TOOLS_PATH=/opt/_internal/tools +/opt/python/cp37-cp37m/bin/python -m venv $TOOLS_PATH +source $TOOLS_PATH/bin/activate + +# Install default packages +pip install -U --require-hashes -r $MY_DIR/requirements.txt +# Install certifi and auditwheel +pip install -U --require-hashes -r $MY_DIR/requirements-tools.txt + +# Make auditwheel available in PATH +ln -s $TOOLS_PATH/bin/auditwheel /usr/local/bin/auditwheel + +# Our openssl doesn't know how to find the system CA trust store +# (https://github.com/pypa/manylinux/issues/53) +# And it's not clear how up-to-date that is anyway +# So let's just use the same one pip and everyone uses +ln -s $(python -c 'import certifi; print(certifi.where())') /opt/_internal/certs.pem +# If you modify this line you also have to modify the versions in the Dockerfiles: +export SSL_CERT_FILE=/opt/_internal/certs.pem + +# Deactivate the tools virtual environment +deactivate + + +for PYTHON in /opt/python/*/bin/python; do + # Smoke test to make sure that our Pythons work, and do indeed detect as + # being manylinux compatible: + $PYTHON $MY_DIR/manylinux-check.py + # Make sure that SSL cert checking works + $PYTHON $MY_DIR/ssl-check.py +done + +# We do not need the precompiled .pyc and .pyo files. Partially cribbed from: +# https://github.com/docker-library/python/blob/master/3.4/slim/Dockerfile +find /opt/_internal -type f -a \( -name '*.pyc' -o -name '*.pyo' \) | xargs rm -rf + +# remove cache +rm -rf /root/.cache diff --git a/docker/build_scripts/install-runtime-packages.sh b/docker/build_scripts/install-runtime-packages.sh new file mode 100755 index 00000000..924ed3ce --- /dev/null +++ b/docker/build_scripts/install-runtime-packages.sh @@ -0,0 +1,141 @@ +#!/bin/bash +# Install packages that will be needed at runtime + +# Stop at any error, show all commands +set -exuo pipefail + +# Set build environment variables +MY_DIR=$(dirname "${BASH_SOURCE[0]}") +source $MY_DIR/build_env_runtime.sh + +# Get build utilities +source $MY_DIR/build_utils.sh + +# Libraries that are allowed as part of the manylinux2014 profile +# Extract from PEP: https://www.python.org/dev/peps/pep-0599/#the-manylinux2014-policy +# On RPM-based systems, they are provided by these packages: +# Package: Libraries +# glib2: libglib-2.0.so.0, libgthread-2.0.so.0, libgobject-2.0.so.0 +# glibc: libresolv.so.2, libutil.so.1, libnsl.so.1, librt.so.1, libpthread.so.0, libdl.so.2, libm.so.6, libc.so.6 +# libICE: libICE.so.6 +# libX11: libX11.so.6 +# libXext: libXext.so.6 +# libXrender: libXrender.so.1 +# libgcc: libgcc_s.so.1 +# libstdc++: libstdc++.so.6 +# mesa: libGL.so.1 +# +# PEP is missing the package for libSM.so.6 for RPM based system + +# MANYLINUX_DEPS: Install development packages (except for libgcc which is provided by gcc install) +# RUNTIME_DEPS: Runtime dependencies. c.f. build.sh +if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then + MANYLINUX_DEPS="glibc-devel libstdc++-devel glib2-devel libX11-devel libXext-devel libXrender-devel mesa-libGL-devel libICE-devel libSM-devel" + RUNTIME_DEPS="zlib bzip2 expat ncurses readline tk gdbm libdb libpcap xz openssl keyutils-libs libkadm5 libcom_err libidn libcurl uuid libffi" +else + echo "Unsupported policy: '${AUDITWHEEL_POLICY}'" + exit 1 +fi + +BASETOOLS="autoconf automake bison bzip2 diffutils file make patch unzip" +if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then + PACKAGE_MANAGER=yum + BASETOOLS="${BASETOOLS} which" + # See https://unix.stackexchange.com/questions/41784/can-yum-express-a-preference-for-x86-64-over-i386-packages + echo "multilib_policy=best" >> /etc/yum.conf + # Error out if requested packages do not exist + echo "skip_missing_names_on_install=False" >> /etc/yum.conf + # Make sure that locale will not be removed + sed -i '/^override_install_langs=/d' /etc/yum.conf + yum -y update + yum -y install yum-utils curl + yum-config-manager --enable extras + TOOLCHAIN_DEPS="devtoolset-9-binutils devtoolset-9-gcc devtoolset-9-gcc-c++ devtoolset-9-gcc-gfortran" + if [ "${AUDITWHEEL_ARCH}" == "x86_64" ]; then + # Software collection (for devtoolset-9) + yum -y install centos-release-scl-rh + # EPEL support (for yasm) + yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm + TOOLCHAIN_DEPS="${TOOLCHAIN_DEPS} yasm" + elif [ "${AUDITWHEEL_ARCH}" == "aarch64" ] || [ "${AUDITWHEEL_ARCH}" == "ppc64le" ] || [ "${AUDITWHEEL_ARCH}" == "s390x" ]; then + # Software collection (for devtoolset-9) + yum -y install centos-release-scl-rh + elif [ "${AUDITWHEEL_ARCH}" == "i686" ]; then + # No yasm on i686 + # Install mayeut/devtoolset-9 repo to get devtoolset-9 + curl -fsSLo /etc/yum.repos.d/mayeut-devtoolset-9.repo https://copr.fedorainfracloud.org/coprs/mayeut/devtoolset-9/repo/custom-1/mayeut-devtoolset-9-custom-1.repo + fi +else + echo "Unsupported policy: '${AUDITWHEEL_POLICY}'" + exit 1 +fi + +if ! which localedef &> /dev/null; then + # somebody messed up glibc-common package to squeeze image size, reinstall the package + if [ ${PACKAGE_MANAGER} == yum ]; then + yum -y reinstall glibc-common + else + echo "Not implemented" + exit 1 + fi +fi + +# upgrading glibc-common can end with removal on en_US.UTF-8 locale +localedef -i en_US -f UTF-8 en_US.UTF-8 + +if [ ${PACKAGE_MANAGER} == yum ]; then + yum -y install ${BASETOOLS} ${TOOLCHAIN_DEPS} ${MANYLINUX_DEPS} ${RUNTIME_DEPS} + yum clean all + rm -rf /var/cache/yum +else + echo "Not implemented" + exit 1 +fi + + +# Fix libc headers to remain compatible with C99 compilers. +find /usr/include/ -type f -exec sed -i 's/\bextern _*inline_*\b/extern __inline __attribute__ ((__gnu_inline__))/g' {} + + +if [ "${DEVTOOLSET_ROOTPATH:-}" != "" ]; then + # remove useless things that have been installed by devtoolset + rm -rf $DEVTOOLSET_ROOTPATH/usr/share/man + find $DEVTOOLSET_ROOTPATH/usr/share/locale -mindepth 1 -maxdepth 1 -not \( -name 'en*' -or -name 'locale.alias' \) | xargs rm -rf +fi + +rm -rf /usr/share/backgrounds + +# if we updated glibc, we need to strip locales again... +if localedef --list-archive | grep -sq -v -i ^en_US.utf8; then + localedef --list-archive | grep -v -i ^en_US.utf8 | xargs localedef --delete-from-archive +fi +if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then + mv -f /usr/lib/locale/locale-archive /usr/lib/locale/locale-archive.tmpl + build-locale-archive +fi + +find /usr/share/locale -mindepth 1 -maxdepth 1 -not \( -name 'en*' -or -name 'locale.alias' \) | xargs rm -rf +if [ -d /usr/local/share/locale ]; then + find /usr/local/share/locale -mindepth 1 -maxdepth 1 -not \( -name 'en*' -or -name 'locale.alias' \) | xargs rm -rf +fi +if [ -d /usr/local/share/man ]; then + rm -rf /usr/local/share/man +fi + + +# Install newest automake +build_automake $AUTOMAKE_ROOT $AUTOMAKE_HASH +automake --version + +# Install newest libtool +build_libtool $LIBTOOL_ROOT $LIBTOOL_HASH +libtool --version + +# Install patchelf (latest with unreleased bug fixes) and apply our patches +build_patchelf $PATCHELF_VERSION $PATCHELF_HASH + +# Install libcrypt.so.1 and libcrypt.so.2 +build_libxcrypt "$LIBXCRYPT_DOWNLOAD_URL" "$LIBXCRYPT_VERSION" "$LIBXCRYPT_HASH" + +# Strip what we can -- and ignore errors, because this just attempts to strip +# *everything*, including non-ELF files: +find /usr/local -type f -print0 | xargs -0 -n1 strip --strip-unneeded 2>/dev/null || true