chore: rework PyPy installation (#1518)

PyPy installation is just a tar extract.
As such, it might be interesting to install PyPy at runtime rather than at build time.
This is especially true for EOL versions of PyPy which would allow to reduce the image size,
and thus, overall build time for users not using EOL PyPy versions.

This commit does not modify default installed PyPy versions yet.

This commit will also be useful for GraalPy installation.

README.rst

@@ -173,7 +173,7 @@ Image content
 
 All images currently contain:
 
-- CPython 3.6, 3.7, 3.8, 3.9, 3.10, 3.11, 3.12 and PyPy 3.7, 3.8, 3.9 installed in
+- CPython 3.6, 3.7, 3.8, 3.9, 3.10, 3.11, 3.12 and PyPy 3.7, 3.8, 3.9, 3.10 installed in
   ``/opt/python/<python tag>-<abi tag>``. The directories are named
   after the PEP 425 tags for each environment --
   e.g. ``/opt/python/cp37-cp37m`` contains a CPython 3.7 build, and
@@ -184,6 +184,49 @@ All images currently contain:
 
 - The `auditwheel <https://pypi.python.org/pypi/auditwheel>`_ tool
 
+- The manylinux-interpreters tool which allows to list all available interpreters & install ones missing from the image
+
+  3 commands are available:
+
+  - ``manylinux-interpreters list``
+
+    .. code-block:: bash
+
+      usage: manylinux-interpreters list [-h] [-v] [-i] [--format {text,json}]
+
+      list available or installed interpreters
+
+      options:
+        -h, --help            show this help message and exit
+        -v, --verbose         display additional information (--format=text only, ignored for --format=json)
+        -i, --installed       only list installed interpreters
+        --format {text,json}  text is not meant to be machine readable (i.e. the format is not stable)
+
+  - ``manylinux-interpreters ensure-all``
+
+    .. code-block:: bash
+
+      usage: manylinux-interpreters ensure-all [-h]
+
+      make sure all interpreters are installed
+
+      options:
+        -h, --help  show this help message and exit
+
+  - ``manylinux-interpreters ensure``
+
+    .. code-block:: bash
+
+      usage: manylinux-interpreters ensure [-h] TAG [TAG ...]
+
+      make sure a list of interpreters are installed
+
+      positional arguments:
+        TAG         tag with format '<python tag>-<abi tag>' e.g. 'pp310-pypy310_pp73'
+
+      options:
+        -h, --help  show this help message and exit
+
 Note that less common or virtually unheard of flag combinations
 (such as ``--with-pydebug`` (``d``) and ``--without-pymalloc`` (absence of ``m``)) are not provided.
 

docker/Dockerfile

@@ -138,43 +138,31 @@ FROM build_cpython AS build_cpython312
 COPY build_scripts/cpython-pubkey-312-313.txt /build_scripts/cpython-pubkeys.txt
 RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.12.0rc2
 
-FROM build_cpython AS all_python
-COPY build_scripts/install-pypy.sh \
-     build_scripts/pypy.sha256 \
-     build_scripts/finalize-python.sh \
-     /build_scripts/
-RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.7 7.3.9
-RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.8 7.3.11
-RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.9 7.3.12
-RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.10 7.3.12
-COPY --from=build_cpython36 /opt/_internal /opt/_internal/
-COPY --from=build_cpython37 /opt/_internal /opt/_internal/
-COPY --from=build_cpython38 /opt/_internal /opt/_internal/
-COPY --from=build_cpython39 /opt/_internal /opt/_internal/
-COPY --from=build_cpython310 /opt/_internal /opt/_internal/
-COPY --from=build_cpython311 /opt/_internal /opt/_internal/
-COPY --from=build_cpython312 /opt/_internal /opt/_internal/
-RUN manylinux-entrypoint /build_scripts/finalize-python.sh
+FROM build_cpython AS all_cpython
+COPY build_scripts/finalize-python.sh /build_scripts/
+RUN --mount=type=bind,target=/build_cpython36,from=build_cpython36 \
+    --mount=type=bind,target=/build_cpython37,from=build_cpython37 \
+    --mount=type=bind,target=/build_cpython38,from=build_cpython38 \
+    --mount=type=bind,target=/build_cpython39,from=build_cpython39 \
+    --mount=type=bind,target=/build_cpython310,from=build_cpython310 \
+    --mount=type=bind,target=/build_cpython311,from=build_cpython311 \
+    --mount=type=bind,target=/build_cpython312,from=build_cpython312 \
+    mkdir -p /opt/_internal && \
+    cp -rf /build_cpython*/opt/_internal/* /opt/_internal/ && \
+    manylinux-entrypoint /build_scripts/finalize-python.sh
 
 
 FROM runtime_base
 COPY --from=build_git /manylinux-rootfs /
 COPY --from=build_cpython /manylinux-rootfs /
-COPY --from=all_python /opt/_internal /opt/_internal/
-COPY build_scripts/finalize.sh \
-     build_scripts/update-system-packages.sh \
-     build_scripts/python-tag-abi-tag.py \
-     build_scripts/requirements3.6.txt \
-     build_scripts/requirements3.7.txt \
-     build_scripts/requirements3.8.txt \
-     build_scripts/requirements3.9.txt \
-     build_scripts/requirements3.10.txt \
-     build_scripts/requirements3.11.txt \
-     build_scripts/requirements3.12.txt \
-     build_scripts/requirements-base-tools.txt \
-     /build_scripts/
-COPY build_scripts/requirements-tools/* /build_scripts/requirements-tools/
-RUN manylinux-entrypoint /build_scripts/finalize.sh && rm -rf /build_scripts
+COPY build_scripts /opt/_internal/build_scripts/
+RUN --mount=type=bind,target=/all_cpython,from=all_cpython \
+    cp -rf /all_cpython/opt/_internal/* /opt/_internal/ && \
+    manylinux-entrypoint /opt/_internal/build_scripts/finalize.sh \
+    pp37-pypy37_pp73 \
+    pp38-pypy38_pp73 \
+    pp39-pypy39_pp73 \
+    pp310-pypy310_pp73
 
 ENV SSL_CERT_FILE=/opt/_internal/certs.pem
 

docker/build_scripts/download-and-install-interpreter.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Stop at any error, show all commands
+set -exuo pipefail
+
+# Get script directory
+MY_DIR=$(dirname "${BASH_SOURCE[0]}")
+
+ABI_TAG=$1
+DOWNLOAD_URL=$2
+SHA256=$3
+
+PREFIX="/opt/_internal/${ABI_TAG}"
+
+case ${DOWNLOAD_URL} in
+  *.tar) COMP=;;
+  *.tar.gz) COMP=z;;
+  *.tar.bz2) COMP=j;;
+  *.tar.xz) COMP=J;;
+  *) echo "unsupported archive"; exit 1;;
+esac
+
+mkdir ${PREFIX}
+
+curl -fsSL ${DOWNLOAD_URL} | tee >(tar -C ${PREFIX} --strip-components 1 -x${COMP}f -) | sha256sum -c <(echo "${SHA256} -")
+
+# remove debug symbols if any
+find ${PREFIX}/bin -name '*.debug' -delete
+
+${MY_DIR}/finalize-one.sh ${PREFIX}

docker/build_scripts/finalize-one.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# Stop at any error, show all commands
+set -exuo pipefail
+
+PREFIX=$1
+
+# Get script directory
+MY_DIR=$(dirname "${BASH_SOURCE[0]}")
+
+# Some python's install as bin/python3. Make them available as
+# bin/python.
+if [ -e ${PREFIX}/bin/python3 ] && [ ! -e ${PREFIX}/bin/python ]; then
+	ln -s python3 ${PREFIX}/bin/python
+fi
+PY_VER=$(${PREFIX}/bin/python -c "import sys; print('.'.join(str(v) for v in sys.version_info[:2]))")
+PY_IMPL=$(${PREFIX}/bin/python -c "import sys; print(sys.implementation.name)")
+
+# Install pinned packages for this python version.
+# Use the already intsalled cpython pip to bootstrap pip if available
+if [ -f /usr/local/bin/python${PY_VER} ]; then
+	/usr/local/bin/python${PY_VER} -m pip --python ${PREFIX}/bin/python install -U --require-hashes -r ${MY_DIR}/requirements${PY_VER}.txt
+else
+	${PREFIX}/bin/python -m ensurepip
+	${PREFIX}/bin/python -m pip install -U --require-hashes -r ${MY_DIR}/requirements${PY_VER}.txt
+fi
+if [ -e ${PREFIX}/bin/pip3 ] && [ ! -e ${PREFIX}/bin/pip ]; then
+	ln -s pip3 ${PREFIX}/bin/pip
+fi
+# Create a symlink to PREFIX using the ABI_TAG in /opt/python/
+ABI_TAG=$(${PREFIX}/bin/python ${MY_DIR}/python-tag-abi-tag.py)
+ln -s ${PREFIX} /opt/python/${ABI_TAG}
+# Make versioned python commands available directly in environment.
+if [[ "${PY_IMPL}" == "cpython" ]]; then
+	ln -s ${PREFIX}/bin/python /usr/local/bin/python${PY_VER}
+fi
+ln -s ${PREFIX}/bin/python /usr/local/bin/${PY_IMPL}${PY_VER}

docker/build_scripts/finalize.sh

@@ -11,30 +11,21 @@ source $MY_DIR/build_utils.sh
 
 mkdir /opt/python
 for PREFIX in $(find /opt/_internal/ -mindepth 1 -maxdepth 1 \( -name 'cpython*' -o -name 'pypy*' \)); do
-	# Some python's install as bin/python3. Make them available as
-	# bin/python.
-	if [ -e ${PREFIX}/bin/python3 ] && [ ! -e ${PREFIX}/bin/python ]; then
-		ln -s python3 ${PREFIX}/bin/python
-	fi
-	${PREFIX}/bin/python -m ensurepip
-	if [ -e ${PREFIX}/bin/pip3 ] && [ ! -e ${PREFIX}/bin/pip ]; then
-		ln -s pip3 ${PREFIX}/bin/pip
-	fi
-	PY_VER=$(${PREFIX}/bin/python -c "import sys; print('.'.join(str(v) for v in sys.version_info[:2]))")
-	# Since we fall back on a canned copy of pip, we might not have
-	# the latest pip and friends. Upgrade them to make sure.
-	${PREFIX}/bin/pip install -U --require-hashes -r ${MY_DIR}/requirements${PY_VER}.txt
-	# Create a symlink to PREFIX using the ABI_TAG in /opt/python/
-	ABI_TAG=$(${PREFIX}/bin/python ${MY_DIR}/python-tag-abi-tag.py)
-	ln -s ${PREFIX} /opt/python/${ABI_TAG}
-	# Make versioned python commands available directly in environment.
-	if [[ "${PREFIX}" == *"/pypy"* ]]; then
-		ln -s ${PREFIX}/bin/python /usr/local/bin/pypy${PY_VER}
-	else
-		ln -s ${PREFIX}/bin/python /usr/local/bin/python${PY_VER}
-	fi
+	${MY_DIR}/finalize-one.sh ${PREFIX}
 done
 
+# create manylinux-interpreters script
+cat <<EOF > /usr/local/bin/manylinux-interpreters
+#!/bin/bash
+
+set -euo pipefail
+
+/opt/python/cp310-cp310/bin/python $MY_DIR/manylinux-interpreters.py "\$@"
+EOF
+chmod 755 /usr/local/bin/manylinux-interpreters
+
+MANYLINUX_INTERPRETERS_NO_CHECK=1 /usr/local/bin/manylinux-interpreters ensure "$@"
+
 # Create venv for auditwheel & certifi
 TOOLS_PATH=/opt/_internal/tools
 /opt/python/cp310-cp310/bin/python -m venv $TOOLS_PATH
@@ -84,6 +75,7 @@ clean_pyc /opt/_internal
 
 # remove cache
 rm -rf /root/.cache
+rm -rf /tmp/* || true
 
 hardlink -cv /opt/_internal