sagemathgh-36141: Use system python packages with --enable-system-sit…

…e-packages

    
Rebased branch of sagemath#29665

Platform-specific testing:
- for example TARGETS_PRE="build-local" tox -e docker-ubuntu-focal-
standard-sitepackages -- build ptest

Fixes sagemath#29665
    
URL: sagemath#36141
Reported by: Michael Orlitzky
Reviewer(s): Dima Pasechnik, Matthias Köppe, Michael Orlitzky

.github/workflows/ci-linux.yml

@@ -63,6 +63,56 @@ jobs:
           ["standard"]
       docker_push_repository: ghcr.io/${{ github.repository }}/
 
+  standard-sitepackages:
+    if: ${{ success() || failure() }}
+    needs: [standard-pre]
+    uses: ./.github/workflows/docker.yml
+    with:
+      # Build incrementally from previous stage (pre)
+      incremental: true
+      free_disk_space: true
+      from_docker_repository: ghcr.io/${{ github.repository }}/
+      from_docker_target: "with-targets-pre"
+      docker_targets: "with-targets with-targets-optional"
+      # FIXME: duplicated from env.TARGETS
+      targets: build doc-html
+      targets_optional: ptest
+      tox_packages_factors: >-
+        ["standard-sitepackages"]
+      # Only test systems with a usable system python (>= 3.9)
+      tox_system_factors: >-
+        ["ubuntu-jammy",
+         "ubuntu-kinetic",
+         "ubuntu-lunar",
+         "ubuntu-mantic",
+         "debian-bullseye",
+         "debian-bookworm",
+         "debian-trixie",
+         "debian-sid",
+         "linuxmint-21",
+         "linuxmint-21.1",
+         "linuxmint-21.2",
+         "fedora-33",
+         "fedora-34",
+         "fedora-35",
+         "fedora-36",
+         "fedora-37",
+         "fedora-38",
+         "fedora-39",
+         "centos-stream-8-python3.9",
+         "centos-stream-9-python3.9",
+         "almalinux-8-python3.9",
+         "gentoo-python3.10",
+         "gentoo-python3.11",
+         "archlinux-latest",
+         "opensuse-15.4-gcc_11-python3.10",
+         "opensuse-15.5-gcc_11-python3.11",
+         "opensuse-tumbleweed-python3.10",
+         "opensuse-tumbleweed",
+         "debian-bullseye-i386"]
+      docker_push_repository: ghcr.io/${{ github.repository }}/
+      max_parallel: 10
+
   minimal-pre:
     if: ${{ success() || failure() }}
     # It does not really "need" it.

.github/workflows/docker.yml

@@ -57,6 +57,7 @@ on:
            "opensuse-15.4-gcc_11-python3.10",
            "opensuse-15.5-gcc_11-python3.11",
            "opensuse-tumbleweed-python3.10",
+           "opensuse-tumbleweed",
            "conda-forge",
            "ubuntu-bionic-gcc_8-i386",
            "debian-bullseye-i386",

build/bin/sage-build-env-config.in

@@ -53,9 +53,13 @@ export SAGE_PARI_CFG="@SAGE_PARI_CFG@"
 
 export SAGE_GLPK_PREFIX="@SAGE_GLPK_PREFIX@"
 export SAGE_FREETYPE_PREFIX="@SAGE_FREETYPE_PREFIX@"
+export SAGE_PIP_INSTALL_FLAGS="@SAGE_PIP_INSTALL_FLAGS@"
 export SAGE_SUITESPARSE_PREFIX="@SAGE_SUITESPARSE_PREFIX@"
 
 export SAGE_CONFIGURE_FFLAS_FFPACK="@SAGE_CONFIGURE_FFLAS_FFPACK@"
 
 export CONFIGURED_SAGE_EDITABLE="@SAGE_EDITABLE@"
 export CONFIGURED_SAGE_WHEELS="@SAGE_WHEELS@"
+
+export ENABLE_SYSTEM_SITE_PACKAGES="@ENABLE_SYSTEM_SITE_PACKAGES@"
+export PYTHON_MINOR="@PYTHON_MINOR@"

build/bin/sage-dist-helpers

@@ -325,7 +325,11 @@ sdh_store_wheel() {
 }
 
 sdh_store_and_pip_install_wheel() {
-    local pip_options=""
+    # The $SAGE_PIP_INSTALL_FLAGS variable is set by sage-build-env-config.
+    # We skip sanity checking its contents since you should either let sage
+    # decide what it contains, or really know what you are doing.
+    local pip_options="${SAGE_PIP_INSTALL_FLAGS}"
+
     while [ $# -gt 0 ]; do
         case $1 in
             -*) pip_options="$pip_options $1"

build/bin/sage-get-system-packages

@@ -1,4 +1,5 @@
 #!/bin/sh
+
 SYSTEM=$1
 if [ -z "$SYSTEM" ]; then
    echo >&2 "usage: $0 {auto|debian|arch|conda|pip|...} SPKGS..."
@@ -10,6 +11,7 @@ SPKGS="$*"
 if [ -z "$SAGE_ROOT" ]; then
     SAGE_ROOT=`pwd`
 fi
+
 case "$SYSTEM" in
     install-requires)
         # Collect install-requires.txt (falling back to requirements.txt) and output it in the format
@@ -44,6 +46,18 @@ case "$SYSTEM" in
         ;;
 esac
 for PKG_BASE in $SPKGS; do
+
+    # Skip this package if it uses the SAGE_PYTHON_PACKAGE_CHECK
+    # macro and if --enable-system-site-packages was NOT passed
+    # to ./configure (or if ./configure has not yet been run).
+    SPKG_CONFIGURE="${SAGE_ROOT}/build/pkgs/${PKG_BASE}/spkg-configure.m4"
+    if [ -z "${ENABLE_SYSTEM_SITE_PACKAGES}" ]; then
+        if grep -q SAGE_PYTHON_PACKAGE_CHECK "${SPKG_CONFIGURE}" 2>/dev/null;
+        then
+            continue;
+        fi
+    fi
+
     for NAME in $SYSTEM_PACKAGES_FILE_NAMES; do
         SYSTEM_PACKAGES_FILE="$SAGE_ROOT"/build/pkgs/$PKG_BASE/$NAME
         if [ -f $SYSTEM_PACKAGES_FILE ]; then

build/bin/sage-spkg-info

@@ -110,17 +110,19 @@ if [ -z "$system" ]; then
     echo "(none known)"
 else
     echo
-    if [ -f "$PKG_SCRIPTS"/spkg-configure.m4 ]; then
-        echo "If the system package is installed, ./configure will check whether it can be used."
-    else
-        echo "However, these system packages will not be used for building Sage"
-        if [ -f "$PKG_SCRIPTS"/install-requires.txt ]; then
-            echo "because using Python site-packages is not supported by the Sage distribution;"
-            echo "see https://github.com/sagemath/sage/issues/29023"
+    SPKG_CONFIGURE="${PKG_SCRIPTS}/spkg-configure.m4"
+    if [ -f "${SPKG_CONFIGURE}" ]; then
+        if grep -q SAGE_PYTHON_PACKAGE_CHECK "${SPKG_CONFIGURE}"; then
+            echo "If the system package is installed and if the (experimental) option"
+            echo "--enable-system-site-packages is passed to ./configure, then ./configure"
+            echo "will check if the system package can be used."
         else
-            echo "because spkg-configure.m4 has not been written for this package;"
-            echo "see https://github.com/sagemath/sage/issues/27330"
+            echo "If the system package is installed, ./configure will check if it can be used."
         fi
+    else
+        echo "However, these system packages will not be used for building Sage"
+        echo "because spkg-configure.m4 has not been written for this package;"
+        echo "see https://github.com/sagemath/sage/issues/27330"
     fi
 fi
 echo

build/bin/write-dockerfile.sh

@@ -5,7 +5,7 @@
 set -e
 SYSTEM="${1:-debian}"
 shopt -s extglob
-SAGE_PACKAGE_LIST_ARGS="${2:- --has-file=spkg-configure.m4 :standard:}"
+SAGE_PACKAGE_LIST_ARGS="${2:-:standard:}"
 WITH_SYSTEM_SPKG="${3:-yes}"
 IGNORE_MISSING_SYSTEM_PACKAGES="${4:-no}"
 EXTRA_SAGE_PACKAGES="${5:-_bootstrap}"
@@ -15,17 +15,13 @@ SAGE_ROOT=.
 export PATH="$SAGE_ROOT"/build/bin:$PATH
 SYSTEM_PACKAGES=$EXTRA_SYSTEM_PACKAGES
 CONFIGURE_ARGS="--enable-option-checking "
-for PKG_BASE in $(sage-package list --has-file=distros/$SYSTEM.txt $SAGE_PACKAGE_LIST_ARGS) $EXTRA_SAGE_PACKAGES; do
-    PKG_SCRIPTS="$SAGE_ROOT"/build/pkgs/$PKG_BASE
-    if [ -d $PKG_SCRIPTS ]; then
-       SYSTEM_PACKAGES_FILE=$PKG_SCRIPTS/distros/$SYSTEM.txt
-       PKG_SYSTEM_PACKAGES=$(echo $(${STRIP_COMMENTS} $SYSTEM_PACKAGES_FILE))
-       if [ -n "PKG_SYSTEM_PACKAGES" ]; then
-           SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES"
-           if [ -f $PKG_SCRIPTS/spkg-configure.m4 ]; then
-               CONFIGURE_ARGS+="--with-system-$PKG_BASE=${WITH_SYSTEM_SPKG} "
-           fi
-       fi
+for SPKG in $(sage-package list --has-file=spkg-configure.m4 $SAGE_PACKAGE_LIST_ARGS) $EXTRA_SAGE_PACKAGES; do
+    SYSTEM_PACKAGE=$(sage-get-system-packages $SYSTEM $SPKG | sed 's/${PYTHON_MINOR}/'${PYTHON_MINOR}'/g')
+    if [ -n "${SYSTEM_PACKAGE}" ]; then
+	# SYSTEM_PACKAGE can be empty if, for example, the environment
+	# variable ENABLE_SYSTEM_SITE_PACKAGES is empty.
+	SYSTEM_PACKAGES+=" ${SYSTEM_PACKAGE}"
+	CONFIGURE_ARGS+="--with-system-${SPKG}=${WITH_SYSTEM_SPKG} "
     fi
 done
 echo "# Automatically generated by SAGE_ROOT/build/bin/write-dockerfile.sh"

build/make/Makefile.in

@@ -227,22 +227,29 @@ SAGE_I_TARGETS = sagelib doc
 	python3_venv _clean-broken-gcc
 
 PYTHON_FOR_VENV = @PYTHON_FOR_VENV@
+PYTHON_MINOR = @PYTHON_MINOR@
+SAGE_VENV_FLAGS = @SAGE_VENV_FLAGS@
 
 ifneq ($(PYTHON_FOR_VENV),)
 # Special rule for making the Python virtualenv from the system Python (Python
 # 3 only).  $(PYTHON) is set in Makefile to python3_venv.
-# Thus $(inst_python3_venv) will be the dependency of every Python package.
+# Thus $(inst_python3_venv) will an (order-only) dependency of every Python package.
 #
 # TODO: If we reconfigure to build our own Python after having used the system
 # Python, files installed to create the virtualenv should be *removed*.  That
 # could either be done here by the makefile, or in an spkg-preinst for python3
 ifeq ($(PYTHON),python3)
 PYTHON = python3_venv
 endif
-inst_python3_venv = $(SAGE_VENV)/pyvenv.cfg
+inst_python3_venv = $(SAGE_VENV)/$(SPKG_INST_RELDIR)/python3_venv-3.$(PYTHON_MINOR)-$(subst /,-,$(PYTHON_FOR_VENV))$(findstring --system-site-packages,$(SAGE_VENV_FLAGS))
 
-$(inst_python3_venv):
-	$(PYTHON_FOR_VENV) $(SAGE_ROOT)/build/bin/sage-venv "$(SAGE_VENV)"
+$(SAGE_VENV)/$(SPKG_INST_RELDIR):
+	mkdir -p "$@"
+
+$(inst_python3_venv): | $(SAGE_VENV)/$(SPKG_INST_RELDIR)
+	$(PYTHON_FOR_VENV) $(SAGE_ROOT)/build/bin/sage-venv $(SAGE_VENV_FLAGS) "$(SAGE_VENV)"
+	rm -f "$(SAGE_VENV)/$(SPKG_INST_RELDIR)"/python3_venv-*
+	touch "$@"
 endif
 
 # Build everything and start Sage.

build/pkgs/admcycles/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | $(PYTHON_TOOLCHAIN)
+ | $(PYTHON_TOOLCHAIN) $(PYTHON)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/alabaster/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | $(PYTHON_TOOLCHAIN)
+ | $(PYTHON_TOOLCHAIN) $(PYTHON)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/alabaster/distros/gentoo.txt

@@ -0,0 +1 @@
+dev-python/alabaster

build/pkgs/alabaster/distros/opensuse.txt

@@ -1 +1 @@
-python3-alabaster
+python3${PYTHON_MINOR}-alabaster

build/pkgs/alabaster/spkg-configure.m4

@@ -0,0 +1 @@
+SAGE_SPKG_CONFIGURE([alabaster], [SAGE_PYTHON_PACKAGE_CHECK([alabaster])])

build/pkgs/appdirs/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | $(PYTHON_TOOLCHAIN)
+ | $(PYTHON_TOOLCHAIN) $(PYTHON)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/appnope/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | $(PYTHON_TOOLCHAIN)
+ | $(PYTHON_TOOLCHAIN) $(PYTHON)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/argon2_cffi/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) argon2_cffi_bindings | $(PYTHON_TOOLCHAIN) flit_core
+ argon2_cffi_bindings | $(PYTHON_TOOLCHAIN) flit_core $(PYTHON)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/argon2_cffi/distros/gentoo.txt

@@ -0,0 +1 @@
+dev-python/argon2-cffi

build/pkgs/argon2_cffi/spkg-configure.m4

@@ -0,0 +1 @@
+SAGE_SPKG_CONFIGURE([argon2_cffi], [SAGE_PYTHON_PACKAGE_CHECK([argon2_cffi])])

build/pkgs/argon2_cffi_bindings/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | $(PYTHON_TOOLCHAIN) cffi setuptools_scm
+ | $(PYTHON_TOOLCHAIN) cffi setuptools_scm $(PYTHON)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/asttokens/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | $(PYTHON_TOOLCHAIN)
+ | $(PYTHON_TOOLCHAIN) $(PYTHON)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/attrs/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) vcversioner | $(PYTHON_TOOLCHAIN)
+ vcversioner | $(PYTHON_TOOLCHAIN) $(PYTHON)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/attrs/distros/gentoo.txt

@@ -0,0 +1 @@
+dev-python/attrs

build/pkgs/attrs/spkg-configure.m4

@@ -0,0 +1 @@
+SAGE_SPKG_CONFIGURE([attrs], [SAGE_PYTHON_PACKAGE_CHECK([attrs])])

build/pkgs/auditwheel_or_delocate/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | $(PYTHON_TOOLCHAIN)
+ | $(PYTHON_TOOLCHAIN) $(PYTHON)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/awali/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) cmake cython nbconvert ncurses
+ cmake cython nbconvert ncurses | $(PYTHON)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/babel/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | $(PYTHON_TOOLCHAIN) pytz
+ | $(PYTHON_TOOLCHAIN) pytz $(PYTHON)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/babel/distros/arch.txt

@@ -0,0 +1 @@
+python-babel

build/pkgs/babel/distros/debian.txt

@@ -0,0 +1 @@
+python3-babel

build/pkgs/babel/distros/fedora.txt

@@ -0,0 +1 @@
+babel

build/pkgs/babel/distros/gentoo.txt

@@ -0,0 +1 @@
+dev-python/Babel

build/pkgs/babel/distros/opensuse.txt

@@ -1 +1 @@
-python3-Babel
+python3${PYTHON_MINOR}-Babel

build/pkgs/babel/install-requires.txt

@@ -1 +1 @@
-babel >=2.6.0
+babel >=2.11.0

build/pkgs/babel/spkg-configure.m4

@@ -0,0 +1 @@
+SAGE_SPKG_CONFIGURE([babel], [SAGE_PYTHON_PACKAGE_CHECK([babel])])

build/pkgs/backcall/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | $(PYTHON_TOOLCHAIN) flit_core tomli
+ | $(PYTHON_TOOLCHAIN) flit_core tomli $(PYTHON)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/backcall/distros/gentoo.txt

@@ -0,0 +1 @@
+dev-python/backcall

build/pkgs/backcall/spkg-configure.m4

@@ -0,0 +1 @@
+SAGE_SPKG_CONFIGURE([backcall], [SAGE_PYTHON_PACKAGE_CHECK([backcall])])

build/pkgs/beautifulsoup4/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) soupsieve | $(PYTHON_TOOLCHAIN)
+ soupsieve | $(PYTHON_TOOLCHAIN) $(PYTHON)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/beniget/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) gast | $(PYTHON_TOOLCHAIN)
+ gast | $(PYTHON_TOOLCHAIN) $(PYTHON)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/beniget/distros/gentoo.txt

@@ -0,0 +1 @@
+dev-python/beniget

build/pkgs/beniget/spkg-configure.m4

@@ -0,0 +1 @@
+SAGE_SPKG_CONFIGURE([beniget], [SAGE_PYTHON_PACKAGE_CHECK([beniget])])

build/pkgs/bleach/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) packaging six webencodings | $(PYTHON_TOOLCHAIN)
+ packaging six webencodings | $(PYTHON_TOOLCHAIN) $(PYTHON)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/bleach/distros/arch.txt

@@ -0,0 +1 @@
+python-bleach

build/pkgs/bleach/distros/debian.txt

@@ -0,0 +1 @@
+python3-bleach

build/pkgs/bleach/distros/fedora.txt

@@ -0,0 +1 @@
+python-bleach

build/pkgs/bleach/distros/gentoo.txt

@@ -0,0 +1 @@
+dev-python/bleach

build/pkgs/bleach/distros/opensuse.txt

@@ -0,0 +1 @@
+python3${PYTHON_MINOR}-bleach

build/pkgs/bleach/spkg-configure.m4

@@ -0,0 +1 @@
+SAGE_SPKG_CONFIGURE([bleach], [SAGE_PYTHON_PACKAGE_CHECK([bleach])])

build/pkgs/certifi/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | $(PYTHON_TOOLCHAIN)
+ | $(PYTHON_TOOLCHAIN) $(PYTHON)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/certifi/distros/arch.txt

@@ -0,0 +1 @@
+python-certifi

build/pkgs/certifi/distros/debian.txt

@@ -0,0 +1 @@
+python3-certifi

build/pkgs/certifi/distros/fedora.txt

@@ -0,0 +1 @@
+python-certifi

build/pkgs/certifi/distros/gentoo.txt

@@ -0,0 +1 @@
+dev-python/certifi

build/pkgs/certifi/distros/opensuse.txt

@@ -1 +1 @@
-python3-certifi
+python3${PYTHON_MINOR}-certifi

build/pkgs/certifi/spkg-configure.m4

@@ -0,0 +1 @@
+SAGE_SPKG_CONFIGURE([certifi], [SAGE_PYTHON_PACKAGE_CHECK([certifi])])

build/pkgs/cffi/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | $(PYTHON_TOOLCHAIN) pycparser
+ | $(PYTHON_TOOLCHAIN) pycparser $(PYTHON)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/cffi/distros/gentoo.txt

@@ -0,0 +1 @@
+virtual/python-cffi

build/pkgs/cffi/distros/opensuse.txt

@@ -1 +1 @@
-python3-cffi
+python3${PYTHON_MINOR}-cffi

build/pkgs/cffi/spkg-configure.m4

@@ -0,0 +1 @@
+SAGE_SPKG_CONFIGURE([cffi], [SAGE_PYTHON_PACKAGE_CHECK([cffi])])