gh-36405: bootstrap-conda: Refactor, generate versioned environment…

… files

    
<!-- ^^^^^
Please provide a concise, informative and self-explanatory title.
Don't put issue numbers in there, do this in the PR body below.
For example, instead of "Fixes #1234" use "Introduce new method to
calculate 1+1"
-->
<!-- Describe your changes here in detail -->
- Environment files are now versioned with the Python minor version, as
proposed in (and cherry-picked from) #35986 by @tobiasdiez
- CI for conda now uses `--enable-system-site-packages` from #36141 by
@orlitzky

Split out from

<!-- Why is this change required? What problem does it solve? -->
<!-- If this PR resolves an open issue, please link to it here. For
example "Fixes #12345". -->
<!-- If your change requires a documentation PR, please link it
appropriately. -->

### 📝 Checklist

<!-- Put an `x` in all the boxes that apply. -->
<!-- If your change requires a documentation PR, please link it
appropriately -->
<!-- If you're unsure about any of these, don't hesitate to ask. We're
here to help! -->
<!-- Feel free to remove irrelevant items. -->

- [x] The title is concise, informative, and self-explanatory.
- [x] The description explains in detail what this PR is about.
- [ ] I have linked a relevant issue or discussion.
- [ ] I have created tests covering the changes.
- [ ] I have updated the documentation accordingly.

### ⌛ Dependencies

<!-- List all open PRs that this PR logically depends on
- #12345: short description why this is a dependency
- #34567: ...
-->

<!-- If you're unsure about any of these, don't hesitate to ask. We're
here to help! -->
    
URL: #36405
Reported by: Matthias Köppe
Reviewer(s): Tobias Diez

.devcontainer/onCreate-conda.sh

@@ -4,7 +4,7 @@ set -e
 # Create conda environment
 ./bootstrap-conda
 conda install mamba -n base -c conda-forge -y
-mamba env create --file src/environment-dev.yml || mamba env update --file src/environment-dev.yml
+mamba env create --file src/environment-dev-3.11.yml || mamba env update --file src/environment-dev-3.11.yml
 conda init bash
 
 # Build sage

.github/workflows/ci-conda.yml

@@ -59,7 +59,7 @@ jobs:
         with:
           path: ~/conda_pkgs_dir
           key:
-            ${{ runner.os }}-conda-${{ hashFiles('src/environment.yml') }}
+            ${{ runner.os }}-conda-${{ hashFiles('src/environment-3.11.yml') }}
 
       - name: Setup Conda
         uses: conda-incubator/setup-miniconda@v2
@@ -68,8 +68,8 @@ jobs:
           mamba-version: "*"
           channels: conda-forge,defaults
           channel-priority: true
-          activate-environment: sage-build
-          environment-file: src/${{ matrix.conda-env }}.yml
+          activate-environment: sage
+          environment-file: src/${{ matrix.conda-env }}-${{ matrix.python }}.yml
 
       - name: Print Conda environment
         shell: bash -l {0}
@@ -83,7 +83,7 @@ jobs:
         run: |
           ./bootstrap
           echo "::add-matcher::.github/workflows/configure-systempackage-problem-matcher.json"
-          ./configure --enable-build-as-root --with-python=$CONDA_PREFIX/bin/python --prefix=$CONDA_PREFIX $(for pkg in $(./sage -package list :standard: --has-file spkg-configure.m4 --has-file distros/conda.txt --exclude rpy2); do echo --with-system-$pkg=force; done)
+          ./configure --enable-build-as-root --with-python=$CONDA_PREFIX/bin/python --prefix=$CONDA_PREFIX --enable-system-site-packages $(for pkg in $(./sage -package list :standard: --has-file spkg-configure.m4 --has-file distros/conda.txt --exclude rpy2); do echo --with-system-$pkg=force; done)
           echo "::remove-matcher owner=configure-system-package-warning::"
           echo "::remove-matcher owner=configure-system-package-error::"
 

.gitignore

@@ -27,11 +27,29 @@
 # no longer generated, but may still be in user worktrees
 /src/lib/pkgconfig
 
+# Environment files generated by bootstrap-conda.
+# The files without Python version are no longer generated
+# but may still be in users' directories.
 /environment.yml
+/environment-3.9.yml
+/environment-3.10.yml
+/environment-3.11.yml
 /environment-optional.yml
+/environment-optional-3.9.yml
+/environment-optional-3.10.yml
+/environment-optional-3.11.yml
 /src/environment.yml
+/src/environment-3.9.yml
+/src/environment-3.10.yml
+/src/environment-3.11.yml
 /src/environment-dev.yml
+/src/environment-dev-3.9.yml
+/src/environment-dev-3.10.yml
+/src/environment-dev-3.11.yml
 /src/environment-optional.yml
+/src/environment-optional-3.9.yml
+/src/environment-optional-3.10.yml
+/src/environment-optional-3.11.yml
 
 /src/setup.cfg
 /src/requirements.txt

.gitpod.yml

@@ -8,7 +8,7 @@ tasks:
     # Create conda environment, then configure and build sage
     init: >-
       ./bootstrap-conda
-      && mamba env create --file src/environment-dev.yml --prefix venv
+      && mamba env create --file src/environment-dev-3.11.yml --prefix venv
       && conda config --append envs_dirs $(pwd)
       && conda activate $(pwd)/venv
       && ./bootstrap

Makefile

@@ -158,11 +158,7 @@ bootstrap-clean:
 	rm -rf config/install-sh config/compile config/config.guess config/config.sub config/missing configure build/make/Makefile-auto.in
 	rm -f src/doc/en/installation/*.txt
 	rm -rf src/doc/en/reference/spkg/*.rst
-	rm -f environment.yml
-	rm -f src/environment.yml
-	rm -f src/environment-dev.yml
-	rm -f environment-optional.yml
-	rm -f src/environment-optional.yml
+	for a in environment environment-optional src/environment src/environment-dev src/environment-optional; do rm -f $$a.yml $$a-3.[89].yml $$a-3.1[0-9].yml; done
 	rm -f src/Pipfile
 	rm -f src/pyproject.toml
 	rm -f src/requirements.txt

bootstrap

@@ -226,10 +226,10 @@ save () {
 	build/make/Makefile-auto.in \
 	src/doc/en/installation/*.txt \
 	src/doc/en/reference/spkg/*.rst \
-	environment.yml \
-	src/environment.yml \
-	environment-optional.yml \
-	src/environment-optional.yml \
+	environment-3.[89].yml environment-3.1[0-9].yml \
+	src/environment-3.[89].yml src/environment-3.1[0-9].yml \
+	environment-optional-3.[89].yml environment-optional-3.1[0-9].yml \
+	src/environment-optional-3.[89].yml src/environment-optional-3.1[0-9].yml \
 	src/Pipfile \
 	src/pyproject.toml \
 	src/requirements.txt \

bootstrap-conda

@@ -11,112 +11,151 @@ STRIP_COMMENTS="sed s/#.*//;"
 shopt -s extglob
 
 DEVELOP_SPKG_PATTERN="@(_develop$(for a in $(head -n 1 build/pkgs/_develop/dependencies); do echo -n "|"$a; done))"
-BOOTSTRAP_PACKAGES=$(echo $(${STRIP_COMMENTS} build/pkgs/_bootstrap/distros/conda.txt))
-SYSTEM_PACKAGES=
-OPTIONAL_SYSTEM_PACKAGES=
-SAGELIB_SYSTEM_PACKAGES=
-SAGELIB_OPTIONAL_SYSTEM_PACKAGES=
-DEVELOP_SYSTEM_PACKAGES=
+BOOTSTRAP_PACKAGES=_bootstrap
+PACKAGES=
+OPTIONAL_PACKAGES=
+SAGELIB_PACKAGES=
+SAGELIB_OPTIONAL_PACKAGES=
+DEVELOP_PACKAGES=
+
 for PKG_BASE in $(sage-package list --has-file distros/conda.txt --exclude _sagemath); do
     PKG_SCRIPTS=build/pkgs/$PKG_BASE
     SYSTEM_PACKAGES_FILE=$PKG_SCRIPTS/distros/conda.txt
     PKG_TYPE=$(cat $PKG_SCRIPTS/type)
     PKG_SYSTEM_PACKAGES=$(echo $(${STRIP_COMMENTS} $SYSTEM_PACKAGES_FILE))
-    if [ -n "PKG_SYSTEM_PACKAGES" ]; then
+    if [ -n "$PKG_SYSTEM_PACKAGES" ]; then
         if [ -f $PKG_SCRIPTS/spkg-configure.m4 ]; then
+            if grep -q SAGE_PYTHON_PACKAGE_CHECK $PKG_SCRIPTS/spkg-configure.m4; then
+                # Python package that would need --enable-system-site-packages to be used
+                # with the Sage distribution, but we do not recommend that for conda.
+                PKG_SAGELIB_ONLY=yes
+            else
+                PKG_SAGELIB_ONLY=no
+            fi
+        else
+            # No spkg-configure, so the Sage distribution is not able to make use of this package.
+            PKG_SAGELIB_ONLY=yes
+        fi
+        [ -n "$BOOTSTRAP_VERBOSE" ] && echo "$PKG_BASE:$PKG_TYPE:$PKG_SAGELIB_ONLY"
+        if [ $PKG_SAGELIB_ONLY = no ]; then
             case "$PKG_BASE:$PKG_TYPE" in
                 *:standard)
-                    SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES"
+                    PACKAGES+=" $PKG_BASE"
                     ;;
                 $DEVELOP_SPKG_PATTERN:*)
-                    DEVELOP_SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES"
+                    DEVELOP_PACKAGES+=" $PKG_BASE"
                     ;;
                 *)
-                    OPTIONAL_SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES"
+                    OPTIONAL_PACKAGES+=" $PKG_BASE"
                     ;;
             esac
         else
             case "$PKG_BASE:$PKG_TYPE" in
                 *:standard)
-                    SAGELIB_SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES"
+                    SAGELIB_PACKAGES+=" $PKG_BASE"
                     ;;
                 $DEVELOP_SPKG_PATTERN:*)
-                    DEVELOP_SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES"
+                    DEVELOP_PACKAGES+=" $PKG_BASE"
                     ;;
                 *)
-                    SAGELIB_OPTIONAL_SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES"
+                    SAGELIB_OPTIONAL_PACKAGES+=" $PKG_BASE"
                     ;;
             esac
         fi
     fi
 done
+unset PKG_SYSTEM_PACKAGES
+
+[ -n "$BOOTSTRAP_VERBOSE" ] && echo "## Collected:" && set | grep PACKAGES=
+
+# Translate to system packages
+export ENABLE_SYSTEM_SITE_PACKAGES=yes  # Disable filtering in sage-get-system-packages
+SYSTEM_PACKAGES=$(sage-get-system-packages conda $PACKAGES)
+BOOTSTRAP_SYSTEM_PACKAGES=$(sage-get-system-packages conda $BOOTSTRAP_PACKAGES)
+OPTIONAL_SYSTEM_PACKAGES=$(sage-get-system-packages conda $OPTIONAL_PACKAGES)
+SAGELIB_SYSTEM_PACKAGES=$(sage-get-system-packages conda $SAGELIB_PACKAGES)
+SAGELIB_OPTIONAL_SYSTEM_PACKAGES=$(sage-get-system-packages conda $SAGELIB_OPTIONAL_PACKAGES)
+DEVELOP_SYSTEM_PACKAGES=$(sage-get-system-packages conda $DEVELOP_PACKAGES)
+unset ENABLE_SYSTEM_SITE_PACKAGES
+
+[ -n "$BOOTSTRAP_VERBOSE" ] && echo "## Translated to system:" && set | grep SYSTEM_PACKAGES=
+
 echo >&2 $0:$LINENO: generate conda environment files
-(
-    echo "name: sage-build"
-    echo "channels:"
-    echo "  - conda-forge"
-    echo "  - nodefaults"
-    echo "dependencies:"
-    for pkg in $SYSTEM_PACKAGES; do
-        echo "  - $pkg"
-    done
-    echo "  # Packages needed for ./bootstrap"
-    for pkg in $BOOTSTRAP_PACKAGES; do
-        echo "  - $pkg"
-    done
-) > environment.yml
 
-(
-    sed 's/name: sage-build/name: sage/' environment.yml
-    echo "  # Additional packages providing all dependencies for the Sage library"
-    for pkg in $SAGELIB_SYSTEM_PACKAGES; do
-        echo "  - $pkg"
-    done
-) > src/environment.yml
+    (
+        echo "name: sage-build"
+        echo "channels:"
+        echo "  - conda-forge"
+        echo "  - nodefaults"
+        echo "dependencies:"
+        for pkg in $SYSTEM_PACKAGES; do
+            echo "  - $pkg"
+        done
+        echo "  # Packages needed for ./bootstrap"
+        for pkg in $BOOTSTRAP_SYSTEM_PACKAGES; do
+            echo "  - $pkg"
+        done
+    ) > environment-template.yml
+    (
+        sed 's/name: sage-build/name: sage/' environment-template.yml
+        echo "  # Additional packages providing all dependencies for the Sage library"
+        for pkg in $SAGELIB_SYSTEM_PACKAGES; do
+            echo "  - $pkg"
+        done
+    ) > src/environment-template.yml
 
-(
-    sed 's/name: sage/name: sage-dev/' src/environment.yml
-    echo "  # Additional dev tools"
-    for pkg in $DEVELOP_SYSTEM_PACKAGES; do
-        echo "  - $pkg"
-    done
-) > src/environment-dev.yml
+    (
+        cat environment-template.yml
+        echo "  # optional packages"
+        for pkg in $OPTIONAL_SYSTEM_PACKAGES; do
+            echo "  - $pkg"
+        done
+    ) > environment-optional-template.yml
 
-(
-    cat environment.yml
-    echo "  # optional packages"
-    for pkg in $OPTIONAL_SYSTEM_PACKAGES; do
-        echo "  - $pkg"
-    done
-) > environment-optional.yml
+    (
+        sed 's/name: sage/name: sage-dev/' src/environment-template.yml
+        echo "  # Additional dev tools"
+        for pkg in $DEVELOP_SYSTEM_PACKAGES; do
+            echo "  - $pkg"
+        done
+    ) > src/environment-dev-template.yml
 
-(
-    cat src/environment.yml
-    echo "  # optional packages"
-    for pkg in $OPTIONAL_SYSTEM_PACKAGES $SAGELIB_OPTIONAL_SYSTEM_PACKAGES; do
-        echo "  - $pkg"
-    done
-) > src/environment-optional.yml
-(
-    echo >&4 "  - pip:"
-    echo >&5 "  - pip:"
-    for PKG_BASE in $((sage-package list :standard: :optional: --has-file requirements.txt --no-file distros/conda.txt --no-file src; sage-package list :standard: :optional: --has-file install-requires.txt --no-file requirements.txt --no-file distros/conda.txt --no-file src) | sort); do
-        PKG_SCRIPTS=build/pkgs/$PKG_BASE
-        SYSTEM_PACKAGES_FILE=$PKG_SCRIPTS/requirements.txt
-        if [ ! -f $SYSTEM_PACKAGES_FILE ]; then
-            SYSTEM_PACKAGES_FILE=$PKG_SCRIPTS/install-requires.txt
-        fi
-        PKG_TYPE=$(cat $PKG_SCRIPTS/type)
-        if grep -q SAGERUNTIME $PKG_SCRIPTS/dependencies $PKG_SCRIPTS/dependencies_order_only 2>/dev/null; then
-            : # cannot install packages that depend on the Sage library
-        else
-            case "$PKG_BASE:$PKG_TYPE" in
-                $DEVELOP_SPKG_PATTERN:*) FD=4;;
-                *)                       FD=5;;
-            esac
-            ${STRIP_COMMENTS} $SYSTEM_PACKAGES_FILE | while read -r line; do
-                [ -n "$line" ] && echo >&$FD "    - $line"
-            done
-        fi
+    (
+        cat src/environment-template.yml
+        echo "  # optional packages"
+        for pkg in $OPTIONAL_SYSTEM_PACKAGES $SAGELIB_OPTIONAL_SYSTEM_PACKAGES; do
+            echo "  - $pkg"
+        done
+    ) > src/environment-optional-template.yml
+
+    (
+        echo >&4 "  - pip:"
+        echo >&5 "  - pip:"
+        for PKG_BASE in $(sage-package list :standard: :optional: --has-file requirements.txt --no-file distros/conda.txt --no-file src; sage-package list :standard: :optional: --has-file install-requires.txt --no-file requirements.txt --no-file distros/conda.txt --no-file src); do
+            PKG_SCRIPTS=build/pkgs/$PKG_BASE
+            SYSTEM_PACKAGES_FILE=$PKG_SCRIPTS/requirements.txt
+            if [ ! -f $SYSTEM_PACKAGES_FILE ]; then
+                SYSTEM_PACKAGES_FILE=$PKG_SCRIPTS/install-requires.txt
+            fi
+            PKG_TYPE=$(cat $PKG_SCRIPTS/type)
+            if grep -q SAGERUNTIME $PKG_SCRIPTS/dependencies $PKG_SCRIPTS/dependencies_order_only 2>/dev/null; then
+                : # cannot install packages that depend on the Sage library
+            else
+                case "$PKG_BASE:$PKG_TYPE" in
+                    $DEVELOP_SPKG_PATTERN:*) FD=4;;
+                    *:standard)              FD="4 5";;
+                    *)                       FD=5;;
+                esac
+                ${STRIP_COMMENTS} $SYSTEM_PACKAGES_FILE | while read -r line; do
+                    [ -n "$line" ] && for fd in $FD; do echo >&$fd "    - $line"; done
+                done
+            fi
+        done
+    ) 4>> /dev/null 5>> src/environment-optional-template.yml
+
+for f in environment environment-optional src/environment src/environment-optional src/environment-dev; do
+    for python_version in 3.9 3.10 3.11; do
+        sed -E 's/^( *- *)python *$/\1python='$python_version'/' $f-template.yml > $f-$python_version.yml
     done
-) 4>> src/environment-dev.yml 5>> src/environment-optional.yml
+    rm -f $f-template.yml
+done

build/pkgs/matplotlib/distros/conda.txt

@@ -1,2 +1,2 @@
 # Trac #33642: For unknown reasons, without the version constraint, conda installs only 3.3.2
-"matplotlib>=3.5.1"
+matplotlib>=3.5.1

src/doc/en/installation/conda.rst

@@ -75,13 +75,16 @@ from source as follows:
   - Create a new conda environment including all standard packages
     recognized by sage, and activate it::
 
-      $ conda env create --file environment.yml --name sage-build
+      $ conda env create --file environment-3.11.yml --name sage-build
       $ conda activate sage-build
 
-    Alternatively, use ``environment-optional.yml`` in place of
+    Alternatively, use ``environment-optional-3.11.yml`` in place of
     ``environment.yml`` to create an environment with all standard and optional
     packages recognized by sage.
 
+    A different Python version can be selected by replacing ``3.11`` by ``3.9``
+    or ``3.10`` in these commands.
+
   - Then the SageMath distribution will be built using the compilers provided by Conda
     and using many packages installed by Conda::
 
@@ -123,16 +126,15 @@ Here we assume that you are using a git checkout.
   - Create and activate a new conda environment with the dependencies of Sage
     and a few additional developer tools::
 
-      $ mamba env create --file src/environment-dev.yml --name sage-dev
+      $ mamba env create --file src/environment-dev-3.11.yml --name sage-dev
       $ conda activate sage-dev
 
-    Alternatively, you can use ``src/environment.yml`` or
-    ``src/environment-optional.yml``, which will only install standard
+    Alternatively, you can use ``src/environment-3.11.yml`` or
+    ``src/environment-optional-3.11.yml``, which will only install standard
     (and optional) packages without any additional developer tools.
 
-    By default, the most recent version of Python supported by Sage is
-    installed. You can use the additional option ``python=3.9`` in the above
-    ``env create`` command to select another Python version (here 3.9).
+    A different Python version can be selected by replacing ``3.11`` by ``3.9``
+    or ``3.10`` in these commands.
 
   - Install the build prerequisites and the Sage library::
 
@@ -142,7 +144,7 @@ Here we assume that you are using a git checkout.
   - Verify that Sage has been installed::
 
       $ sage -c 'print(version())'
-      SageMath version 9.6.beta5, Release Date: 2022-03-12
+      SageMath version 10.2.beta4, Release Date: 2023-09-24
 
 Note that ``make`` is not used at all. All dependencies
 (including all Python packages) are provided by conda.
@@ -162,7 +164,7 @@ After editing any Cython files, rebuild the Sage library using::
 
 In order to update the conda environment later, you can run::
 
-  $ mamba env update --file src/environment-dev.yml --name sage-dev
+  $ mamba env update --file src/environment-dev-3.11.yml --name sage-dev
 
 To build the documentation, use::