From 6127016ec90f4c2f66a99b6e6966cc05df9eeb6c Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 4 Oct 2023 21:23:47 -0700 Subject: [PATCH 01/10] bootstrap-conda: Generate environment files with hard-coded Python minor versions (environment-3.9.yml etc.) This is convenient for conda-lock. Partial cherry-pick from "Use conda-lock for reproducible conda env" --- .devcontainer/onCreate-conda.sh | 2 +- .gitignore | 18 ++++ .gitpod.yml | 2 +- bootstrap-conda | 134 ++++++++++++------------ build/pkgs/matplotlib/distros/conda.txt | 2 +- 5 files changed, 89 insertions(+), 69 deletions(-) diff --git a/.devcontainer/onCreate-conda.sh b/.devcontainer/onCreate-conda.sh index eaa045cd27b..2cbb94e3492 100755 --- a/.devcontainer/onCreate-conda.sh +++ b/.devcontainer/onCreate-conda.sh @@ -4,7 +4,7 @@ set -e # Create conda environment ./bootstrap-conda conda install mamba -n base -c conda-forge -y -mamba env create --file src/environment-dev.yml || mamba env update --file src/environment-dev.yml +mamba env create --file src/environment-dev-3.11.yml || mamba env update --file src/environment-dev-3.11.yml conda init bash # Build sage diff --git a/.gitignore b/.gitignore index 2cec8a0cf62..43f58abcafe 100644 --- a/.gitignore +++ b/.gitignore @@ -27,11 +27,29 @@ # no longer generated, but may still be in user worktrees /src/lib/pkgconfig +# Environment files generated by bootstrap-conda. +# The files without Python version are no longer generated +# but may still be in users' directories. /environment.yml +/environment-3.9.yml +/environment-3.10.yml +/environment-3.11.yml /environment-optional.yml +/environment-optional-3.9.yml +/environment-optional-3.10.yml +/environment-optional-3.11.yml /src/environment.yml +/src/environment-3.9.yml +/src/environment-3.10.yml +/src/environment-3.11.yml /src/environment-dev.yml +/src/environment-dev-3.9.yml +/src/environment-dev-3.10.yml +/src/environment-dev-3.11.yml /src/environment-optional.yml +/src/environment-optional-3.9.yml +/src/environment-optional-3.10.yml +/src/environment-optional-3.11.yml /src/setup.cfg /src/requirements.txt diff --git a/.gitpod.yml b/.gitpod.yml index 52ac8d7184c..51c2687c5d3 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -8,7 +8,7 @@ tasks: # Create conda environment, then configure and build sage init: >- ./bootstrap-conda - && mamba env create --file src/environment-dev.yml --prefix venv + && mamba env create --file src/environment-dev-3.11.yml --prefix venv && conda config --append envs_dirs $(pwd) && conda activate $(pwd)/venv && ./bootstrap diff --git a/bootstrap-conda b/bootstrap-conda index ed4bb9e0d08..811dcbf43b0 100755 --- a/bootstrap-conda +++ b/bootstrap-conda @@ -51,72 +51,74 @@ for PKG_BASE in $(sage-package list --has-file distros/conda.txt --exclude _sage fi done echo >&2 $0:$LINENO: generate conda environment files -( - echo "name: sage-build" - echo "channels:" - echo " - conda-forge" - echo " - nodefaults" - echo "dependencies:" - for pkg in $SYSTEM_PACKAGES; do - echo " - $pkg" - done - echo " # Packages needed for ./bootstrap" - for pkg in $BOOTSTRAP_PACKAGES; do - echo " - $pkg" - done -) > environment.yml +for python_version in 3.9 3.10 3.11; do + ( + echo "name: sage-build" + echo "channels:" + echo " - conda-forge" + echo " - nodefaults" + echo "dependencies:" + echo " - python=$python_version" + for pkg in $SYSTEM_PACKAGES; do + echo " - $pkg" + done + echo " # Packages needed for ./bootstrap" + for pkg in $BOOTSTRAP_PACKAGES; do + echo " - $pkg" + done + ) > environment-$python_version.yml + ( + sed 's/name: sage-build/name: sage/' environment-$python_version.yml + echo " # Additional packages providing all dependencies for the Sage library" + for pkg in $SAGELIB_SYSTEM_PACKAGES; do + echo " - $pkg" + done + ) > src/environment-$python_version.yml -( - sed 's/name: sage-build/name: sage/' environment.yml - echo " # Additional packages providing all dependencies for the Sage library" - for pkg in $SAGELIB_SYSTEM_PACKAGES; do - echo " - $pkg" - done -) > src/environment.yml + ( + cat environment-$python_version.yml + echo " # optional packages" + for pkg in $OPTIONAL_SYSTEM_PACKAGES; do + echo " - $pkg" + done + ) > environment-optional-$python_version.yml -( - sed 's/name: sage/name: sage-dev/' src/environment.yml - echo " # Additional dev tools" - for pkg in $DEVELOP_SYSTEM_PACKAGES; do - echo " - $pkg" - done -) > src/environment-dev.yml - -( - cat environment.yml - echo " # optional packages" - for pkg in $OPTIONAL_SYSTEM_PACKAGES; do - echo " - $pkg" - done -) > environment-optional.yml - -( - cat src/environment.yml - echo " # optional packages" - for pkg in $OPTIONAL_SYSTEM_PACKAGES $SAGELIB_OPTIONAL_SYSTEM_PACKAGES; do - echo " - $pkg" - done -) > src/environment-optional.yml -( - echo >&4 " - pip:" - echo >&5 " - pip:" - for PKG_BASE in $((sage-package list :standard: :optional: --has-file requirements.txt --no-file distros/conda.txt --no-file src; sage-package list :standard: :optional: --has-file install-requires.txt --no-file requirements.txt --no-file distros/conda.txt --no-file src) | sort); do - PKG_SCRIPTS=build/pkgs/$PKG_BASE - SYSTEM_PACKAGES_FILE=$PKG_SCRIPTS/requirements.txt - if [ ! -f $SYSTEM_PACKAGES_FILE ]; then - SYSTEM_PACKAGES_FILE=$PKG_SCRIPTS/install-requires.txt - fi - PKG_TYPE=$(cat $PKG_SCRIPTS/type) - if grep -q SAGERUNTIME $PKG_SCRIPTS/dependencies $PKG_SCRIPTS/dependencies_order_only 2>/dev/null; then - : # cannot install packages that depend on the Sage library - else - case "$PKG_BASE:$PKG_TYPE" in - $DEVELOP_SPKG_PATTERN:*) FD=4;; - *) FD=5;; - esac - ${STRIP_COMMENTS} $SYSTEM_PACKAGES_FILE | while read -r line; do - [ -n "$line" ] && echo >&$FD " - $line" + ( + ( + sed 's/name: sage/name: sage-dev/' src/environment-$python_version.yml + echo " # Additional dev tools" + for pkg in $DEVELOP_SYSTEM_PACKAGES; do + echo " - $pkg" done - fi - done -) 4>> src/environment-dev.yml 5>> src/environment-optional.yml + ) >&4 + ( + cat src/environment-$python_version.yml + echo " # optional packages" + for pkg in $OPTIONAL_SYSTEM_PACKAGES $SAGELIB_OPTIONAL_SYSTEM_PACKAGES; do + echo " - $pkg" + done + ) >&5 + echo >&4 " - pip:" + echo >&5 " - pip:" + for PKG_BASE in $((sage-package list :standard: :optional: --has-file requirements.txt --no-file distros/conda.txt --no-file src; sage-package list :standard: :optional: --has-file install-requires.txt --no-file requirements.txt --no-file distros/conda.txt --no-file src) | sort); do + PKG_SCRIPTS=build/pkgs/$PKG_BASE + SYSTEM_PACKAGES_FILE=$PKG_SCRIPTS/requirements.txt + if [ ! -f $SYSTEM_PACKAGES_FILE ]; then + SYSTEM_PACKAGES_FILE=$PKG_SCRIPTS/install-requires.txt + fi + PKG_TYPE=$(cat $PKG_SCRIPTS/type) + if grep -q SAGERUNTIME $PKG_SCRIPTS/dependencies $PKG_SCRIPTS/dependencies_order_only 2>/dev/null; then + : # cannot install packages that depend on the Sage library + else + case "$PKG_BASE:$PKG_TYPE" in + $DEVELOP_SPKG_PATTERN:*) FD=4;; + *:standard) FD="4 5";; + *) FD=5;; + esac + ${STRIP_COMMENTS} $SYSTEM_PACKAGES_FILE | while read -r line; do + [ -n "$line" ] && for fd in $FD; do echo >&$fd " - $line"; done + done + fi + done + ) 4>> src/environment-dev-$python_version.yml 5>> src/environment-optional-$python_version.yml +done diff --git a/build/pkgs/matplotlib/distros/conda.txt b/build/pkgs/matplotlib/distros/conda.txt index 8b3901c7c9a..9fdcdfeb47f 100644 --- a/build/pkgs/matplotlib/distros/conda.txt +++ b/build/pkgs/matplotlib/distros/conda.txt @@ -1,2 +1,2 @@ # Trac #33642: For unknown reasons, without the version constraint, conda installs only 3.3.2 -"matplotlib>=3.5.1" +matplotlib>=3.5.1 From 03a2acfeade61d5bc57c1bfb79be802b8b10b7f2 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sun, 8 Oct 2023 13:07:57 -0700 Subject: [PATCH 02/10] src/doc/en/installation/conda.rst: Update for versioned environment-... files --- src/doc/en/installation/conda.rst | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/doc/en/installation/conda.rst b/src/doc/en/installation/conda.rst index 6b4cb871dd7..14fe52ad2e2 100644 --- a/src/doc/en/installation/conda.rst +++ b/src/doc/en/installation/conda.rst @@ -75,13 +75,16 @@ from source as follows: - Create a new conda environment including all standard packages recognized by sage, and activate it:: - $ conda env create --file environment.yml --name sage-build + $ conda env create --file environment-3.11.yml --name sage-build $ conda activate sage-build - Alternatively, use ``environment-optional.yml`` in place of + Alternatively, use ``environment-optional-3.11.yml`` in place of ``environment.yml`` to create an environment with all standard and optional packages recognized by sage. + A different Python version can be selected by replacing ``3.11`` by ``3.9`` + or ``3.10`` in these commands. + - Then the SageMath distribution will be built using the compilers provided by Conda and using many packages installed by Conda:: @@ -123,16 +126,15 @@ Here we assume that you are using a git checkout. - Create and activate a new conda environment with the dependencies of Sage and a few additional developer tools:: - $ mamba env create --file src/environment-dev.yml --name sage-dev + $ mamba env create --file src/environment-dev-3.11.yml --name sage-dev $ conda activate sage-dev - Alternatively, you can use ``src/environment.yml`` or - ``src/environment-optional.yml``, which will only install standard + Alternatively, you can use ``src/environment-3.11.yml`` or + ``src/environment-optional-3.11.yml``, which will only install standard (and optional) packages without any additional developer tools. - By default, the most recent version of Python supported by Sage is - installed. You can use the additional option ``python=3.9`` in the above - ``env create`` command to select another Python version (here 3.9). + A different Python version can be selected by replacing ``3.11`` by ``3.9`` + or ``3.10`` in these commands. - Install the build prerequisites and the Sage library:: @@ -142,7 +144,7 @@ Here we assume that you are using a git checkout. - Verify that Sage has been installed:: $ sage -c 'print(version())' - SageMath version 9.6.beta5, Release Date: 2022-03-12 + SageMath version 10.2.beta4, Release Date: 2023-09-24 Note that ``make`` is not used at all. All dependencies (including all Python packages) are provided by conda. @@ -162,7 +164,7 @@ After editing any Cython files, rebuild the Sage library using:: In order to update the conda environment later, you can run:: - $ mamba env update --file src/environment-dev.yml --name sage-dev + $ mamba env update --file src/environment-dev-3.11.yml --name sage-dev To build the documentation, use:: From 5c0bac856fb04cf589209ca5059a02f121e4b3ff Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 27 Sep 2023 12:37:35 -0700 Subject: [PATCH 03/10] bootstrap, Makefile (bootstrap-clean): Update for versioned environment-... files --- Makefile | 6 +----- bootstrap | 8 ++++---- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 11a7b77665a..ae90db4e9e2 100644 --- a/Makefile +++ b/Makefile @@ -158,11 +158,7 @@ bootstrap-clean: rm -rf config/install-sh config/compile config/config.guess config/config.sub config/missing configure build/make/Makefile-auto.in rm -f src/doc/en/installation/*.txt rm -rf src/doc/en/reference/spkg/*.rst - rm -f environment.yml - rm -f src/environment.yml - rm -f src/environment-dev.yml - rm -f environment-optional.yml - rm -f src/environment-optional.yml + for a in environment environment-optional src/environment src/environment-dev src/environment-optional; do rm -f $$a.yml $$a-3.[89].yml $$a-3.1[0-9].yml; done rm -f src/Pipfile rm -f src/pyproject.toml rm -f src/requirements.txt diff --git a/bootstrap b/bootstrap index 54d0a156239..2ae99949025 100755 --- a/bootstrap +++ b/bootstrap @@ -226,10 +226,10 @@ save () { build/make/Makefile-auto.in \ src/doc/en/installation/*.txt \ src/doc/en/reference/spkg/*.rst \ - environment.yml \ - src/environment.yml \ - environment-optional.yml \ - src/environment-optional.yml \ + environment-3.[89].yml environment-3.1[0-9].yml \ + src/environment-3.[89].yml src/environment-3.1[0-9].yml \ + environment-optional-3.[89].yml environment-optional-3.1[0-9].yml \ + src/environment-optional-3.[89].yml src/environment-optional-3.1[0-9].yml \ src/Pipfile \ src/pyproject.toml \ src/requirements.txt \ From c2be172347daec2c6c292e248eb598b406776577 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 27 Sep 2023 12:43:59 -0700 Subject: [PATCH 04/10] tox.ini (local-conda-environment): Update for versioned environment-... files --- tox.ini | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index c084e4c821e..9e124a281a6 100644 --- a/tox.ini +++ b/tox.ini @@ -527,9 +527,14 @@ setenv = local-conda-miniconda: CONDA_INSTALLER_FILE=Miniconda3-latest-{env:CONDA_OS}-x86_64.sh local-conda: SETENV=. {env:CONDA_PREFIX}/bin/activate base local-conda-environment: CONDA_SAGE_ENVIRONMENT=sage-build - local-conda-environment: CONDA_SAGE_ENVIRONMENT_FILE=environment.yml - local-conda-environment-optional: CONDA_SAGE_ENVIRONMENT_FILE=environment-optional.yml + local-conda-environment: CONDA_SAGE_ENVIRONMENT_DIR= + local-conda-environment-src: CONDA_SAGE_ENVIRONMENT=sage local-conda-environment-src: CONDA_SAGE_ENVIRONMENT_DIR=src/ + local-conda-environment: CONDA_SAGE_ENVIRONMENT_FILE=environment-{env:PYTHON_MAJOR}.{env:PYTHON_MINOR}.yml + local-conda-environment-optional: CONDA_SAGE_ENVIRONMENT_FILE=environment-optional-{env:PYTHON_MAJOR}.{env:PYTHON_MINOR}.yml + local-conda-environment-dev: CONDA_SAGE_ENVIRONMENT=sage-dev + local-conda-environment-dev: CONDA_SAGE_ENVIRONMENT_DIR=src/ + local-conda-environment-dev: CONDA_SAGE_ENVIRONMENT_FILE=environment-dev-{env:PYTHON_MAJOR}.{env:PYTHON_MINOR}.yml local-conda-environment: SETENV_CONFIGURE=( {env:CONDA_PREFIX}/bin/conda env create -n {env:CONDA_SAGE_ENVIRONMENT} --file {env:CONDA_SAGE_ENVIRONMENT_DIR:}{env:CONDA_SAGE_ENVIRONMENT_FILE} || {env:CONDA_PREFIX}/bin/conda env update -n {env:CONDA_SAGE_ENVIRONMENT} --file {env:CONDA_SAGE_ENVIRONMENT_DIR:}{env:CONDA_SAGE_ENVIRONMENT_FILE} ) && . {env:CONDA_PREFIX}/bin/activate {env:CONDA_SAGE_ENVIRONMENT} # # Configuration factors From b3ba591faf23946ff7bdd777a8c865bd474f039c Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 27 Sep 2023 13:17:26 -0700 Subject: [PATCH 05/10] .github/workflows/ci-conda.yml: Update for versioned environment*.yml --- .github/workflows/ci-conda.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-conda.yml b/.github/workflows/ci-conda.yml index 75babf3ab8c..231f0572f75 100644 --- a/.github/workflows/ci-conda.yml +++ b/.github/workflows/ci-conda.yml @@ -59,7 +59,7 @@ jobs: with: path: ~/conda_pkgs_dir key: - ${{ runner.os }}-conda-${{ hashFiles('src/environment.yml') }} + ${{ runner.os }}-conda-${{ hashFiles('src/environment-3.11.yml') }} - name: Setup Conda uses: conda-incubator/setup-miniconda@v2 @@ -69,7 +69,7 @@ jobs: channels: conda-forge,defaults channel-priority: true activate-environment: sage-build - environment-file: src/${{ matrix.conda-env }}.yml + environment-file: src/${{ matrix.conda-env }}-${{ matrix.python }}.yml - name: Print Conda environment shell: bash -l {0} From 018de2b76b3fefb561460e3feef3a69fa1c5e3be Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Tue, 3 Oct 2023 15:47:29 -0700 Subject: [PATCH 06/10] .github/workflows/ci-conda.yml: Fix environment name --- .github/workflows/ci-conda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-conda.yml b/.github/workflows/ci-conda.yml index 231f0572f75..81ab9fe3a61 100644 --- a/.github/workflows/ci-conda.yml +++ b/.github/workflows/ci-conda.yml @@ -68,7 +68,7 @@ jobs: mamba-version: "*" channels: conda-forge,defaults channel-priority: true - activate-environment: sage-build + activate-environment: sage environment-file: src/${{ matrix.conda-env }}-${{ matrix.python }}.yml - name: Print Conda environment From fc9bab5e35bf54388d9f5edd5e9e0d9668d4755c Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 27 Sep 2023 14:01:14 -0700 Subject: [PATCH 07/10] tox.ini (conda-environment-{src,dev}), .github/workflows/ci-conda.yml: Use configure --enable-system-site-packages --- .github/workflows/ci-conda.yml | 2 +- tox.ini | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-conda.yml b/.github/workflows/ci-conda.yml index 81ab9fe3a61..e520fa17104 100644 --- a/.github/workflows/ci-conda.yml +++ b/.github/workflows/ci-conda.yml @@ -83,7 +83,7 @@ jobs: run: | ./bootstrap echo "::add-matcher::.github/workflows/configure-systempackage-problem-matcher.json" - ./configure --enable-build-as-root --with-python=$CONDA_PREFIX/bin/python --prefix=$CONDA_PREFIX $(for pkg in $(./sage -package list :standard: --has-file spkg-configure.m4 --has-file distros/conda.txt --exclude rpy2); do echo --with-system-$pkg=force; done) + ./configure --enable-build-as-root --with-python=$CONDA_PREFIX/bin/python --prefix=$CONDA_PREFIX --enable-system-site-packages $(for pkg in $(./sage -package list :standard: --has-file spkg-configure.m4 --has-file distros/conda.txt --exclude rpy2); do echo --with-system-$pkg=force; done) echo "::remove-matcher owner=configure-system-package-warning::" echo "::remove-matcher owner=configure-system-package-error::" diff --git a/tox.ini b/tox.ini index 9e124a281a6..0f4540605e2 100644 --- a/tox.ini +++ b/tox.ini @@ -191,6 +191,7 @@ setenv = sitepackages: ENABLE_SYSTEM_SITE_PACKAGES=yes sitepackages: CONFIG_CONFIGURE_ARGS_SITEPACKAGES=--enable-system-site-packages conda-environment: SAGE_PACKAGE_LIST_ARGS=_prereq + conda-environment-{src,dev}: CONFIG_CONFIGURE_ARGS_SITEPACKAGES=--enable-system-site-packages # Whether to add the system packages needed for bootstrapping EXTRA_SAGE_PACKAGES_0=_bootstrap nobootstrap: EXTRA_SAGE_PACKAGES_0= From b02ef8eb62678bde34550ab1eb9ed8458ed9b099 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 4 Oct 2023 21:24:22 -0700 Subject: [PATCH 08/10] bootstrap-conda: Refactor using sage-get-system-packages --- bootstrap-conda | 90 +++++++++++++++++++++++++++++++++---------------- 1 file changed, 61 insertions(+), 29 deletions(-) diff --git a/bootstrap-conda b/bootstrap-conda index 811dcbf43b0..90d27ab7be7 100755 --- a/bootstrap-conda +++ b/bootstrap-conda @@ -11,45 +11,75 @@ STRIP_COMMENTS="sed s/#.*//;" shopt -s extglob DEVELOP_SPKG_PATTERN="@(_develop$(for a in $(head -n 1 build/pkgs/_develop/dependencies); do echo -n "|"$a; done))" -BOOTSTRAP_PACKAGES=$(echo $(${STRIP_COMMENTS} build/pkgs/_bootstrap/distros/conda.txt)) -SYSTEM_PACKAGES= -OPTIONAL_SYSTEM_PACKAGES= -SAGELIB_SYSTEM_PACKAGES= -SAGELIB_OPTIONAL_SYSTEM_PACKAGES= -DEVELOP_SYSTEM_PACKAGES= +BOOTSTRAP_PACKAGES=_bootstrap +PACKAGES= +OPTIONAL_PACKAGES= +SAGELIB_PACKAGES= +SAGELIB_OPTIONAL_PACKAGES= +DEVELOP_PACKAGES= + for PKG_BASE in $(sage-package list --has-file distros/conda.txt --exclude _sagemath); do PKG_SCRIPTS=build/pkgs/$PKG_BASE SYSTEM_PACKAGES_FILE=$PKG_SCRIPTS/distros/conda.txt PKG_TYPE=$(cat $PKG_SCRIPTS/type) PKG_SYSTEM_PACKAGES=$(echo $(${STRIP_COMMENTS} $SYSTEM_PACKAGES_FILE)) - if [ -n "PKG_SYSTEM_PACKAGES" ]; then + if [ -n "$PKG_SYSTEM_PACKAGES" ]; then if [ -f $PKG_SCRIPTS/spkg-configure.m4 ]; then + if grep -q SAGE_PYTHON_PACKAGE_CHECK $PKG_SCRIPTS/spkg-configure.m4; then + # Python package that would need --enable-system-site-packages to be used + # with the Sage distribution, but we do not recommend that for conda. + PKG_SAGELIB_ONLY=yes + else + PKG_SAGELIB_ONLY=no + fi + else + # No spkg-configure, so the Sage distribution is not able to make use of this package. + PKG_SAGELIB_ONLY=yes + fi + [ -n "$BOOTSTRAP_VERBOSE" ] && echo "$PKG_BASE:$PKG_TYPE:$PKG_SAGELIB_ONLY" + if [ $PKG_SAGELIB_ONLY = no ]; then case "$PKG_BASE:$PKG_TYPE" in *:standard) - SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES" + PACKAGES+=" $PKG_BASE" ;; $DEVELOP_SPKG_PATTERN:*) - DEVELOP_SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES" + DEVELOP_PACKAGES+=" $PKG_BASE" ;; *) - OPTIONAL_SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES" + OPTIONAL_PACKAGES+=" $PKG_BASE" ;; esac else case "$PKG_BASE:$PKG_TYPE" in *:standard) - SAGELIB_SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES" + SAGELIB_PACKAGES+=" $PKG_BASE" ;; $DEVELOP_SPKG_PATTERN:*) - DEVELOP_SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES" + DEVELOP_PACKAGES+=" $PKG_BASE" ;; *) - SAGELIB_OPTIONAL_SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES" + SAGELIB_OPTIONAL_PACKAGES+=" $PKG_BASE" ;; esac fi fi done +unset PKG_SYSTEM_PACKAGES + +[ -n "$BOOTSTRAP_VERBOSE" ] && echo "## Collected:" && set | grep PACKAGES= + +# Translate to system packages +export ENABLE_SYSTEM_SITE_PACKAGES=yes # Disable filtering in sage-get-system-packages +SYSTEM_PACKAGES=$(sage-get-system-packages conda $PACKAGES) +BOOTSTRAP_SYSTEM_PACKAGES=$(sage-get-system-packages conda $BOOTSTRAP_PACKAGES) +OPTIONAL_SYSTEM_PACKAGES=$(sage-get-system-packages conda $OPTIONAL_PACKAGES) +SAGELIB_SYSTEM_PACKAGES=$(sage-get-system-packages conda $SAGELIB_PACKAGES) +SAGELIB_OPTIONAL_SYSTEM_PACKAGES=$(sage-get-system-packages conda $SAGELIB_OPTIONAL_PACKAGES) +DEVELOP_SYSTEM_PACKAGES=$(sage-get-system-packages conda $DEVELOP_PACKAGES) +unset ENABLE_SYSTEM_SITE_PACKAGES + +[ -n "$BOOTSTRAP_VERBOSE" ] && echo "## Translated to system:" && set | grep SYSTEM_PACKAGES= + echo >&2 $0:$LINENO: generate conda environment files for python_version in 3.9 3.10 3.11; do ( @@ -63,7 +93,7 @@ for python_version in 3.9 3.10 3.11; do echo " - $pkg" done echo " # Packages needed for ./bootstrap" - for pkg in $BOOTSTRAP_PACKAGES; do + for pkg in $BOOTSTRAP_SYSTEM_PACKAGES; do echo " - $pkg" done ) > environment-$python_version.yml @@ -84,23 +114,25 @@ for python_version in 3.9 3.10 3.11; do ) > environment-optional-$python_version.yml ( - ( - sed 's/name: sage/name: sage-dev/' src/environment-$python_version.yml - echo " # Additional dev tools" - for pkg in $DEVELOP_SYSTEM_PACKAGES; do - echo " - $pkg" - done - ) >&4 - ( - cat src/environment-$python_version.yml - echo " # optional packages" - for pkg in $OPTIONAL_SYSTEM_PACKAGES $SAGELIB_OPTIONAL_SYSTEM_PACKAGES; do - echo " - $pkg" - done - ) >&5 + sed 's/name: sage/name: sage-dev/' src/environment-$python_version.yml + echo " # Additional dev tools" + for pkg in $DEVELOP_SYSTEM_PACKAGES; do + echo " - $pkg" + done + ) > src/environment-dev-$python_version.yml + + ( + cat src/environment-$python_version.yml + echo " # optional packages" + for pkg in $OPTIONAL_SYSTEM_PACKAGES $SAGELIB_OPTIONAL_SYSTEM_PACKAGES; do + echo " - $pkg" + done + ) > src/environment-optional-$python_version.yml + + ( echo >&4 " - pip:" echo >&5 " - pip:" - for PKG_BASE in $((sage-package list :standard: :optional: --has-file requirements.txt --no-file distros/conda.txt --no-file src; sage-package list :standard: :optional: --has-file install-requires.txt --no-file requirements.txt --no-file distros/conda.txt --no-file src) | sort); do + for PKG_BASE in $(sage-package list :standard: :optional: --has-file requirements.txt --no-file distros/conda.txt --no-file src; sage-package list :standard: :optional: --has-file install-requires.txt --no-file requirements.txt --no-file distros/conda.txt --no-file src); do PKG_SCRIPTS=build/pkgs/$PKG_BASE SYSTEM_PACKAGES_FILE=$PKG_SCRIPTS/requirements.txt if [ ! -f $SYSTEM_PACKAGES_FILE ]; then From 235ce91795439949e3b86f49f37bc0a72dba740b Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sun, 8 Oct 2023 23:23:18 -0700 Subject: [PATCH 09/10] bootstrap-conda: Only one 'python' line + speed up --- bootstrap-conda | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/bootstrap-conda b/bootstrap-conda index 90d27ab7be7..19e56e3c5fa 100755 --- a/bootstrap-conda +++ b/bootstrap-conda @@ -81,14 +81,13 @@ unset ENABLE_SYSTEM_SITE_PACKAGES [ -n "$BOOTSTRAP_VERBOSE" ] && echo "## Translated to system:" && set | grep SYSTEM_PACKAGES= echo >&2 $0:$LINENO: generate conda environment files -for python_version in 3.9 3.10 3.11; do + ( echo "name: sage-build" echo "channels:" echo " - conda-forge" echo " - nodefaults" echo "dependencies:" - echo " - python=$python_version" for pkg in $SYSTEM_PACKAGES; do echo " - $pkg" done @@ -96,38 +95,38 @@ for python_version in 3.9 3.10 3.11; do for pkg in $BOOTSTRAP_SYSTEM_PACKAGES; do echo " - $pkg" done - ) > environment-$python_version.yml + ) > environment-template.yml ( - sed 's/name: sage-build/name: sage/' environment-$python_version.yml + sed 's/name: sage-build/name: sage/' environment-template.yml echo " # Additional packages providing all dependencies for the Sage library" for pkg in $SAGELIB_SYSTEM_PACKAGES; do echo " - $pkg" done - ) > src/environment-$python_version.yml + ) > src/environment-template.yml ( - cat environment-$python_version.yml + cat environment-template.yml echo " # optional packages" for pkg in $OPTIONAL_SYSTEM_PACKAGES; do echo " - $pkg" done - ) > environment-optional-$python_version.yml + ) > environment-optional-template.yml ( - sed 's/name: sage/name: sage-dev/' src/environment-$python_version.yml + sed 's/name: sage/name: sage-dev/' src/environment-template.yml echo " # Additional dev tools" for pkg in $DEVELOP_SYSTEM_PACKAGES; do echo " - $pkg" done - ) > src/environment-dev-$python_version.yml + ) > src/environment-dev-template.yml ( - cat src/environment-$python_version.yml + cat src/environment-template.yml echo " # optional packages" for pkg in $OPTIONAL_SYSTEM_PACKAGES $SAGELIB_OPTIONAL_SYSTEM_PACKAGES; do echo " - $pkg" done - ) > src/environment-optional-$python_version.yml + ) > src/environment-optional-template.yml ( echo >&4 " - pip:" @@ -152,5 +151,11 @@ for python_version in 3.9 3.10 3.11; do done fi done - ) 4>> src/environment-dev-$python_version.yml 5>> src/environment-optional-$python_version.yml + ) 4>> src/environment-dev-template.yml 5>> src/environment-optional-template.yml + +for f in environment environment-optional src/environment src/environment-optional src/environment-dev; do + for python_version in 3.9 3.10 3.11; do + sed -E 's/^( *- *)python *$/\1python='$python_version'/' $f-template.yml > $f-$python_version.yml + done + rm -f $f-template.yml done From 73ab316abb4ffef3953b46bf0620d4a4cb2866e1 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 9 Oct 2023 19:47:55 -0700 Subject: [PATCH 10/10] bootstrap-conda (src/environment-dev*.yml): Suppress pip section, for now --- bootstrap-conda | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap-conda b/bootstrap-conda index 19e56e3c5fa..faa29513db4 100755 --- a/bootstrap-conda +++ b/bootstrap-conda @@ -151,7 +151,7 @@ echo >&2 $0:$LINENO: generate conda environment files done fi done - ) 4>> src/environment-dev-template.yml 5>> src/environment-optional-template.yml + ) 4>> /dev/null 5>> src/environment-optional-template.yml for f in environment environment-optional src/environment src/environment-optional src/environment-dev; do for python_version in 3.9 3.10 3.11; do