Merge tag '9.7.beta2' into t/28925/modify_find_python_sources__clean_…

…stale_files_to_support_modularization_of_sagelib_by_native_namespace_packages__pep_420_

SageMath version 9.7.beta2, Release Date: 2022-06-12

.github/workflows/build.yml

@@ -29,11 +29,13 @@ jobs:
 
       - name: Prepare
         run: |
+          # Install test tools.
+          # Installation of python3-venv can be removed as soon as a
+          # base image with a release including #33822 is available
+          apt-get install -y git python3-venv
           # Reuse built SAGE_LOCAL contained in the Docker image 
           ./bootstrap
           ./configure --enable-build-as-root --prefix=/sage/local --with-sage-venv --enable-editable --enable-download-from-upstream-url
-          # Install test tools
-          apt-get install -y git
       
       - name: Static code check with pyright
         run: pyright    
@@ -44,6 +46,16 @@ jobs:
           MAKE: make -j2
           SAGE_NUM_THREADS: 2
 
+      - name: Pytest
+        if: contains(github.ref, 'pytest')
+        run: |
+          ../sage -python -m pip install coverage pytest-xdist
+          ../sage -python -m coverage run -m pytest -c tox.ini --doctest-modules || true
+        working-directory: ./src
+        env:
+          # Increase the length of the lines in the "short summary"
+          COLUMNS: 120
+
       - name: Test
         run: |
           ../sage -python -m pip install coverage

.github/workflows/doc-build.yml

@@ -24,6 +24,7 @@ jobs:
           # Reuse built SAGE_LOCAL contained in the Docker image 
           ./bootstrap
           ./configure --enable-build-as-root --prefix=/sage/local --with-sage-venv --enable-download-from-upstream-url
+          make furo
 
       - name: Build
         run: make doc-html

.github/workflows/tox.yml

@@ -38,7 +38,7 @@ jobs:
       fail-fast: false
       max-parallel: 20
       matrix:
-        tox_system_factor: [gitpod, ubuntu-trusty, ubuntu-xenial, ubuntu-bionic, ubuntu-focal, ubuntu-hirsute, ubuntu-impish, ubuntu-jammy,  debian-stretch, debian-buster, debian-bullseye, debian-bookworm, debian-sid, linuxmint-17, linuxmint-18, linuxmint-19, linuxmint-19.3, linuxmint-20.1, linuxmint-20.2, linuxmint-20.3, fedora-26, fedora-27, fedora-28, fedora-29, fedora-30, fedora-31, fedora-32, fedora-33, fedora-34, fedora-35, fedora-36, centos-7, centos-stream-8, gentoo-python3.9, gentoo-python3.10, archlinux-latest, opensuse-15.3, opensuse-tumbleweed, slackware-14.2, conda-forge, ubuntu-bionic-i386, manylinux-2_24-i686, debian-buster-i386, centos-7-i386]
+        tox_system_factor: [ubuntu-trusty, ubuntu-xenial, ubuntu-bionic, ubuntu-focal, ubuntu-hirsute, ubuntu-impish, ubuntu-jammy,  debian-stretch, debian-buster, debian-bullseye, debian-bookworm, debian-sid, linuxmint-17, linuxmint-18, linuxmint-19, linuxmint-19.3, linuxmint-20.1, linuxmint-20.2, linuxmint-20.3, fedora-26, fedora-27, fedora-28, fedora-29, fedora-30, fedora-31, fedora-32, fedora-33, fedora-34, fedora-35, fedora-36, centos-7, centos-stream-8, gentoo-python3.9, gentoo-python3.10, archlinux-latest, opensuse-15.3, opensuse-tumbleweed, slackware-14.2, conda-forge, ubuntu-bionic-i386, manylinux-2_24-i686, debian-buster-i386, centos-7-i386]
         tox_packages_factor: [minimal, standard]
     env:
       TOX_ENV: docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}

.gitignore

@@ -166,10 +166,16 @@ build/bin/sage-build-env-config
 
 /pkgs/sagemath-objects/setup.cfg
 /pkgs/sagemath-categories/setup.cfg
+/pkgs/sagemath-environment/setup.cfg
+/pkgs/sagemath-repl/setup.cfg
 /pkgs/sagemath-objects/pyproject.toml
 /pkgs/sagemath-categories/pyproject.toml
+/pkgs/sagemath-environment/pyproject.toml
+/pkgs/sagemath-repl/pyproject.toml
 /pkgs/sagemath-objects/requirements.txt
 /pkgs/sagemath-categories/requirements.txt
+/pkgs/sagemath-environment/requirements.txt
+/pkgs/sagemath-repl/requirements.txt
 /pkgs/sagemath-categories/MANIFEST.in
 
 # same for old locations - before Trac #31577

.gitpod.yml

@@ -1,41 +1,32 @@
 # Use custom docker image. https://www.gitpod.io/docs/config-docker
 image:
-# Each of these two options works:
-# - Directly use the "-with-targets" image built by .github/workflows/tox.yml
-#   image: ghcr.io/sagemath/sage/sage-docker-gitpod-standard-with-targets:dev
-# - or go through a custom Dockerfile, which builds a smaller image
-#   based on the "-with-system-packages" image built by .github/workflows/tox.yml
-#   with the built SAGE_LOCAL from the "-with-targets" image copied in.
-    file: docker/.gitpod.Dockerfile
-
-# We use the following layout:
-#
-# $HOME/sage            Source tree used by the Docker build; see SAGE_ROOT/tox.ini (gitpod).
-#                       - We delete it in every invocation of the 'before' script
-#                         and replace it by a symlink to /workspace/...
-#                         (This symlink is needed because the package-removal scripts
-#                         ({prefix,venv}/var/lib/sage/scripts/*/{spkg-piprm,spkg-prerm,spkg-postrm)
-#                         hardcode SAGE_ROOT and SAGE_SRC from package installation time)
-# $HOME/sage/logs       Logs of the Docker build.
-#                       - In the first invocation of the 'before' script, we move it
-#                         to /workspace/.../logs
-# $HOME/sage-local      The configured prefix (SAGE_LOCAL) of the Sage installation.
-#                       - During the Docker build, this is the physical location.
-#                       - In the first invocation of the 'before' script, we move it
-#                         to the new physical location /workspace/.../local
-#                         (because gitpod only preserves the contents of /workspace)
-#                         and replace it by a symlink to the new physical location.
-#                       - In subsequent invocations of the 'before' script, we
-#                         remove it and replace it by a symlink to the physical
-#                         location /workspace/.../local
-# /worktree/.../local   The physical location of the Sage installation,
-#                       established in the first run of the 'before' script and
-#                       preserved by gitpod.
+  file: docker/.gitpod.Dockerfile
 
 # Start up tasks. https://www.gitpod.io/docs/config-start-tasks/
 tasks:
   - name: Setup
-    before: |
+    init: |
+      # Create conda environment
+      ./bootstrap-conda
+      mamba env create --file src/environment-dev.yml --prefix venv
+      conda config --append envs_dirs /workspace/sagetrac-mirror
+      conda activate /workspace/sagetrac-mirror/venv
+      
+      # Build sage
+      ./bootstrap
+      ./configure --enable-build-as-root --with-python=$CONDA_PREFIX/bin/python --prefix=$CONDA_PREFIX 
+      pip install --no-build-isolation -v -v -e ./pkgs/sage-conf ./pkgs/sage-setup 
+      pip install --no-build-isolation -v -v -e ./src
+
+    command: |
+      # Activate conda environment
+      conda config --append envs_dirs /workspace/sagetrac-mirror
+      conda activate /workspace/sagetrac-mirror/venv
+
+      # RestructuredText extension recommends python extension, although we have already installed it
+      ## So disable the recommendation dialog
+      echo "{\"restructuredtext.pythonRecommendation.disabled\": true}" > /workspace/.vscode-remote/data/Machine/settings.json
+
       # Setup trac as remote
       ## In order to push to trac, generate a new key with `ssh-keygen -f tempkey` and save the private key to gitpod `gp env PRIVATE_SSH_KEY="$(<tempkey)"` (or by following https://www.gitpod.io/docs/environment-variables#using-the-account-settings)
       ## then follow https://doc.sagemath.org/html/en/developer/trac.html#linking-your-public-key-to-your-trac-account to register the public key with trac.
@@ -47,7 +38,7 @@ tasks:
         echo $PRIVATE_SSH_KEY | sed 's/\(-----\(BEGIN\|END\) OPENSSH PRIVATE KEY-----\)/\n\1\n/g' > ~/.ssh/id_rsa
         sed -i '/^$/d' ~/.ssh/id_rsa
         chmod 600 ~/.ssh/id_rsa
-        unset PRIVATE_SSH_KEY
+        echo "PubkeyAcceptedKeyTypes +ssh-rsa" > ~/.ssh/config
         ssh-keyscan -H trac.sagemath.org >> ~/.ssh/known_hosts
 
         # Setup trac repo
@@ -60,43 +51,7 @@ tasks:
         git remote add trac https://github.com/sagemath/sagetrac-mirror.git -t master -t develop
         git remote set-url --push trac pushing-needs-ssh-key
       fi
-      
-      ## No need for pyenv
-      pyenv shell --unset 2> /dev/null
-      pyenv global system 2> /dev/null
-      if [ -d local ]; then
-        mkdir -p logs && echo '### .gitpod.yml Setup.before: Prebuild init script has been run. Running "make" again in case the build had timed out.' >> logs/install.log
-        # The init script has already populated the SAGE_LOCAL,
-        # but only /workspace is preserved; and the $HOME/sage-local may contain a resurrected
-        # copy of sage-local.  Replace it again by a symlink.
-        rm -Rf $HOME/sage-local
-        ln -sf $(pwd)/local $HOME/sage-local
-        # Now run make. No timeout here.
-        MAKE='make -j24' make build V=0
-      else
-        # Prebuild init script has not been run
-        # Only /workspace is preserved during build.
-        # If the Docker image contains a built SAGE_LOCAL, use it to populate the SAGE_LOCAL in the workspace.
-        if [ -d $HOME/sage-local ]; then
-          mv $HOME/sage-local local
-        fi
-        rm -Rf $HOME/sage-local
-        ln -sf $(pwd)/local $HOME/sage-local
-        # Save the logs of the source tree used by the Docker build
-        if [ -d $HOME/sage/logs ]; then
-          mv $HOME/sage/logs logs
-        fi
-      fi
-      # Remove the source tree used by the Docker build and replace it by a symlink
-      rm -Rf $HOME/sage
-      ln -s $(pwd) $HOME/sage
-    init: |
-      # Start build
-      mkdir -p logs && echo '### .gitpod.yml Setup.init: Starting build' >> logs/install.log
-      ./bootstrap
-      ./configure --enable-editable --enable-download-from-upstream-url --prefix=$HOME/sage-local --with-sage-venv
-      ## Gitpod has a timeout of 1h, so make sure we are below this to ensure that the prebuild is always successful
-      MAKE='make -j24' timeout 51m make build V=0 || echo "(ignoring error)"
+
     env:
       SAGE_NUM_THREADS: 8
 
@@ -109,6 +64,8 @@ vscode:
     - lextudio.restructuredtext
     - streetsidesoftware.code-spell-checker
     - ms-toolsai.jupyter
+    - ms-toolsai.jupyter-keymap
+    - ms-toolsai.jupyter-renderers
 
 # https://www.gitpod.io/docs/prebuilds#github-specific-configuration
 github:

.vscode/settings.json

@@ -21,7 +21,10 @@
         "--doctest-modules",
     ],
     "python.testing.unittestEnabled": false,
+    "python.linting.pycodestyleEnabled": true,
+    "python.linting.enabled": true,
     "cSpell.words": [
+        "furo"
         "Conda",
         "Cython"
     ],

.zenodo.json

@@ -1,10 +1,10 @@
 {
     "description": "Mirror of the Sage https://sagemath.org/ source tree", 
     "license": "other-open", 
-    "title": "sagemath/sage: 9.7.beta1", 
-    "version": "9.7.beta1", 
+    "title": "sagemath/sage: 9.7.beta2", 
+    "version": "9.7.beta2", 
     "upload_type": "software", 
-    "publication_date": "2022-05-26", 
+    "publication_date": "2022-06-12", 
     "creators": [
         {
             "affiliation": "SageMath.org", 
@@ -15,7 +15,7 @@
     "related_identifiers": [
         {
             "scheme": "url", 
-            "identifier": "https://github.com/sagemath/sage/tree/9.7.beta1", 
+            "identifier": "https://github.com/sagemath/sage/tree/9.7.beta2", 
             "relation": "isSupplementTo"
         }, 
         {

Makefile

@@ -87,6 +87,8 @@ pypi-sdists: sage_setup
 	./sage --sh build/pkgs/sagelib/spkg-src
 	./sage --sh build/pkgs/sagemath_objects/spkg-src
 	./sage --sh build/pkgs/sagemath_categories/spkg-src
+	./sage --sh build/pkgs/sagemath_environment/spkg-src
+	./sage --sh build/pkgs/sagemath_repl/spkg-src
 	@echo "Built sdists are in upstream/"
 
 # ssl: build Sage, and also install pyOpenSSL. This is necessary for
@@ -168,7 +170,6 @@ bootstrap-clean:
 	rm -rf config configure build/make/Makefile-auto.in
 	rm -f src/doc/en/installation/*.txt
 	rm -rf src/doc/en/reference/spkg/*.rst
-	rm -f src/doc/en/reference/repl/*.txt
 	rm -f environment.yml
 	rm -f src/environment.yml
 	rm -f src/environment-dev.yml

README.md

@@ -251,7 +251,7 @@ in the Installation Guide.
       ``ExtUtils::MakeMaker``), `ranlib`, `git`, `tar`, `bc`.
 
     - Python 3.4 or later, or Python 2.7, a full installation including
-      `urllib`; but ideally version 3.7.x, 3.8.x, 3.9.x, or 3.10.x, which
+      `urllib`; but ideally version 3.8.x, 3.9.x, or 3.10.x, which
       will avoid having to build Sage's own copy of Python 3.
 
     We have collected lists of system packages that provide these build

VERSION.txt

@@ -1 +1 @@
-SageMath version 9.7.beta1, Release Date: 2022-05-26
+SageMath version 9.7.beta2, Release Date: 2022-06-12

bootstrap

@@ -85,12 +85,12 @@ bootstrap () {
     rm -f m4/sage_spkg_configures.m4
     spkg_configures=""
     # initialize SAGE_ENABLE... options for standard packages
-    for pkgname in $(./sage --package list :standard: | sort); do
+    for pkgname in $(sage-package list :standard: | sort); do
         spkg_configures="$spkg_configures
 AS_VAR_SET_IF([SAGE_ENABLE_$pkgname], [], [AS_VAR_SET([SAGE_ENABLE_$pkgname], [yes])])"
     done
     # --enable-SPKG options
-    for pkgname in $(./sage --package list :optional: :experimental: | sort); do
+    for pkgname in $(sage-package list :optional: :experimental: | sort); do
                 # Trac #29629: Temporary solution for Sage 9.1: Do not provide
                 # --enable-SPKG options for installing pip packages
                 if [ ! -f build/pkgs/$pkgname/requirements.txt ]; then
@@ -109,12 +109,12 @@ SAGE_SPKG_ENABLE([$pkgname], [$pkgtype], [$(grep -v ^= build/pkgs/$pkgname/SPKG.
                     esac
                 fi
     done
-    for pkgname in $(./sage --package list --has-file spkg-configure.m4 | sort); do
+    for pkgname in $(sage-package list --has-file spkg-configure.m4 | sort); do
         echo "m4_sinclude([build/pkgs/$pkgname/spkg-configure.m4])"
         spkg_configures="$spkg_configures
 SAGE_SPKG_CONFIGURE_$(echo ${pkgname} | tr '[a-z]' '[A-Z]')"
     done >> m4/sage_spkg_configures.m4
-    for pkgname in $(./sage --package list | sort); do
+    for pkgname in $(sage-package list | sort); do
         DIR=build/pkgs/$pkgname
         pkgtype="$(cat $DIR/type)"
         if test -f "$DIR/requirements.txt"; then
@@ -143,7 +143,7 @@ SAGE_SPKG_FINALIZE([$pkgname], [$pkgtype], [$SPKG_SOURCE], [$SPKG_TREE_VAR])"
     done
     echo "$spkg_configures" >> m4/sage_spkg_configures.m4
 
-    for pkgname in $(./sage --package list --has-file bootstrap); do
+    for pkgname in $(sage-package list --has-file bootstrap); do
         (cd build/pkgs/$pkgname && ./bootstrap) || exit 1
     done
 
@@ -247,7 +247,6 @@ save () {
 	build/make/Makefile-auto.in \
 	src/doc/en/installation/*.txt \
 	src/doc/en/reference/spkg/*.rst \
-	src/doc/en/reference/repl/*.txt \
 	environment.yml \
 	src/environment.yml \
 	environment-optional.yml \
@@ -262,10 +261,10 @@ save () {
 
     # Compute checksum
     if [ "${BOOTSTRAP_QUIET}" = "no" ]; then
-	./sage --package fix-checksum configure
+	sage-package fix-checksum configure
     else
 	# Hide the "Updating checksum..." message
-	./sage --package fix-checksum configure > /dev/null
+	sage-package fix-checksum configure > /dev/null
     fi
 }
 

bootstrap-conda

@@ -4,6 +4,8 @@
 # Generate auto-generated conda environment files
 #########################################################################
 
+export PATH="$(pwd)/build/bin:$PATH"
+
 STRIP_COMMENTS="sed s/#.*//;"
 RECOMMENDED_SPKG_PATTERN="@(_recommended$(for a in $(head -n 1 build/pkgs/_recommended/dependencies); do echo -n "|"$a; done))"
 
@@ -13,7 +15,7 @@ OPTIONAL_SYSTEM_PACKAGES=
 SAGELIB_SYSTEM_PACKAGES=
 SAGELIB_OPTIONAL_SYSTEM_PACKAGES=
 RECOMMENDED_SYSTEM_PACKAGES=
-for PKG_BASE in $(./sage --package list --has-file distros/conda.txt); do
+for PKG_BASE in $(sage-package list --has-file distros/conda.txt); do
     PKG_SCRIPTS=build/pkgs/$PKG_BASE
     SYSTEM_PACKAGES_FILE=$PKG_SCRIPTS/distros/conda.txt
     PKG_TYPE=$(cat $PKG_SCRIPTS/type)
@@ -65,6 +67,7 @@ echo "  # Additional dev tools" >> src/environment-dev.yml
 echo "  - openssh" >> src/environment-dev.yml
 echo "  - pycodestyle" >> src/environment-dev.yml
 echo "  - pytest" >> src/environment-dev.yml
+echo "  - esbonio" >> src/environment-dev.yml
 
 cp environment.yml environment-optional.yml
     echo "  # optional packages" >> environment-optional.yml

build/bin/sage-bootstrap-python

@@ -11,7 +11,17 @@ if [ -z "$SAGE_ORIG_PATH" ]; then
     # If not we're running from within sage-env just set the existing path
     SAGE_ORIG_PATH="$PATH"
 fi
-
+IFS=':'
+NEW_PATH=""
+for path in $SAGE_ORIG_PATH
+do
+    case "$path" in
+        */.pyenv/shims*);;
+        *) NEW_PATH="$NEW_PATH$path:";;
+    esac
+done
+unset IFS
+SAGE_ORIG_PATH=${NEW_PATH%%':'}
 # In particular, it is invoked by "bootstrap -d" for sage-download-file,
 # i.e., before a configure run, and by "sage-spkg", also for sage-download-file.
 # So it needs to find a python that has the urllib module.