Merge branch 'develop' into some_ruff_fixes_for_UP

.ci/retrofit-worktree.sh

@@ -0,0 +1,39 @@
+#!/bin/sh
+if [ $# != 2 ]; then
+    echo >&2 "usage: $0 WORKTREE_NAME WORKTREE_DIRECTORY"
+    echo >&2 "Ensures that the current working directory is a git repository,"
+    echo >&2 "then makes WORKTREE_DIRECTORY a git worktree named WORKTREE_NAME."
+fi
+WORKTREE_NAME="$1"
+WORKTREE_DIRECTORY="$2"
+
+export GIT_AUTHOR_NAME="ci-sage workflow"
+export GIT_AUTHOR_EMAIL="[email protected]"
+export GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME"
+export GIT_COMMITTER_EMAIL="$GIT_AUTHOR_EMAIL"
+
+set -ex
+
+# If actions/checkout downloaded our source tree using the GitHub REST API
+# instead of with git (because do not have git installed in our image),
+# we first make the source tree a repo.
+if [ ! -d .git ]; then git init && git add -A && git commit --quiet -m "new"; fi
+
+# Tag this state of the source tree "new". This is what we want to build and test.
+git tag -f new
+
+# Our container image contains a source tree in $WORKTREE_DIRECTORY with a full build of Sage.
+# But $WORKTREE_DIRECTORY is not a git repository.
+# We make $WORKTREE_DIRECTORY a worktree whose index is at tag "new".
+# We then commit the current sources and set the tag "old". (This keeps all mtimes unchanged.)
+# Then we update worktree and index with "git reset --hard new".
+# (This keeps mtimes of unchanged files unchanged and mtimes of changed files newer than unchanged files.)
+# Finally we reset the index to "old". (This keeps all mtimes unchanged.)
+# The changed files now show up as uncommitted changes.
+# The final "git add -N" makes sure that files that were added in "new" do not show
+# as untracked files, which would be removed by "git clean -fx".
+git worktree add --detach $WORKTREE_NAME
+rm -rf $WORKTREE_DIRECTORY/.git && mv $WORKTREE_NAME/.git $WORKTREE_DIRECTORY/
+rm -rf $WORKTREE_NAME && ln -s $WORKTREE_DIRECTORY $WORKTREE_NAME
+if [ ! -f $WORKTREE_NAME/.gitignore ]; then cp .gitignore $WORKTREE_NAME/; fi
+(cd $WORKTREE_NAME && git add -A && git commit --quiet --allow-empty -m "old" -a && git tag -f old && git reset --hard new && git reset --quiet old && git add -N . && git status)

.github/workflows/build.yml

@@ -69,30 +69,8 @@ jobs:
         id: worktree
         run: |
           set -ex
-          git config --global user.email "[email protected]"
-          git config --global user.name "Build & Test workflow"
           git config --global --add safe.directory $(pwd)
-          # If actions/checkout downloaded our source tree using the GitHub REST API
-          # instead of with git (because do not have git installed in our image),
-          # we first make the source tree a repo.
-          if [ ! -d .git ]; then git init && git add -A && git commit --quiet -m "new"; fi
-          # Tag this state of the source tree "new". This is what we want to build and test.
-          git tag -f new
-          # Our container image contains a source tree in /sage with a full build of Sage.
-          # But /sage is not a git repository.
-          # We make /sage a worktree whose index is at tag "new".
-          # We then commit the current sources and set the tag "old". (This keeps all mtimes unchanged.)
-          # Then we update worktree and index with "git reset --hard new".
-          # (This keeps mtimes of unchanged files unchanged and mtimes of changed files newer than unchanged files.)
-          # Finally we reset the index to "old". (This keeps all mtimes unchanged.)
-          # The changed files now show up as uncommitted changes.
-          # The final "git add -N" makes sure that files that were added in "new" do not show
-          # as untracked files, which would be removed by "git clean -fx".
-          git worktree add --detach worktree-image
-          rm -rf /sage/.git && mv worktree-image/.git /sage/
-          rm -rf worktree-image && ln -s /sage worktree-image
-          if [ ! -f worktree-image/.gitignore ]; then cp .gitignore worktree-image/; fi
-          (cd worktree-image && git add -A && git commit --quiet --allow-empty -m "old" -a && git tag -f old && git reset --hard new && git reset --quiet old && git add -N . && git status)
+          .ci/retrofit-worktree.sh worktree-image /sage
 
       - name: Download upstream artifact
         uses: actions/download-artifact@v3
@@ -107,20 +85,19 @@ jobs:
             (cd worktree-image && git commit -q -m "current changes" --allow-empty -a && git am; git reset --quiet old; git add -N .) < upstream/ci_fixes.patch
           fi
 
-      - name: Incremental build, test changed files (sage -t --new)
+      - name: Incremental build
         id: incremental
         run: |
           # Now re-bootstrap and build. The build is incremental because we were careful with the timestamps.
-          # We run tests with "sage -t --new"; this only tests the uncommitted changes.
-          ./bootstrap && make build && ./sage -t --new -p2
+          ./bootstrap && make build
         working-directory: ./worktree-image
         env:
           MAKE: make -j2 --output-sync=recurse
           SAGE_NUM_THREADS: 2
 
-      - name: Build and test modularized distributions
+      - name: Build modularized distributions
         if: always() && steps.worktree.outcome == 'success'
-        run: make V=0 tox && make pypi-wheels
+        run: make V=0 tox && make SAGE_CHECK=no pypi-wheels
         working-directory: ./worktree-image
         env:
           MAKE: make -j2 --output-sync=recurse
@@ -165,6 +142,26 @@ jobs:
           MAKE: make -j2 --output-sync=recurse
           SAGE_NUM_THREADS: 2
 
+      # Testing
+
+      - name: Test changed files (sage -t --new)
+        if: always() && steps.build.outcome == 'success'
+        run: |
+          # We run tests with "sage -t --new"; this only tests the uncommitted changes.
+          ./sage -t --new -p2
+        working-directory: ./worktree-image
+        env:
+          MAKE: make -j2 --output-sync=recurse
+          SAGE_NUM_THREADS: 2
+
+      - name: Test modularized distributions
+        if: always() && steps.build.outcome == 'success'
+        run: make V=0 tox && make pypi-wheels-check
+        working-directory: ./worktree-image
+        env:
+          MAKE: make -j2 --output-sync=recurse
+          SAGE_NUM_THREADS: 2
+
       - name: Pytest
         if: contains(github.ref, 'pytest')
         run: |

.github/workflows/ci-conda.yml

@@ -39,18 +39,6 @@ jobs:
         env:
           GH_TOKEN: ${{ github.token }}
 
-      - name: Check for Miniconda
-        id: check_conda
-        run: echo ::set-output name=installed::$CONDA
-
-      # Miniconda is installed by default in the ubuntu-latest, however not in the act-image to run it locally
-      - name: Install Miniconda
-        if: steps.check_conda.outputs.installed == ''
-        run: |
-          wget https://repo.anaconda.com/miniconda/Miniconda3-py39_4.10.3-Linux-x86_64.sh -O ~/miniconda.sh
-          bash ~/miniconda.sh -b -p $HOME/miniconda
-          echo "CONDA=$HOME/miniconda" >> $GITHUB_ENV
-
       - name: Create conda environment files
         run: ./bootstrap-conda
 
@@ -61,12 +49,13 @@ jobs:
           key:
             ${{ runner.os }}-conda-${{ hashFiles('src/environment-3.11.yml') }}
 
-      - name: Setup Conda
+      - name: Setup Conda environment
         uses: conda-incubator/setup-miniconda@v2
         with:
           python-version: ${{ matrix.python }}
-          mamba-version: "*"
-          channels: conda-forge,defaults
+          miniforge-version: latest
+          use-mamba: true
+          channels: conda-forge
           channel-priority: true
           activate-environment: sage
           environment-file: src/${{ matrix.conda-env }}-${{ matrix.python }}.yml
@@ -92,7 +81,7 @@ jobs:
         run: |
           # Use --no-deps and pip check below to verify that all necessary dependencies are installed via conda.
           pip install --no-build-isolation --no-deps -v -v -e ./pkgs/sage-conf ./pkgs/sage-setup
-          pip install --no-build-isolation --no-deps -v -v -e ./src
+          pip install --no-build-isolation --no-deps --config-settings editable_mode=compat -v -v -e ./src
         env:
           SAGE_NUM_THREADS: 2
 

.github/workflows/ci-linux.yml

@@ -34,7 +34,46 @@ permissions:
 
 jobs:
 
+  # standard-pre for the default platform (used by build.yml etc.)
+  default-pre:
+    uses: ./.github/workflows/docker.yml
+    with:
+      # Build from scratch
+      docker_targets: "with-system-packages configured with-targets-pre"
+      # FIXME: duplicated from env.TARGETS
+      targets_pre: all-sage-local
+      tox_system_factors: >-
+        ["ubuntu-focal"]
+      tox_packages_factors: >-
+        ["standard"]
+      docker_push_repository: ghcr.io/${{ github.repository }}/
+
+  # standard for the default platform (used by build.yml etc.)
+  default:
+    if: ${{ success() || failure() }}
+    needs: [default-pre]
+    uses: ./.github/workflows/docker.yml
+    with:
+      # Build incrementally from previous stage (pre)
+      incremental: true
+      free_disk_space: true
+      from_docker_repository: ghcr.io/${{ github.repository }}/
+      from_docker_target: "with-targets-pre"
+      docker_targets: "with-targets with-targets-optional"
+      # FIXME: duplicated from env.TARGETS
+      targets: build doc-html
+      targets_optional: ptest
+      tox_system_factors: >-
+        ["ubuntu-focal"]
+      tox_packages_factors: >-
+          ["standard"]
+      docker_push_repository: ghcr.io/${{ github.repository }}/
+
+  # All platforms. This duplicates the default platform, but why not,
+  # it makes it more robust regarding random timeouts.
+
   standard-pre:
+    if: ${{ success() || failure() }}
     uses: ./.github/workflows/docker.yml
     with:
       # Build from scratch

.github/workflows/dist.yml

@@ -18,9 +18,8 @@ jobs:
 
   release_dist:
 
-    # This job, in contrast to "dist" in ci-macos.yml,
-    # does not use "configure --enable-download-from-upstream-url"
-    # (the default since #32390).
+    # This job first checks whether "configure --enable-download-from-upstream-url"
+    # (the default since #32390) is needed.
     #
     # In this way, we check that all necessary package tarballs
     # have already been uploaded to the Sage server at the time
@@ -37,13 +36,42 @@ jobs:
         run: |
           sudo DEBIAN_FRONTEND=noninteractive apt-get update
           sudo DEBIAN_FRONTEND=noninteractive apt-get install $(build/bin/sage-get-system-packages debian _bootstrap)
-      - name: make dist
+      - name: make dist (--disable-download-from-upstream-url)
         run: |
           ./bootstrap -D && ./configure --disable-download-from-upstream-url && make dist
+        env:
+          MAKE: make -j8
+      - name: make dist (--enable-download-from-upstream-url)
+        if: failure()
+        run: |
+          ./configure && make dist
+        env:
+          MAKE: make -j8
       - uses: actions/upload-artifact@v3
+        if: success() || failure()
+        with:
+          path: |
+            dist/*.tar.gz
+            upstream
+          name: release_dist
+
+  release:
+
+    needs: release_dist
+    runs-on: ubuntu-latest
+    if: (success() || failure()) && github.repository == 'sagemath/sage' && startsWith(github.ref, 'refs/tags/') && !contains(github.ref, 'beta') && !contains(github.ref, 'rc')
+    steps:
+      - uses: actions/download-artifact@v3
         with:
-          path: "dist/*.tar.gz"
           name: release_dist
+      - uses: softprops/action-gh-release@v1
+        with:
+          generate_release_notes: true
+          files: |
+            dist/*
+            upstream/*
+    permissions:
+      contents: write
 
   sdists_for_pypi:
 

.github/workflows/doc-build-pdf.yml

@@ -62,31 +62,10 @@ jobs:
       - name: Add prebuilt tree as a worktree
         id: worktree
         run: |
-          set -ex
+          git config --global --add safe.directory $(pwd)
           git config --global user.email "[email protected]"
           git config --global user.name "Build & Test workflow"
-          git config --global --add safe.directory $(pwd)
-          # If actions/checkout downloaded our source tree using the GitHub REST API
-          # instead of with git (because do not have git installed in our image),
-          # we first make the source tree a repo.
-          if [ ! -d .git ]; then git init && git add -A && git commit --quiet -m "new"; fi
-          # Tag this state of the source tree "new". This is what we want to build and test.
-          git tag -f new
-          # Our container image contains a source tree in /sage with a full build of Sage.
-          # But /sage is not a git repository.
-          # We make /sage a worktree whose index is at tag "new".
-          # We then commit the current sources and set the tag "old". (This keeps all mtimes unchanged.)
-          # Then we update worktree and index with "git reset --hard new".
-          # (This keeps mtimes of unchanged files unchanged and mtimes of changed files newer than unchanged files.)
-          # Finally we reset the index to "old". (This keeps all mtimes unchanged.)
-          # The changed files now show up as uncommitted changes.
-          # The final "git add -N" makes sure that files that were added in "new" do not show
-          # as untracked files, which would be removed by "git clean -fx".
-          git worktree add --detach worktree-image
-          rm -rf /sage/.git && mv worktree-image/.git /sage/
-          rm -rf worktree-image && ln -s /sage worktree-image
-          if [ ! -f worktree-image/.gitignore ]; then cp .gitignore worktree-image/; fi
-          (cd worktree-image && git add -A && git commit --quiet --allow-empty -m "old" -a && git tag -f old && git reset --hard new && git reset --quiet old && git add -N . && git status)
+          .ci/retrofit-worktree.sh worktree-image /sage
           # Keep track of changes to built HTML
           new_version=$(cat src/VERSION.txt); (cd /sage/local/share/doc/sage/html/en && find . -name "*.html" | xargs sed -i '/class="sidebar-brand-text"/s/Sage [0-9a-z.]* /Sage '$new_version' /'; git init && (echo "*.svg binary"; echo "*.pdf binary") >> .gitattributes && (echo ".buildinfo"; echo '*.inv'; echo '.git*'; echo '*.svg'; echo '*.pdf'; echo '*.png'; echo 'searchindex.js') > .gitignore; git add -A && git commit --quiet -m "old")
 
@@ -110,27 +89,28 @@ jobs:
           ./bootstrap && make build
         working-directory: ./worktree-image
         env:
-          MAKE: make -j2
+          MAKE: make -j2 --output-sync=recurse
           SAGE_NUM_THREADS: 2
 
       - name: Build (fallback to non-incremental)
         id: build
         if: always() && steps.worktree.outcome == 'success' && steps.incremental.outcome != 'success'
         run: |
           set -ex
-          make doc-clean doc-uninstall sagelib-clean && git clean -fx src/sage && ./config.status && make build
+          make sagelib-clean && git clean -fx src/sage && ./config.status && make build
         working-directory: ./worktree-image
         env:
-          MAKE: make -j2
+          MAKE: make -j2 --output-sync=recurse
           SAGE_NUM_THREADS: 2
 
       - name: Build docs (PDF)
         id: docbuild
         if: always() && (steps.incremental.outcome == 'success' || steps.build.outcome == 'success')
-        run: make build V=0 && make doc-pdf
+        run: |
+          make doc-clean doc-uninstall; make doc-pdf
         working-directory: ./worktree-image
         env:
-          MAKE: make -j2
+          MAKE: make -j2 --output-sync=recurse
           SAGE_NUM_THREADS: 2
 
       - name: Copy docs

.github/workflows/doc-build.yml

@@ -53,31 +53,10 @@ jobs:
       - name: Add prebuilt tree as a worktree
         id: worktree
         run: |
-          set -ex
+          git config --global --add safe.directory $(pwd)
           git config --global user.email "[email protected]"
           git config --global user.name "Build & Test workflow"
-          git config --global --add safe.directory $(pwd)
-          # If actions/checkout downloaded our source tree using the GitHub REST API
-          # instead of with git (because do not have git installed in our image),
-          # we first make the source tree a repo.
-          if [ ! -d .git ]; then git init && git add -A && git commit --quiet -m "new"; fi
-          # Tag this state of the source tree "new". This is what we want to build and test.
-          git tag -f new
-          # Our container image contains a source tree in /sage with a full build of Sage.
-          # But /sage is not a git repository.
-          # We make /sage a worktree whose index is at tag "new".
-          # We then commit the current sources and set the tag "old". (This keeps all mtimes unchanged.)
-          # Then we update worktree and index with "git reset --hard new".
-          # (This keeps mtimes of unchanged files unchanged and mtimes of changed files newer than unchanged files.)
-          # Finally we reset the index to "old". (This keeps all mtimes unchanged.)
-          # The changed files now show up as uncommitted changes.
-          # The final "git add -N" makes sure that files that were added in "new" do not show
-          # as untracked files, which would be removed by "git clean -fx".
-          git worktree add --detach worktree-image
-          rm -rf /sage/.git && mv worktree-image/.git /sage/
-          rm -rf worktree-image && ln -s /sage worktree-image
-          if [ ! -f worktree-image/.gitignore ]; then cp .gitignore worktree-image/; fi
-          (cd worktree-image && git add -A && git commit --quiet --allow-empty -m "old" -a && git tag -f old && git reset --hard new && git reset --quiet old && git add -N . && git status)
+          .ci/retrofit-worktree.sh worktree-image /sage
           # Keep track of changes to built HTML
           new_version=$(cat src/VERSION.txt); (cd /sage/local/share/doc/sage/html/en && find . -name "*.html" | xargs sed -i '/class="sidebar-brand-text"/s/Sage [0-9a-z.]* /Sage '$new_version' /'; git init && (echo "*.svg binary"; echo "*.pdf binary") >> .gitattributes && (echo ".buildinfo"; echo '*.inv'; echo '.git*'; echo '*.svg'; echo '*.pdf'; echo '*.png'; echo 'searchindex.js') > .gitignore; git add -A && git commit --quiet -m "old")
 
@@ -109,7 +88,7 @@ jobs:
         if: always() && steps.worktree.outcome == 'success' && steps.incremental.outcome != 'success'
         run: |
           set -ex
-          make doc-clean doc-uninstall sagelib-clean && git clean -fx src/sage && ./config.status && make build
+          make sagelib-clean && git clean -fx src/sage && ./config.status && make build
         working-directory: ./worktree-image
         env:
           MAKE: make -j2 --output-sync=recurse
@@ -124,7 +103,7 @@ jobs:
           set -ex
           export SAGE_USE_CDNS=yes
           mv /sage/local/share/doc/sage/html/en/.git /sage/.git-doc
-          make doc-clean doc-uninstall sagelib-clean && git clean -fx src/sage
+          make doc-clean doc-uninstall
           mkdir -p /sage/local/share/doc/sage/html/en/ && mv /sage/.git-doc /sage/local/share/doc/sage/html/en/.git
           ./config.status && make doc-html
         working-directory: ./worktree-image