Merge branch 'develop' into random_subset_etc

.ci/merge-fixes.sh

@@ -0,0 +1,36 @@
+#!/bin/sh
+# Merge open PRs from sagemath/sage labeled "blocker".
+REPO="sagemath/sage"
+GH="gh -R $REPO"
+PRs="$($GH pr list --label "p: blocker / 1" --json number --jq '.[].number')"
+if [ -z "$PRs" ]; then
+    echo 'Nothing to do: Found no open PRs with "blocker" status.'
+else
+    echo "Found PRs: " $PRs
+    export GIT_AUTHOR_NAME="ci-sage workflow"
+    export GIT_AUTHOR_EMAIL="[email protected]"
+    export GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME"
+    export GIT_COMMITTER_EMAIL="$GIT_AUTHOR_EMAIL"
+    git tag -f test_base
+    git commit -q -m "Uncommitted changes" --no-allow-empty -a
+    for a in $PRs; do
+        echo "::group::Merging PR https://github.com/$REPO/pull/$a"
+        git tag -f test_head
+        $GH pr checkout -b pr-$a $a
+        git fetch --unshallow --all
+        git checkout -q test_head
+        if git merge --no-edit --squash -q pr-$a; then
+            echo "::endgroup::"
+            if git commit -q -m "Merge https://github.com/$REPO/pull/$a" -a --no-allow-empty; then
+                echo "Merged #$a"
+            else
+                echo "Empty, skipped"
+            fi
+        else
+            echo "::endgroup::"
+            echo "Failure merging #$a, resetting"
+            git reset --hard
+        fi
+    done
+    git log test_base..HEAD
+fi

.github/workflows/build.yml

@@ -24,9 +24,30 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  get_ci_fixes:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        id: checkout
+        uses: actions/checkout@v4
+      - name: Merge CI fixes from sagemath/sage
+        run: |
+          .ci/merge-fixes.sh
+        env:
+          GH_TOKEN: ${{ github.token }}
+      - name: Store CI fixes in upstream artifact
+        run: |
+          mkdir -p upstream
+          git format-patch --stdout test_base > upstream/ci_fixes.patch
+      - uses: actions/upload-artifact@v3
+        with:
+          path: upstream
+          name: upstream
+
   build:
     runs-on: ubuntu-latest
     container: ghcr.io/sagemath/sage/sage-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-with-targets:${{ github.event.inputs.docker_tag || 'dev'}}
+    needs: [get_ci_fixes]
     steps:
       - name: Checkout
         id: checkout
@@ -68,6 +89,19 @@ jobs:
           if [ ! -f worktree-image/.gitignore ]; then cp .gitignore worktree-image/; fi
           (cd worktree-image && git add -A && git commit --quiet --allow-empty -m "old" -a && git tag -f old && git reset --hard new && git reset --quiet old && git add -N . && git status)
 
+      - name: Download upstream artifact
+        uses: actions/download-artifact@v3
+        with:
+          path: upstream
+          name: upstream
+
+      - name: Apply CI fixes from sagemath/sage
+        # After applying the fixes, make sure all changes are marked as uncommitted changes.
+        run: |
+          if [ -r upstream/ci_fixes.patch ]; then
+            (cd worktree-image && git commit -q -m "current changes" --allow-empty -a && git am; git reset --quiet old; git add -N .) < upstream/ci_fixes.patch
+          fi
+
       - name: Incremental build, test changed files (sage -t --new)
         id: incremental
         run: |
@@ -76,15 +110,15 @@ jobs:
           ./bootstrap && make build && ./sage -t --new -p2
         working-directory: ./worktree-image
         env:
-          MAKE: make -j2
+          MAKE: make -j2 --output-sync=recurse
           SAGE_NUM_THREADS: 2
 
       - name: Build and test modularized distributions
         if: always() && steps.worktree.outcome == 'success'
         run: make V=0 tox && make pypi-wheels
         working-directory: ./worktree-image
         env:
-          MAKE: make -j2
+          MAKE: make -j2 --output-sync=recurse
           SAGE_NUM_THREADS: 2
 
       - name: Set up node to install pyright
@@ -123,7 +157,7 @@ jobs:
           make build
         working-directory: ./worktree-image
         env:
-          MAKE: make -j2
+          MAKE: make -j2 --output-sync=recurse
           SAGE_NUM_THREADS: 2
 
       - name: Pytest

.github/workflows/ci-conda.yml

@@ -51,6 +51,12 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - name: Merge CI fixes from sagemath/sage
+        run: |
+          .ci/merge-fixes.sh
+        env:
+          GH_TOKEN: ${{ github.token }}
+
       - name: Check for Miniconda
         id: check_conda
         run: echo ::set-output name=installed::$CONDA
@@ -99,9 +105,6 @@ jobs:
           echo "::remove-matcher owner=configure-system-package-warning::"
           echo "::remove-matcher owner=configure-system-package-error::"
 
-          # Manually install ptyprocess for now, until https://github.com/sagemath/sage/issues/32147 is fixed
-          pip install --no-build-isolation -v -v ptyprocess==0.5.1
-
       - name: Build
         shell: bash -l {0}
         run: |

.github/workflows/ci-linux-incremental.yml

@@ -60,7 +60,8 @@ jobs:
           echo "uninstall_targets=$(echo $(for a in '' ${{ steps.changed-packages.outputs.configures_all_changed_files }}; do echo $a | sed -E 's,build/pkgs/([_.a-z0-9]*)/spkg-configure[.]m4 *,\1-uninstall,'; done | sort -u))" >> $GITHUB_OUTPUT
           echo "build_targets=$(echo $(for a in '' ${{ steps.changed-packages.outputs.pkgs_all_changed_files }}; do echo $a | sed -E 's,-,_,g;s,(build/)?pkgs/([-_.a-z0-9]*)/[^ ]* *,\2-ensure,;'; done | sort -u))" >> $GITHUB_OUTPUT
           cat $GITHUB_OUTPUT
-  minimal:
+
+  test:
     needs: [changed_files]
     if: |
       github.event_name != 'pull_request' ||
@@ -94,3 +95,38 @@ jobs:
           ["standard",
            "minimal"]
       docker_push_repository: ghcr.io/${{ github.repository }}/
+
+  site:
+    needs: [changed_files]
+    if: |
+      github.event_name != 'pull_request' ||
+      ((github.event.action != 'labeled' &&
+         (contains(github.event.pull_request.labels.*.name, 'c: packages: standard') ||
+          contains(github.event.pull_request.labels.*.name, 'c: packages: optional'))) ||
+       (github.event.action == 'labeled' &&
+         (github.event.label.name == 'c: packages: optional' ||
+          github.event.label.name == 'c: packages: standard')))
+    uses: ./.github/workflows/docker.yml
+    with:
+      # Build incrementally from published Docker image
+      incremental: true
+      free_disk_space: true
+      from_docker_repository: ghcr.io/sagemath/sage/
+      from_docker_target: "with-targets"
+      from_docker_tag: "dev"
+      docker_targets: "with-targets"
+      targets: "${{needs.changed_files.outputs.uninstall_targets}} ${{needs.changed_files.outputs.build_targets}} build doc-html ptest"
+      # Only test systems with a usable system python (>= 3.9)
+      tox_system_factors: >-
+        ["ubuntu-jammy",
+         "ubuntu-mantic",
+         "debian-bullseye",
+         "debian-bookworm",
+         "fedora-33",
+         "fedora-38",
+         "gentoo-python3.11",
+         "archlinux",
+         "debian-bullseye-i386"]
+      tox_packages_factors: >-
+          ["standard-sitepackages"]
+      docker_push_repository: ghcr.io/${{ github.repository }}/

.github/workflows/ci-linux.yml

@@ -82,7 +82,6 @@ jobs:
       # Only test systems with a usable system python (>= 3.9)
       tox_system_factors: >-
         ["ubuntu-jammy",
-         "ubuntu-kinetic",
          "ubuntu-lunar",
          "ubuntu-mantic",
          "debian-bullseye",

.github/workflows/ci-macos.yml

@@ -30,89 +30,40 @@ env:
   TARGETS_OPTIONAL: ptest
 
 jobs:
-  local-macos:
+  stage-1:
+    uses: ./.github/workflows/macos.yml
+    with:
+      stage: "1"
 
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        stage: ["1", "2", "2-optional-0-o", "2-optional-p-z", "2-experimental-0-o", "2-experimental-p-z"]
-        # python3_xcode is only accepted if enough packages are available from the system
-        # --> to test "minimal", we would need https://github.com/sagemath/sage/issues/30949
-        tox_env: [homebrew-macos-usrlocal-minimal, homebrew-macos-usrlocal-standard, homebrew-macos-usrlocal-maximal, homebrew-macos-usrlocal-python3_xcode-standard, conda-forge-macos-minimal, conda-forge-macos-standard, conda-forge-macos-maximal]
-        xcode_version_factor: [default]
-        os: [ macos-11, macos-12 ]
-    env:
-      TOX_ENV: local-${{ matrix.tox_env }}
-      LOCAL_ARTIFACT_NAME: sage-local-commit-${{ github.sha }}-tox-local-${{ matrix.tox_env }}-${{ matrix.os }}-xcode_${{ matrix.xcode_version_factor }}
-      LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-local-${{ matrix.tox_env }}--${{ matrix.os }}-xcode_${{ matrix.xcode_version_factor }}
-    steps:
-      - uses: actions/checkout@v4
-      - name: Select Xcode version
-        run: |
-          if [ ${{ matrix.xcode_version_factor }} != default ]; then sudo xcode-select -s /Applications/Xcode_${{ matrix.xcode_version_factor }}.app; fi
-      - name: Install test prerequisites
-        run: |
-          brew install tox
-      - uses: actions/download-artifact@v3
-        with:
-          path: sage-local-artifact
-          name: ${{ env.LOCAL_ARTIFACT_NAME }}
-        if: contains(matrix.stage, '2')
-      - name: Extract sage-local artifact
-        # This is macOS tar -- cannot use --listed-incremental
-        run: |
-          export SAGE_LOCAL=$(pwd)/.tox/$TOX_ENV/local
-          .github/workflows/extract-sage-local.sh sage-local-artifact/sage-local-*.tar
-        if: contains(matrix.stage, '2')
-      - name: Build and test with tox
-        # We use a high parallelization on purpose in order to catch possible parallelization bugs in the build scripts.
-        # For doctesting, we use a lower parallelization to avoid timeouts.
-        run: |
-          case "${{ matrix.stage }}" in
-            1)               export TARGETS_PRE="all-sage-local" TARGETS="all-sage-local" TARGETS_OPTIONAL="build/make/Makefile"
-                             ;;
-            2)               export TARGETS_PRE="all-sage-local" TARGETS="build doc-html" TARGETS_OPTIONAL="ptest"
-                             ;;
-            2-optional*)     export TARGETS_PRE="build/make/Makefile" TARGETS="build/make/Makefile"
-                             targets_pattern="${{ matrix.stage }}"
-                             targets_pattern="${targets_pattern#2-optional-}"
-                             export TARGETS_OPTIONAL=$( echo $(export PATH=build/bin:$PATH && sage-package list :optional: --has-file 'spkg-install.in|spkg-install|requirements.txt' --no-file huge|has_nonfree_dependencies |  grep -v sagemath_doc | grep "^[$targets_pattern]" ) )
-                             ;;
-            2-experimental*) export TARGETS_PRE="build/make/Makefile" TARGETS="build/make/Makefile"
-                             targets_pattern="${{ matrix.stage }}"
-                             targets_pattern="${targets_pattern#2-experimental-}"
-                             export TARGETS_OPTIONAL=$( echo $(export PATH=build/bin:$PATH && sage-package list :experimental: --has-file 'spkg-install.in|spkg-install|requirements.txt' --no-file huge|has_nonfree_dependencies |  grep -v sagemath_doc | grep "^[$targets_pattern]" ) )
-                             ;;
-          esac
-          MAKE="make -j12" tox -e $TOX_ENV -- SAGE_NUM_THREADS=4 $TARGETS
-      - name: Prepare logs artifact
-        run: |
-          mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"; cp -r .tox/*/log "artifacts/$LOGS_ARTIFACT_NAME"
-        if: always()
-      - uses: actions/upload-artifact@v3
-        with:
-          path: artifacts
-          name: ${{ env.LOGS_ARTIFACT_NAME }}
-        if: always()
-      - name: Print out logs for immediate inspection
-        # and markup the output with GitHub Actions logging commands
-        run: |
-          .github/workflows/scan-logs.sh "artifacts/$LOGS_ARTIFACT_NAME"
-        if: always()
-      - name: Prepare sage-local artifact
-        # This also includes the copies of homebrew or conda installed in the tox environment.
-        # We use absolute pathnames in the tar file.
-        # This is macOS tar -- cannot use --remove-files.
-        # We remove the $SAGE_LOCAL/lib64 link, which will be recreated by the next stage.
-        run: |
-          mkdir -p sage-local-artifact && (cd .tox/$TOX_ENV && rm -f "local/lib64" && tar -cf - $(pwd)) > sage-local-artifact/sage-${{ env.TOX_ENV }}-${{ matrix.stage }}.tar
-        if: contains(matrix.stage, '1')
-      - uses: actions/upload-artifact@v3
-        with:
-          path: sage-local-artifact/sage-${{ env.TOX_ENV }}-${{ matrix.stage }}.tar
-          name: ${{ env.LOCAL_ARTIFACT_NAME }}
-        if: always()
+  stage-2:
+    uses: ./.github/workflows/macos.yml
+    with:
+      stage: "2"
+    needs: [stage-1]
+
+  stage-2-optional-0-o:
+    uses: ./.github/workflows/macos.yml
+    with:
+      stage: "2-optional-0-o"
+    needs: [stage-2]
+
+  stage-2-optional-p-z:
+    uses: ./.github/workflows/macos.yml
+    with:
+      stage: "2-optional-p-z"
+    needs: [stage-2-optional-0-o]
+
+  stage-2-experimental-0-o:
+    uses: ./.github/workflows/macos.yml
+    with:
+      stage: "2-optional-0-o"
+    needs: [stage-2-optional-p-z]
+
+  stage-2-experimental-p-z:
+    uses: ./.github/workflows/macos.yml
+    with:
+      stage: "2-experimental-p-z"
+    needs: [stage-2-experimental-0-o]
 
   dist:
 

.github/workflows/doc-build-pdf.yml

@@ -21,6 +21,12 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
+      - name: Merge CI fixes from sagemath/sage
+        run: |
+          .ci/merge-fixes.sh
+        env:
+          GH_TOKEN: ${{ github.token }}
+
       - name: Prepare
         run: |
           apt-get update && apt-get install -y zip