Merge branch 'develop' into merge-group

.ci/merge-fixes.sh

@@ -0,0 +1,36 @@
+#!/bin/sh
+# Merge open PRs from sagemath/sage labeled "blocker".
+REPO="sagemath/sage"
+GH="gh -R $REPO"
+PRs="$($GH pr list --label "p: blocker / 1" --json number --jq '.[].number')"
+if [ -z "$PRs" ]; then
+    echo 'Nothing to do: Found no open PRs with "blocker" status.'
+else
+    echo "Found PRs: " $PRs
+    export GIT_AUTHOR_NAME="ci-sage workflow"
+    export GIT_AUTHOR_EMAIL="[email protected]"
+    export GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME"
+    export GIT_COMMITTER_EMAIL="$GIT_AUTHOR_EMAIL"
+    git tag -f test_base
+    git commit -q -m "Uncommitted changes" --no-allow-empty -a
+    for a in $PRs; do
+        echo "::group::Merging PR https://github.com/$REPO/pull/$a"
+        git tag -f test_head
+        $GH pr checkout -b pr-$a $a
+        git fetch --unshallow --all
+        git checkout -q test_head
+        if git merge --no-edit --squash -q pr-$a; then
+            echo "::endgroup::"
+            if git commit -q -m "Merge https://github.com/$REPO/pull/$a" -a --no-allow-empty; then
+                echo "Merged #$a"
+            else
+                echo "Empty, skipped"
+            fi
+        else
+            echo "::endgroup::"
+            echo "Failure merging #$a, resetting"
+            git reset --hard
+        fi
+    done
+    git log test_base..HEAD
+fi

.github/sync_labels.py

@@ -56,7 +56,7 @@ class ReviewDecision(Enum):
     """
     changes_requested = 'CHANGES_REQUESTED'
     approved = 'APPROVED'
-    unclear = 'COMMENTED'
+    unclear = 'UNCLEAR'
 
 class Priority(Enum):
     r"""
@@ -319,12 +319,15 @@ def get_review_decision(self):
             return None
 
         if self._review_decision is not None:
+            if self._review_decision == ReviewDecision.unclear:
+                return None
             return self._review_decision
 
         data = self.view('reviewDecision')
         if data:
             self._review_decision = ReviewDecision(data)
         else:
+            # To separate a not supplied value from not cached (see https://github.com/sagemath/sage/pull/36177#issuecomment-1704022893 ff)
             self._review_decision = ReviewDecision.unclear
         info('Review decision for %s: %s' % (self._issue, self._review_decision.value))
         return self._review_decision
@@ -349,9 +352,9 @@ def get_reviews(self, complete=False):
             self.get_commits()
 
         date = self._commit_date
-        no_rev = ReviewDecision.unclear.value
+        unproper_rev = RevState.commented.value
         new_revs = [rev for rev in self._reviews if rev['submittedAt'] > date]
-        proper_new_revs = [rev for rev in new_revs if rev['state'] != no_rev]
+        proper_new_revs = [rev for rev in new_revs if rev['state'] != unproper_rev]
         info('Proper reviews after %s for %s: %s' % (date, self._issue, proper_new_revs))
         return proper_new_revs
 
@@ -463,11 +466,6 @@ def approve_allowed(self):
         r"""
         Return if the actor has permission to approve this PR.
         """
-        revs = self.get_reviews(complete=True)
-        if not any(rev['authorAssociation'] in ('MEMBER', 'OWNER') for rev in revs):
-            info('PR %s can\'t be approved because of missing member review' % (self._issue))
-            return False
-
         revs = self.get_reviews()
         revs = [rev for rev in revs if rev['author']['login'] != self._actor]
         ch_req = ReviewDecision.changes_requested

.github/workflows/build.yml

@@ -27,13 +27,34 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  get_ci_fixes:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        id: checkout
+        uses: actions/checkout@v4
+      - name: Merge CI fixes from sagemath/sage
+        run: |
+          .ci/merge-fixes.sh
+        env:
+          GH_TOKEN: ${{ github.token }}
+      - name: Store CI fixes in upstream artifact
+        run: |
+          mkdir -p upstream
+          git format-patch --stdout test_base > upstream/ci_fixes.patch
+      - uses: actions/upload-artifact@v3
+        with:
+          path: upstream
+          name: upstream
+
   build:
     runs-on: ubuntu-latest
     container: ghcr.io/sagemath/sage/sage-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-with-targets:${{ github.event.inputs.docker_tag || 'dev'}}
+    needs: [get_ci_fixes]
     steps:
       - name: Checkout
         id: checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Update system packages
         id: prepare
@@ -71,6 +92,19 @@ jobs:
           if [ ! -f worktree-image/.gitignore ]; then cp .gitignore worktree-image/; fi
           (cd worktree-image && git add -A && git commit --quiet --allow-empty -m "old" -a && git tag -f old && git reset --hard new && git reset --quiet old && git add -N . && git status)
 
+      - name: Download upstream artifact
+        uses: actions/download-artifact@v3
+        with:
+          path: upstream
+          name: upstream
+
+      - name: Apply CI fixes from sagemath/sage
+        # After applying the fixes, make sure all changes are marked as uncommitted changes.
+        run: |
+          if [ -r upstream/ci_fixes.patch ]; then
+            (cd worktree-image && git commit -q -m "current changes" --allow-empty -a && git am; git reset --quiet old; git add -N .) < upstream/ci_fixes.patch
+          fi
+
       - name: Incremental build, test changed files (sage -t --new)
         id: incremental
         run: |
@@ -79,15 +113,15 @@ jobs:
           ./bootstrap && make build && ./sage -t --new -p2
         working-directory: ./worktree-image
         env:
-          MAKE: make -j2
+          MAKE: make -j2 --output-sync=recurse
           SAGE_NUM_THREADS: 2
 
       - name: Build and test modularized distributions
         if: always() && steps.worktree.outcome == 'success'
         run: make V=0 tox && make pypi-wheels
         working-directory: ./worktree-image
         env:
-          MAKE: make -j2
+          MAKE: make -j2 --output-sync=recurse
           SAGE_NUM_THREADS: 2
 
       - name: Set up node to install pyright
@@ -106,17 +140,29 @@ jobs:
         run: pyright
         working-directory: ./worktree-image
 
-      - name: Build (fallback to non-incremental)
-        id: build
+      - name: Clean (fallback to non-incremental)
+        id: clean
         if: always() && steps.worktree.outcome == 'success' && steps.incremental.outcome != 'success'
         run: |
           set -ex
-          ./bootstrap && make doc-clean doc-uninstall sagelib-clean && git clean -fx src/sage && ./config.status && make build
+          ./bootstrap && make doc-clean doc-uninstall sagelib-clean && git clean -fx src/sage && ./config.status
         working-directory: ./worktree-image
         env:
           MAKE: make -j2
           SAGE_NUM_THREADS: 2
 
+      - name: Build
+        # This step is needed because building the modularized distributions installs some optional packages,
+        # so the editable install of sagelib needs to build the corresponding optional extension modules.
+        id: build
+        if: always() && (steps.incremental.outcome == 'success' || steps.clean.outcome == 'success')
+        run: |
+          make build
+        working-directory: ./worktree-image
+        env:
+          MAKE: make -j2 --output-sync=recurse
+          SAGE_NUM_THREADS: 2
+
       - name: Pytest
         if: contains(github.ref, 'pytest')
         run: |
@@ -128,22 +174,22 @@ jobs:
           COLUMNS: 120
 
       - name: Test all files (sage -t --all --long)
-        if: always() && (steps.incremental.outcome == 'success' || steps.build.outcome == 'success')
+        if: always() && steps.build.outcome == 'success'
         run: |
           ../sage -python -m pip install coverage
           ../sage -python -m coverage run ./bin/sage-runtests --all --long -p2 --random-seed=286735480429121101562228604801325644303
         working-directory: ./worktree-image/src
 
       - name: Prepare coverage results
-        if: always() && (steps.incremental.outcome == 'success' || steps.build.outcome == 'success')
+        if: always() && steps.build.outcome == 'success'
         run: |
           ./venv/bin/python3 -m coverage combine src/.coverage/
           ./venv/bin/python3 -m coverage xml
           find . -name *coverage*
         working-directory: ./worktree-image
 
       - name: Upload coverage to codecov
-        if: always() && (steps.incremental.outcome == 'success' || steps.build.outcome == 'success')
+        if: always() && steps.build.outcome == 'success'
         uses: codecov/codecov-action@v3
         with:
           files: ./worktree-image/coverage.xml

.github/workflows/ci-conda.yml

@@ -8,6 +8,11 @@ on:
       - 'public/build/**-runci'
   pull_request:
     types:
+      # Defaults
+      - opened
+      - synchronize
+      - reopened
+      # When a CI label is added
       - labeled
   workflow_dispatch:
     # Allow to run manually
@@ -21,23 +26,36 @@ jobs:
   test:
     name: Conda
     runs-on: ${{ matrix.os }}
-    
-    # Run on push, workflow dispatch and when certain labels are added 
+
+    # Run on push, workflow dispatch and when certain labels are added or are present
     if: |
-      github.event.action != 'labeled' ||
-      github.event.label.name == 'c: packages: optional' || 
-      github.event.label.name == 'c: packages: standard' || 
-      github.event.label.name == 's: run conda ci'
+      github.event_name != 'pull_request' ||
+      ((github.event.action != 'labeled' &&
+         (contains(github.event.pull_request.labels.*.name, 'c: packages: standard') ||
+          contains(github.event.pull_request.labels.*.name, 'c: packages: optional') ||
+          contains(github.event.pull_request.labels.*.name, 's: run conda ci'))) ||
+       (github.event.action == 'labeled' &&
+         (github.event.label.name == 'c: packages: optional' ||
+          github.event.label.name == 'c: packages: standard' ||
+          github.event.label.name == 's: run conda ci')))
 
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest]
         python: ['3.9', '3.10', '3.11']
-        conda-env: [environment, environment-optional]
+        # Optional environment is disabled for now as its not yet working
+        # environment: [environment, environment-optional]
+        conda-env: [environment]
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+
+      - name: Merge CI fixes from sagemath/sage
+        run: |
+          .ci/merge-fixes.sh
+        env:
+          GH_TOKEN: ${{ github.token }}
 
       - name: Check for Miniconda
         id: check_conda
@@ -87,9 +105,6 @@ jobs:
           echo "::remove-matcher owner=configure-system-package-warning::"
           echo "::remove-matcher owner=configure-system-package-error::"
 
-          # Manually install ptyprocess for now, until https://github.com/sagemath/sage/issues/32147 is fixed
-          pip install --no-build-isolation -v -v ptyprocess==0.5.1
-
       - name: Build
         shell: bash -l {0}
         run: |
@@ -100,19 +115,20 @@ jobs:
           SAGE_NUM_THREADS: 2
 
       - name: Verify dependencies
-        if: always()
+        if: success() || failure()
         shell: bash -l {0}
         run: pip check
 
       - name: Test
+        if: success() || failure()
         shell: bash -l {0}
         run: ./sage -t --all -p0
 
       - name: Print logs
+        if: always()
         run: |
           for file in $(find . -type f -name "*.log"); do
               echo "::group::$file"
               cat "$file"
               echo "::endgroup::"
           done
-        if: always()