build.yml

name: Build & Test

on:
  pull_request:
  merge_group:
  push:
    branches:
      - master
      - develop
    # Ignore pushes on tags to prevent two uploads of codecov reports
    tags-ignore: ['**']
  workflow_dispatch:
    # Allow to run manually
    inputs:
      platform:
        description: 'Platform'
        required: true
        default: 'ubuntu-focal-standard'
      docker_tag:
        description: 'Docker tag'
        required: true
        default: 'dev'

concurrency:
  # Cancel previous runs of this workflow for the same branch
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  get_ci_fixes:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        id: checkout
        uses: actions/checkout@v4
      - name: Merge CI fixes from sagemath/sage
        run: |
          mkdir -p upstream
          .ci/merge-fixes.sh 2>&1 | tee upstream/ci_fixes.log
        env:
          GH_TOKEN: ${{ github.token }}
          SAGE_CI_FIXES_FROM_REPOSITORIES: ${{ vars.SAGE_CI_FIXES_FROM_REPOSITORIES }}
      - name: Store CI fixes in upstream artifact
        run: |
          if git format-patch --stdout test_base > ci_fixes.patch; then
            cp ci_fixes.patch upstream/
          fi
      - uses: actions/upload-artifact@v3
        with:
          path: upstream
          name: upstream

  build:
    runs-on: ubuntu-latest
    container: ghcr.io/sagemath/sage/sage-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-with-targets:${{ github.event.inputs.docker_tag || 'dev'}}
    needs: [get_ci_fixes]
    steps:
      - name: Checkout
        id: checkout
        uses: actions/checkout@v4

      - name: Update system packages
        id: prepare
        run: |
          export PATH="build/bin:$PATH"
          eval $(sage-print-system-package-command auto update)
          eval $(sage-print-system-package-command auto --spkg --yes --no-install-recommends install git)

      - name: Add prebuilt tree as a worktree
        id: worktree
        run: |
          set -ex
          git config --global --add safe.directory $(pwd)
          .ci/retrofit-worktree.sh worktree-image /sage

      - name: Download upstream artifact
        uses: actions/download-artifact@v3
        with:
          path: upstream
          name: upstream

      - name: Apply CI fixes from sagemath/sage
        # After applying the fixes, make sure all changes are marked as uncommitted changes.
        run: |
          if [ -r upstream/ci_fixes.patch ]; then
            (cd worktree-image && git commit -q -m "current changes" --allow-empty -a && git am; git reset --quiet old; git add -N .) < upstream/ci_fixes.patch
          fi

      - name: Incremental build
        id: incremental
        run: |
          # Now re-bootstrap and build. The build is incremental because we were careful with the timestamps.
          ./bootstrap && make build
        working-directory: ./worktree-image
        env:
          MAKE: make -j2 --output-sync=recurse
          SAGE_NUM_THREADS: 2

      - name: Build modularized distributions
        if: (success() || failure()) && steps.worktree.outcome == 'success'
        run: make V=0 tox && make SAGE_CHECK=no pypi-wheels
        working-directory: ./worktree-image
        env:
          MAKE: make -j2 --output-sync=recurse
          SAGE_NUM_THREADS: 2

      - name: Static code check with pyright
        if: (success() || failure()) && steps.worktree.outcome == 'success'
        uses: jakebailey/pyright-action@v1
        with:
          version: 1.1.332
          # Many warnings issued by pyright are not yet helpful because there is not yet enough type information.
          no-comments: true
          working-directory: ./worktree-image
        env:
          # To avoid out of memory errors
          NODE_OPTIONS: --max-old-space-size=8192
      
      - name: Static code check with pyright (annotated)
        if: (success() || failure()) && steps.worktree.outcome == 'success'
        uses: jakebailey/pyright-action@v1
        with:
          version: 1.1.332
          # Issue errors
          no-comments: false
          level: error
          working-directory: ./worktree-image
        env:
          # To avoid out of memory errors
          NODE_OPTIONS: --max-old-space-size=8192

      - name: Clean (fallback to non-incremental)
        id: clean
        if: (success() || failure()) && steps.worktree.outcome == 'success' && steps.incremental.outcome != 'success'
        run: |
          set -ex
          ./bootstrap && make doc-clean doc-uninstall sagelib-clean && git clean -fx src/sage && ./config.status
        working-directory: ./worktree-image
        env:
          MAKE: make -j2
          SAGE_NUM_THREADS: 2

      - name: Build
        # This step is needed because building the modularized distributions installs some optional packages,
        # so the editable install of sagelib needs to build the corresponding optional extension modules.
        id: build
        if: (success() || failure()) && (steps.incremental.outcome == 'success' || steps.clean.outcome == 'success')
        run: |
          make build
        working-directory: ./worktree-image
        env:
          MAKE: make -j2 --output-sync=recurse
          SAGE_NUM_THREADS: 2

      # Testing

      - name: Test changed files (sage -t --new)
        if: (success() || failure()) && steps.build.outcome == 'success'
        run: |
          # We run tests with "sage -t --new"; this only tests the uncommitted changes.
          ./sage -t --new -p2
        working-directory: ./worktree-image
        env:
          MAKE: make -j2 --output-sync=recurse
          SAGE_NUM_THREADS: 2

      - name: Test modularized distributions
        if: (success() || failure()) && steps.build.outcome == 'success'
        run: make V=0 tox && make pypi-wheels-check
        working-directory: ./worktree-image
        env:
          MAKE: make -j2 --output-sync=recurse
          SAGE_NUM_THREADS: 2

      - name: Check that all modules can be imported
        run: |
          # The following command checks that all modules can be imported.
          # The output also includes a long list of modules together with the number of tests in each module.
          # This can be ignored.
          ../sage -python -m pip install pytest-xdist
          ../sage -python -m pytest -c tox.ini -qq --doctest --collect-only || true
        working-directory: ./worktree-image/src
        env:
          # Increase the length of the lines in the "short summary"
          COLUMNS: 120

      - name: Pytest
        if: contains(github.ref, 'pytest')
        run: |
          ../sage -python -m pip install coverage pytest-xdist
          ../sage -python -m coverage run -m pytest -c tox.ini --doctest || true
        working-directory: ./worktree-image/src
        env:
          # Increase the length of the lines in the "short summary"
          COLUMNS: 120

      - name: Test all files (sage -t --all --long)
        if: (success() || failure()) && steps.build.outcome == 'success'
        run: |
          ./sage -python -m pip install coverage
          ./sage -python -m coverage run --rcfile=src/tox.ini src/bin/sage-runtests --all --long -p2 --format github --random-seed=286735480429121101562228604801325644303
        working-directory: ./worktree-image

      - name: Prepare coverage results
        if: (success() || failure()) && steps.build.outcome == 'success'
        run: |
          ./sage -python -m coverage combine --rcfile=src/tox.ini
          ./sage -python -m coverage xml --rcfile=src/tox.ini
          mkdir -p coverage-report
          mv coverage.xml coverage-report/
        working-directory: ./worktree-image

      - name: Upload coverage to codecov
        if: (success() || failure()) && steps.build.outcome == 'success'
        uses: codecov/codecov-action@v3
        with:
          directory: ./worktree-image/coverage-report