Merge tag '9.7.beta7' into t/32601/modularization_of_sagelib__break_o…

…ut_a_separate_package_sagemath_standard_no_symbolics

SageMath version 9.7.beta7, Release Date: 2022-08-01

.github/workflows/build.yml

@@ -25,18 +25,11 @@ jobs:
     container: ghcr.io/sagemath/sage/sage-docker-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-with-targets:${{ github.event.inputs.docker_tag || 'dev'}}
     steps:
       - name: Checkout
+        id: checkout
         uses: actions/checkout@v2
 
-      - name: Set up node to install pyright
-        uses: actions/setup-node@v1
-        with:
-          node-version: '12'
-
-      - name: Install pyright
-        # Fix to v232 due to bug https://github.com/microsoft/pyright/issues/3239
-        run: npm install -g [email protected]
-
       - name: Prepare
+        id: prepare
         run: |
           # Install test tools.
           if apt-get update && apt-get install -y git python3-venv; then
@@ -53,10 +46,31 @@ jobs:
           ./bootstrap
           ./configure --enable-build-as-root --prefix=/sage/local --with-sage-venv --enable-editable --enable-download-from-upstream-url
       
+      - name: Build and test modularized distributions
+        if: always() && steps.prepare.outcome == 'success'
+        run: make V=0 tox && make pypi-wheels
+        env:
+          MAKE: make -j2
+          SAGE_NUM_THREADS: 2
+
+      - name: Set up node to install pyright
+        if: always() && steps.prepare.outcome == 'success'
+        uses: actions/setup-node@v1
+        with:
+          node-version: '12'
+
+      - name: Install pyright
+        if: always() && steps.prepare.outcome == 'success'
+        # Fix to v232 due to bug https://github.com/microsoft/pyright/issues/3239
+        run: npm install -g [email protected]
+
       - name: Static code check with pyright
+        if: always() && steps.prepare.outcome == 'success'
         run: pyright    
 
       - name: Build
+        id: build
+        if: always() && steps.prepare.outcome == 'success'
         run: make build
         env:
           MAKE: make -j2
@@ -73,20 +87,21 @@ jobs:
           COLUMNS: 120
 
       - name: Test
+        if: always() && steps.build.outcome == 'success'
         run: |
           ../sage -python -m pip install coverage
           ../sage -python -m coverage run ./bin/sage-runtests --all -p2
         working-directory: ./src
 
       - name: Prepare coverage results
-        if: always()
+        if: always() && steps.build.outcome == 'success'
         run: |
           ./venv/bin/python3 -m coverage combine src/.coverage/
           ./venv/bin/python3 -m coverage xml
           find . -name *coverage*
       
       - name: Upload coverage to codecov
-        if: always()
+        if: always() && steps.build.outcome == 'success'
         uses: codecov/codecov-action@v2
         with:
           files: ./coverage.xml

.github/workflows/ci-linux.yml

@@ -0,0 +1,81 @@
+name: CI Linux
+
+## This GitHub Actions workflow runs SAGE_ROOT/tox.ini with select environments,
+## whenever a GitHub pull request is opened or synchronized in a repository
+## where GitHub Actions are enabled.
+##
+## It builds and checks some sage spkgs as defined in TARGETS.
+##
+## A job succeeds if there is no error.
+##
+## The build is run with "make V=0", so the build logs of individual packages are suppressed.
+##
+## At the end, all package build logs that contain an error are printed out.
+##
+## After all jobs have finished (or are canceled) and a short delay,
+## tar files of all logs are made available as "build artifacts".
+
+#on: [push, pull_request]
+
+on:
+  pull_request:
+    types: [opened, synchronize]
+  push:
+    tags:
+      - '*'
+  workflow_dispatch:
+    # Allow to run manually
+
+env:
+  TARGETS_PRE: all-sage-local
+  TARGETS: build doc-html
+  TARGETS_OPTIONAL: ptest
+
+jobs:
+
+  docker:
+    uses: ./.github/workflows/docker.yml
+    with:
+      # FIXME: duplicated from env.TARGETS
+      targets_pre: all-sage-local
+      targets: build doc-html
+      targets_optional: ptest
+      docker_push_repository: ghcr.io/${{ github.repository }}/
+
+  local-ubuntu:
+
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      max-parallel: 1
+      matrix:
+        tox_system_factor: [conda-forge-ubuntu]
+        tox_packages_factor: [minimal, standard, environment, environment-optional]
+    env:
+      TOX_ENV: local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
+      LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install test prerequisites
+        run: |
+          sudo DEBIAN_FRONTEND=noninteractive apt-get update
+          sudo DEBIAN_FRONTEND=noninteractive apt-get install tox
+      - name: Build and test with tox
+        # We use a high parallelization on purpose in order to catch possible parallelization bugs in the build scripts.
+        # For doctesting, we use a lower parallelization to avoid timeouts.
+        run: |
+          MAKE="make -j12" tox -e $TOX_ENV -- SAGE_NUM_THREADS=4 $TARGETS
+      - name: Prepare logs artifact
+        run: |
+          mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"; cp -r .tox/*/log "artifacts/$LOGS_ARTIFACT_NAME"
+        if: always()
+      - uses: actions/upload-artifact@v1
+        with:
+          path: artifacts
+          name: ${{ env.LOGS_ARTIFACT_NAME }}
+        if: always()
+      - name: Print out logs for immediate inspection
+        # and markup the output with GitHub Actions logging commands
+        run: |
+          .github/workflows/scan-logs.sh "artifacts/$LOGS_ARTIFACT_NAME"
+        if: always()

.github/workflows/docker.yml

@@ -0,0 +1,216 @@
+name: Reusable workflow for Docker-based portability CI
+
+on:
+  workflow_call:
+    inputs:
+      targets_pre:
+        default: build/make/Makefile
+        type: string
+      targets:
+        default: build/make/Makefile
+        type: string
+      targets_optional:
+        default: build/make/Makefile
+        type: string
+      tox_system_factors:
+        description: 'Stringified JSON object listing tox system factors'
+        type: string
+        default: >-
+          ["ubuntu-trusty-toolchain-gcc_9",
+           "ubuntu-xenial-toolchain-gcc_9",
+           "ubuntu-bionic",
+           "ubuntu-focal",
+           "ubuntu-jammy",
+           "ubuntu-kinetic",
+           "debian-stretch",
+           "debian-buster",
+           "debian-bullseye",
+           "debian-bookworm",
+           "debian-sid",
+           "linuxmint-19",
+           "linuxmint-19.3",
+           "linuxmint-20.1",
+           "linuxmint-20.2",
+           "linuxmint-20.3",
+           "linuxmint-21",
+           "fedora-26",
+           "fedora-27",
+           "fedora-28",
+           "fedora-29",
+           "fedora-30",
+           "fedora-31",
+           "fedora-32",
+           "fedora-33",
+           "fedora-34",
+           "fedora-35",
+           "fedora-36",
+           "fedora-37",
+           "centos-7-devtoolset-gcc_11",
+           "centos-stream-8",
+           "centos-stream-9",
+           "gentoo-python3.9",
+           "gentoo-python3.10",
+           "archlinux-latest",
+           "opensuse-15.3",
+           "opensuse-15.4",
+           "opensuse-tumbleweed",
+           "conda-forge",
+           "ubuntu-bionic-i386",
+           "manylinux-2_24-i686",
+           "debian-buster-i386",
+           ]
+      tox_packages_factors:
+        description: 'Stringified JSON object listing tox packages factors'
+        type: string
+        default: >-
+          ["minimal",
+           "standard",
+           ]
+      max_parallel:
+        type: number
+        default: 20
+      #
+      # Publishing to GitHub Packages
+      #
+      docker_push_repository:
+        required: false
+        type: string
+      #
+      # For use in upstream CIs
+      #
+      upstream_artifact:
+        required: false
+        type: string
+      sage_repo:
+        required: false
+        type: string
+      sage_trac_git:
+        required: false
+        type: string
+      sage_trac_ticket:
+        required: false
+        type: string
+      sage_ref:
+        required: false
+        type: string
+
+jobs:
+  linux:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      max-parallel: ${{ inputs.max_parallel }}
+      matrix:
+        tox_system_factor:   ${{ fromJson(inputs.tox_system_factors) }}
+        tox_packages_factor: ${{ fromJson(inputs.tox_packages_factors) }}
+    env:
+      TOX_ENV:            docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
+      LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
+      DOCKER_TARGETS:     with-system-packages configured with-targets-pre with-targets with-targets-optional
+      TARGETS_PRE:        ${{ inputs.targets_pre }}
+      TARGETS:            ${{ inputs.targets }}
+      TARGETS_OPTIONAL:   ${{ inputs.targets_optional }}
+    steps:
+      - name: Check out SageMath
+        uses: actions/checkout@v2
+        with:
+          repository: ${{ inputs.sage_repo }}
+          ref: ${{ inputs.sage_ref }}
+          fetch-depth: 2000
+      - name: fetch tags
+        run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
+      - name: free disk space
+        run: |
+          df -h
+          sudo swapoff -a
+          sudo rm -f /swapfile
+          sudo apt-get clean
+          docker rmi $(docker image ls -aq)
+          echo "Largest packages:"
+          dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 50
+          sudo apt-get --fix-broken --yes remove $(dpkg-query -f '${Package}\n' -W | grep -E '^(ghc-|google-cloud-sdk|google-chrome|firefox|mysql-server|dotnet-sdk|hhvm|mono)') || echo "(error ignored)"
+          df -h
+      - name: Check out git-trac-command
+        uses: actions/checkout@v2
+        with:
+          repository: sagemath/git-trac-command
+          path: git-trac-command
+        if: inputs.sage_trac_git != ''
+      - name: Check out SageMath from trac.sagemath.org
+        shell: bash {0}
+        # Random sleep and retry to limit the load on trac.sagemath.org
+        run: |
+          git config --global user.email "[email protected]"
+          git config --global user.name "ci-sage workflow"
+          if [ ! -d .git ]; then git init; fi; git remote add trac ${{ inputs.sage_trac_git }} && x=1 && while [ $x -le 5 ]; do x=$(( $x + 1 )); sleep $(( $RANDOM % 60 + 1 )); if git-trac-command/git-trac fetch ${{ inputs.sage_trac_ticket }}; then git merge FETCH_HEAD || echo "(ignored)"; exit 0; fi; sleep 40; done; exit 1
+        if: inputs.sage_trac_git != ''
+
+      - name: Download upstream artifact
+        uses: actions/download-artifact@v2
+        with:
+          path: upstream
+          name: ${{ inputs.upstream_artifact }}
+        if: inputs.upstream_artifact
+      - name: Install test prerequisites
+        run: |
+          sudo DEBIAN_FRONTEND=noninteractive apt-get update
+          sudo DEBIAN_FRONTEND=noninteractive apt-get install tox
+          sudo apt-get clean
+          df -h
+      - name: Update Sage packages from upstream artifact
+        run: |
+          (export PATH=$(pwd)/build/bin:$PATH; (cd upstream && bash -x update-pkgs.sh) && sed -i.bak '/upstream/d' .dockerignore && echo "/:toolchain:/i ADD upstream upstream" | sed -i.bak -f - build/bin/write-dockerfile.sh && git diff)
+        if: inputs.upstream_artifact
+
+      - name: Try to login to ghcr.io
+        if: inputs.docker_push_repository != ''
+        # https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable
+        run: |
+            TOKEN="${{ secrets.DOCKER_PKG_GITHUB_TOKEN }}"
+            if [ -z "$TOKEN" ]; then
+              TOKEN="${{ secrets.GITHUB_TOKEN }}"
+            fi
+            if echo "$TOKEN" | docker login ghcr.io -u ${{ github.actor }} --password-stdin; then
+              echo "DOCKER_PUSH_REPOSITORY=${{ inputs.docker_push_repository }}" >> $GITHUB_ENV
+              echo "DOCKER_CONFIG_FILE=$HOME/.docker/config.json" >> $GITHUB_ENV
+            fi
+            # From the docker documentation via .ci/update-env.sh:
+            # "A tag name must be valid ASCII and may
+            # contain lowercase and uppercase letters, digits, underscores, periods and
+            # dashes. A tag name may not start with a period or a dash and may contain a
+            # maximum of 128 characters."
+            EXTRA_DOCKER_TAGS=`echo $GITHUB_REF_NAME | tr -d '[:space:]' | tr -c '[:alnum:]_.-' '-' | sed 's/^[-.]*//' | cut -c1-128`
+            shopt -s extglob
+            case "$GITHUB_REF_NAME" in
+              +([0-9]).+([0-9])?(.+([0-9])) )
+                EXTRA_DOCKER_TAGS="latest dev $EXTRA_DOCKER_TAGS";;
+              +([0-9]).+([0-9])?(.+([0-9]))?(.)@(a|alpha|b|beta|rc)+([0-9]) )
+                EXTRA_DOCKER_TAGS="dev $EXTRA_DOCKER_TAGS";;
+            esac
+            echo "EXTRA_DOCKER_TAGS=$EXTRA_DOCKER_TAGS" >> $GITHUB_ENV
+      - name: Configure and build Sage distribution within a Docker container
+        run: |
+          set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--build-arg NUMPROC=4 --build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;"
+      - name: Copy logs from the Docker image or build container
+        run: |
+          mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"
+          cp -r .tox/$TOX_ENV/Dockerfile .tox/$TOX_ENV/log "artifacts/$LOGS_ARTIFACT_NAME"
+          if [ -f .tox/$TOX_ENV/Dockertags ]; then CONTAINERS=$(docker create $(tail -1 .tox/$TOX_ENV/Dockertags) /bin/bash || true); fi
+          if [ -n "$CONTAINERS" ]; then for CONTAINER in $CONTAINERS; do for ARTIFACT in /sage/logs; do docker cp $CONTAINER:$ARTIFACT artifacts/$LOGS_ARTIFACT_NAME && HAVE_LOG=1; done; if [ -n "$HAVE_LOG" ]; then break; fi; done; fi
+        if: always()
+      - uses: actions/upload-artifact@v2
+        with:
+          path: artifacts
+          name: ${{ env.LOGS_ARTIFACT_NAME }}
+        if: always()
+      - name: Print out logs for immediate inspection
+        # and markup the output with GitHub Actions logging commands
+        run: |
+          .github/workflows/scan-logs.sh "artifacts/$LOGS_ARTIFACT_NAME"
+        if: always()
+      - name: List docker images
+        run: |
+          if [ -f .tox/$TOX_ENV/Dockertags ]; then
+             cat .tox/$TOX_ENV/Dockertags
+          fi
+        if: always()