Merge tag '9.2.beta8' into #29257

SageMath version 9.2.beta8, Release Date: 2020-08-10

.ci/build-docker.sh

@@ -47,6 +47,9 @@ rsync html_/ html/ -a --copy-links
 
 # Build the release image without build artifacts.
 docker_build --target sagemath --tag "$DOCKER_IMAGE_CLI" .
+# Tag the sagemath:$DOCKER_TAG image that CI has just built as
+# sagemath:$COMMIT_HASH so we can refer to it uniquely later.
+docker tag "$DOCKER_IMAGE_CLI" "$DOCKER_IMAGE_BINDER"
 # Display the layers of this image
 docker history "$DOCKER_IMAGE_CLI"
 # Build the developer image with the build artifacts intact.

.ci/push-dockerhub.sh

@@ -26,4 +26,8 @@ if [ -z "$DOCKER_USER" -o -z "$SECRET_DOCKER_PASS" ]; then
 else
   cat "$SECRET_DOCKER_PASS" | docker login -u $DOCKER_USER --password-stdin
   docker push ${DOCKER_NAMESPACE:-sagemath}/$1:$DOCKER_TAG
+
+  # For historical reasons, we also provide a -py3 tag. It's identical to the non-py3 tag.
+  docker tag ${DOCKER_NAMESPACE:-sagemath}/$1:$DOCKER_TAG ${DOCKER_NAMESPACE:-sagemath}/$1:$DOCKER_TAG-py3
+  docker push ${DOCKER_NAMESPACE:-sagemath}/$1:$DOCKER_TAG-py3
 fi

.ci/update-env.sh

@@ -45,6 +45,8 @@ fi
 export DOCKER_IMAGE_CLI=${DOCKER_NAMESPACE:-sagemath}/sagemath:$DOCKER_TAG
 export DOCKER_IMAGE_DEV=${DOCKER_NAMESPACE:-sagemath}/sagemath-dev:$DOCKER_TAG
 
+export DOCKER_IMAGE_BINDER="${DOCKER_NAMESPACE:-sagemath}/sagemath:${CI_COMMIT_SHA}"
+
 # Seed the build cache with this image (set to source-clean to build from
 # scratch.)
 export ARTIFACT_BASE=${ARTIFACT_BASE:-$DEFAULT_ARTIFACT_BASE}

.gitattributes

@@ -0,0 +1,5 @@
+# Force LF normalization
+* text=auto eol=lf
+# except for Windows batch files
+*.{cmd,[cC][mM][dD]} text eol=crlf
+*.{bat,[bB][aA][tT]} text eol=crlf

.github/workflows/extract-sage-local.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# to be run from $SAGE_ROOT, with arguments sage-local-${{ env.PREVIOUS_STAGES }}.tar
+
+# Show all tar files
+ls -l $*
+
+# We specifically use the cygwin tar so that symlinks are saved/restored correctly on Windows.
+for a in $*; do
+    echo Extracting $a
+    tar xf $a
+    rm -f $a
+done
+
+# Also get rid of the stages that were not extracted
+rm -f sage-local-*.tar
+
+# We set the installation records to the same mtime so that no rebuilds due to dependencies
+# among these packages are triggered.
+(cd local/var/lib/sage/installed/ && touch .dummy && touch --reference=.dummy *)
+
+# Show what has been built already.
+ls -l local local/var/lib/sage/installed/
+df -h
+
+# Rebase!
+src/bin/sage-rebase.sh local

.github/workflows/scan-logs.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+# The markup in the output is a GitHub Actions logging command
+# https://help.github.com/en/actions/automating-your-workflow-with-github-actions/development-tools-for-github-actions
+LOGS=${1-logs}
+for a in $(find "$LOGS" -type f -name "*.log"); do
+    if tail -100 "$a" 2>/dev/null | grep "^[A-Za-z]*Error" >/dev/null; then
+        echo :":"error file=$a:":" ==== ERROR IN LOG FILE $a ====
+        cat "$a"
+    elif tail -20 "$a" 2>/dev/null | grep -E "^(Warning: Error testing|^sage .*doctest.*failed)" >/dev/null; then
+        echo :":"warning file=$a:":" ==== TESTSUITE FAILURE IN LOG FILE $a ====
+        cat "$a"
+    fi
+done

.github/workflows/tox-experimental.yml

@@ -0,0 +1,144 @@
+name: Test experimental packages with tox
+
+## This GitHub Actions workflow runs SAGE_ROOT/tox.ini with select environments,
+## whenever a GitHub pull request is opened or synchronized in a repository
+## where GitHub Actions are enabled.
+##
+## It builds and checks some sage spkgs as defined in TARGETS.
+##
+## A job succeeds if there is no error.
+##
+## The build is run with "make V=0", so the build logs of individual packages are suppressed.
+##
+## At the end, all package build logs that contain an error are printed out.
+##
+## After all jobs have finished (or are canceled) and a short delay,
+## tar files of all logs are made available as "build artifacts".
+
+#on: [push, pull_request]
+
+on:
+  pull_request:
+    types: [opened, synchronize]
+  push:
+    tags:
+      - '*'
+
+env:
+  TARGETS_PRE: build/make/Makefile
+  TARGETS:     build/make/Makefile
+  # TARGETS_OPTIONAL see below
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      max-parallel: 6
+      matrix:
+        tox_system_factor: [ubuntu-trusty, ubuntu-xenial, ubuntu-bionic, ubuntu-eoan, ubuntu-focal, debian-jessie, debian-stretch, debian-buster, debian-bullseye, debian-sid, linuxmint-17, linuxmint-18, linuxmint-19, linuxmint-19.3, fedora-26, fedora-27, fedora-28, fedora-29, fedora-30, fedora-31, centos-7, centos-8, gentoo, archlinux-latest, slackware-14.2, conda-forge, ubuntu-bionic-i386, ubuntu-eoan-i386, debian-buster-i386, centos-7-i386]
+        tox_packages_factor: [maximal]
+        targets_pattern: [0-g, h-o, p, q-z]
+    env:
+      TOX_ENV: docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
+      LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
+      DOCKER_TARGETS: configured with-targets with-targets-optional
+      TARGETS_OPTIONAL: "$( echo $(PATH=build/bin:$PATH build/bin/sage-package list :experimental: | grep -v  database_stein_watkins\\$ | grep -v polytopes_db_4d | grep '^[${{ matrix.targets_pattern }}]' ) )"
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 500
+      - name: fetch tags
+        run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
+      - name: free disk space
+        run: |
+          sudo swapoff -a
+          sudo rm -f /swapfile
+          sudo apt clean
+          docker rmi $(docker image ls -aq)
+          df -h
+      - name: Install test prerequisites
+        run: |
+          sudo DEBIAN_FRONTEND=noninteractive apt-get update
+          sudo DEBIAN_FRONTEND=noninteractive apt-get install python-tox
+      - run: |
+          set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;"
+      - name: Copy logs from the docker image or build container
+        run: |
+          mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"
+          cp -r .tox/$TOX_ENV/Dockerfile .tox/$TOX_ENV/log "artifacts/$LOGS_ARTIFACT_NAME"
+          if [ -f .tox/$TOX_ENV/Dockertags ]; then CONTAINERS=$(docker create $(tail -1 .tox/$TOX_ENV/Dockertags) /bin/bash || true); fi
+          if [ -n "$CONTAINERS" ]; then for CONTAINER in $CONTAINERS; do for ARTIFACT in /sage/logs; do docker cp $CONTAINER:$ARTIFACT artifacts/$LOGS_ARTIFACT_NAME && HAVE_LOG=1; done; if [ -n "$HAVE_LOG" ]; then break; fi; done; fi
+        if: always()
+      - uses: actions/upload-artifact@v1
+        with:
+          path: artifacts
+          name: ${{ env.LOGS_ARTIFACT_NAME }}
+        if: always()
+      - name: Print out logs for immediate inspection
+        # and markup the output with GitHub Actions logging commands
+        run: |
+          .github/workflows/scan-logs.sh "artifacts/$LOGS_ARTIFACT_NAME"
+        if: always()
+      - name: Push docker images
+        run: |
+          if [ -f .tox/$TOX_ENV/Dockertags ]; then
+            TOKEN="${{ secrets.DOCKER_PKG_GITHUB_TOKEN }}"
+            if [ -z "$TOKEN" ]; then
+              TOKEN="${{ secrets.GITHUB_TOKEN }}"
+            fi
+            echo "$TOKEN" | docker login docker.pkg.github.com -u ${{ github.actor }} --password-stdin
+            for a in $(cat .tox/$TOX_ENV/Dockertags); do
+              FULL_TAG=docker.pkg.github.com/${{ github.repository }}/$a
+              docker tag $a $FULL_TAG
+              echo Pushing $FULL_TAG
+              docker push $FULL_TAG
+            done || echo "(Ignoring errors)"
+          fi
+        if: always()
+
+  local-macos:
+
+    runs-on: macos-latest
+    strategy:
+      fail-fast: false
+      max-parallel: 3
+      matrix:
+        tox_system_factor: [homebrew-macos, homebrew-macos-python3_xcode, homebrew-macos-python3_xcode-nokegonly, homebrew-macos-python3_pythonorg, conda-forge-macos]
+        tox_packages_factor: [maximal]
+        targets_pattern: [0-g, h-o, p, q-z]
+    env:
+      TOX_ENV: local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
+      LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
+      TARGETS_OPTIONAL: "$( echo $(PATH=build/bin:$PATH build/bin/sage-package list :experimental: | grep -v  database_stein_watkins\\$ | grep -v polytopes_db_4d | grep '^[${{ matrix.targets_pattern }}]' ) )"
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install test prerequisites
+        run: |
+          brew install tox
+      - name: Install python3 from python.org
+        # As of 2020-03-30 (https://github.com/actions/virtual-environments/blob/master/images/macos/macos-10.15-Readme.md),
+        # Python 3.7.7 is installed on GitHub Actions runners.  But we install our own copy from the python.org binary package.
+        run: |
+          curl -o python3.pkg https://www.python.org/ftp/python/3.7.7/python-3.7.7-macosx10.9.pkg
+          sudo installer -verbose -pkg python3.pkg -target /
+        if: contains(matrix.tox_system_factor, 'python3_pythonorg')
+      - name: Build and test with tox
+        # We use a high parallelization on purpose in order to catch possible parallelization bugs in the build scripts.
+        # For doctesting, we use a lower parallelization to avoid timeouts.
+        run: |
+          MAKE="make -j12" tox -e $TOX_ENV -- SAGE_NUM_THREADS=4 $TARGETS
+      - name: Prepare logs artifact
+        run: |
+          mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"; cp -r .tox/*/log "artifacts/$LOGS_ARTIFACT_NAME"
+        if: always()
+      - uses: actions/upload-artifact@v1
+        with:
+          path: artifacts
+          name: ${{ env.LOGS_ARTIFACT_NAME }}
+        if: always()
+      - name: Print out logs for immediate inspection
+        # and markup the output with GitHub Actions logging commands
+        run: |
+          .github/workflows/scan-logs.sh "artifacts/$LOGS_ARTIFACT_NAME"
+        if: always()

.github/workflows/tox-gcc_spkg.yml

@@ -0,0 +1,134 @@
+name: Test Linux --without-system-gcc
+
+## This GitHub Actions workflow runs SAGE_ROOT/tox.ini with select environments,
+## whenever a GitHub pull request is opened or synchronized in a repository
+## where GitHub Actions are enabled.
+##
+## It builds and checks some sage spkgs as defined in TARGETS.
+##
+## A job succeeds if there is no error.
+##
+## The build is run with "make V=0", so the build logs of individual packages are suppressed.
+##
+## At the end, all package build logs that contain an error are printed out.
+##
+## After all jobs have finished (or are canceled) and a short delay,
+## tar files of all logs are made available as "build artifacts".
+
+#on: [push, pull_request]
+
+on:
+  pull_request:
+    types: [opened, synchronize]
+  push:
+    tags:
+      - '*'
+
+env:
+  TARGETS_PRE: sagelib-build-deps
+  TARGETS: build doc-html
+  TARGETS_OPTIONAL: ptest
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      max-parallel: 5
+      matrix:
+        tox_system_factor: [ubuntu-trusty, ubuntu-focal, fedora-27, fedora-31, debian-buster-i386]
+        tox_packages_factor: [minimal-gcc_spkg, standard-gcc_spkg]
+    env:
+      TOX_ENV: docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
+      LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
+      DOCKER_TARGETS: configured with-targets with-targets-optional
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 500
+      - name: fetch tags
+        run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
+      - name: free disk space
+        run: |
+          sudo swapoff -a
+          sudo rm -f /swapfile
+          sudo apt clean
+          docker rmi $(docker image ls -aq)
+          df -h
+      - name: Install test prerequisites
+        run: |
+          sudo DEBIAN_FRONTEND=noninteractive apt-get update
+          sudo DEBIAN_FRONTEND=noninteractive apt-get install python-tox
+      - run: |
+          set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;"
+      - name: Copy logs from the docker image or build container
+        run: |
+          mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"
+          cp -r .tox/$TOX_ENV/Dockerfile .tox/$TOX_ENV/log "artifacts/$LOGS_ARTIFACT_NAME"
+          if [ -f .tox/$TOX_ENV/Dockertags ]; then CONTAINERS=$(docker create $(tail -1 .tox/$TOX_ENV/Dockertags) /bin/bash || true); fi
+          if [ -n "$CONTAINERS" ]; then for CONTAINER in $CONTAINERS; do for ARTIFACT in /sage/logs; do docker cp $CONTAINER:$ARTIFACT artifacts/$LOGS_ARTIFACT_NAME && HAVE_LOG=1; done; if [ -n "$HAVE_LOG" ]; then break; fi; done; fi
+        if: always()
+      - uses: actions/upload-artifact@v1
+        with:
+          path: artifacts
+          name: ${{ env.LOGS_ARTIFACT_NAME }}
+        if: always()
+      - name: Print out logs for immediate inspection
+        # and markup the output with GitHub Actions logging commands
+        run: |
+          .github/workflows/scan-logs.sh "artifacts/$LOGS_ARTIFACT_NAME"
+        if: always()
+      - name: Push docker images
+        run: |
+          if [ -f .tox/$TOX_ENV/Dockertags ]; then
+            TOKEN="${{ secrets.DOCKER_PKG_GITHUB_TOKEN }}"
+            if [ -z "$TOKEN" ]; then
+              TOKEN="${{ secrets.GITHUB_TOKEN }}"
+            fi
+            echo "$TOKEN" | docker login docker.pkg.github.com -u ${{ github.actor }} --password-stdin
+            for a in $(cat .tox/$TOX_ENV/Dockertags); do
+              FULL_TAG=docker.pkg.github.com/${{ github.repository }}/$a
+              docker tag $a $FULL_TAG
+              echo Pushing $FULL_TAG
+              docker push $FULL_TAG
+            done || echo "(Ignoring errors)"
+          fi
+        if: always()
+
+  local-ubuntu:
+
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      max-parallel: 1
+      matrix:
+        tox_system_factor: [conda-forge-ubuntu]
+        tox_packages_factor: [minimal-gcc_spkg, standard-gcc_spkg]
+    env:
+      TOX_ENV: local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
+      LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install test prerequisites
+        run: |
+          sudo DEBIAN_FRONTEND=noninteractive apt-get update
+          sudo DEBIAN_FRONTEND=noninteractive apt-get install python-tox
+      - name: Build and test with tox
+        # We use a high parallelization on purpose in order to catch possible parallelization bugs in the build scripts.
+        # For doctesting, we use a lower parallelization to avoid timeouts.
+        run: |
+          MAKE="make -j12" tox -e $TOX_ENV -- SAGE_NUM_THREADS=4 $TARGETS
+      - name: Prepare logs artifact
+        run: |
+          mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"; cp -r .tox/*/log "artifacts/$LOGS_ARTIFACT_NAME"
+        if: always()
+      - uses: actions/upload-artifact@v1
+        with:
+          path: artifacts
+          name: ${{ env.LOGS_ARTIFACT_NAME }}
+        if: always()
+      - name: Print out logs for immediate inspection
+        # and markup the output with GitHub Actions logging commands
+        run: |
+          .github/workflows/scan-logs.sh "artifacts/$LOGS_ARTIFACT_NAME"
+        if: always()