Skip to content

Commit

Permalink
Merge branch 'develop' into p/introduce-negated-optional-tag
Browse files Browse the repository at this point in the history
  • Loading branch information
kwankyu authored Nov 25, 2024
2 parents 494aa0a + b9e396a commit 03ffc18
Show file tree
Hide file tree
Showing 590 changed files with 14,736 additions and 25,349 deletions.
6 changes: 6 additions & 0 deletions .ci/create-changes-html.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,12 @@ echo '<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highli
echo '<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js"></script>' >> CHANGES.html
echo '<script>hljs.highlightAll();</script>' >> CHANGES.html
cat >> CHANGES.html << EOF
<style>
p.diff a:first-child {
font-weight: bold;
font-size: x-large;
}
</style>
<script>
document.addEventListener('DOMContentLoaded', () => {
// This URL is hardcoded in the file .github/workflows/doc-publish.yml.
Expand Down
7 changes: 5 additions & 2 deletions .devcontainer/onCreate-conda.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,11 @@
set -e

# Create conda environment
conda install mamba -n base -c conda-forge -y
mamba env create --file src/environment-dev-3.11-linux.yml || mamba env update --file src/environment-dev-3.11-linux.yml
conda config --env --add channels conda-forge
conda config --env --set channel_priority strict
conda update -y --all --override-channels -c conda-forge
conda install mamba=1 -n base -y
mamba env create -y --file environment-dev-3.11-linux.yml || mamba env update -y --file environment-dev-3.11-linux.yml
conda init bash

# Build sage
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/ci-conda.yml
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ jobs:
with:
path: ~/conda_pkgs_dir
key:
${{ runner.os }}-conda-${{ hashFiles('src/environment-3.11.yml') }}
${{ runner.os }}-conda-${{ hashFiles('environment-3.11.yml') }}

- name: Setup Conda environment
uses: conda-incubator/setup-miniconda@v3
Expand All @@ -70,7 +70,7 @@ jobs:
channels: conda-forge
channel-priority: true
activate-environment: sage
environment-file: src/${{ matrix.conda-env }}-${{ matrix.python }}-${{ startsWith(matrix.os, 'macos') && (startsWith(runner.arch, 'ARM') && 'macos' || 'macos-x86_64') || 'linux' }}.yml
environment-file: ${{ matrix.conda-env }}-${{ matrix.python }}-${{ startsWith(matrix.os, 'macos') && (startsWith(runner.arch, 'ARM') && 'macos' || 'macos-x86_64') || 'linux' }}.yml

- name: Print Conda environment
shell: bash -l {0}
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/ci-macos.yml
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ jobs:
- name: make dist
run: |
./configure --enable-download-from-upstream-url && make dist
- uses: actions/upload-artifact@v3
- uses: actions/upload-artifact@v4
with:
path: "dist/*.tar.gz"
name: dist
Expand All @@ -119,7 +119,7 @@ jobs:
steps:
- uses: actions/checkout@v4
if: "!contains(matrix.tox_system_factor, 'nobootstrap')"
- uses: actions/download-artifact@v3
- uses: actions/download-artifact@v4
with:
path: .
name: dist
Expand Down Expand Up @@ -147,7 +147,7 @@ jobs:
run: |
mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"; cp -r .tox/*/log "artifacts/$LOGS_ARTIFACT_NAME"
if: always()
- uses: actions/upload-artifact@v3
- uses: actions/upload-artifact@v4
with:
path: artifacts
name: ${{ env.LOGS_ARTIFACT_NAME }}
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/ci-meson.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ jobs:
with:
path: ~/conda_pkgs_dir
key:
${{ runner.os }}-conda-${{ hashFiles('src/environment-3.11-linux.yml') }}
${{ runner.os }}-conda-${{ hashFiles('environment-3.11-linux.yml') }}

- name: Compiler cache
uses: hendrikmuhs/[email protected]
Expand All @@ -55,7 +55,7 @@ jobs:
channels: conda-forge
channel-priority: true
activate-environment: sage
environment-file: src/environment-${{ matrix.python }}-${{ startsWith(matrix.os, 'macos') && (startsWith(runner.arch, 'ARM') && 'macos' || 'macos-x86_64') || 'linux' }}.yml
environment-file: environment-${{ matrix.python }}-${{ startsWith(matrix.os, 'macos') && (startsWith(runner.arch, 'ARM') && 'macos' || 'macos-x86_64') || 'linux' }}.yml

- name: Print Conda environment
shell: bash -l {0}
Expand Down
45 changes: 34 additions & 11 deletions .github/workflows/conda-lock-update.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,30 +4,53 @@
import subprocess

script_dir = Path(__file__).resolve().parent
root_dir = script_dir / '..' / '..'
root_dir = script_dir / ".." / ".."

subprocess.run([str(root_dir / "bootstrap-conda")])

platforms = {
"linux-64": "linux",
"linux-aarch64": "linux-aarch64",
"osx-64": "macos-x86_64",
"osx-arm64": "macos"
#"win-64": "win",
"osx-arm64": "macos",
# "win-64": "win",
}
pythons = ["3.9", "3.10", "3.11"]
tags = ["", "-dev"]
sources = ["", "src"]

for platform_key, platform_value in platforms.items():
for python in pythons:
for tag in tags:
for src in sources:
env_file = root_dir / src / f"environment{tag}-{python}.yml"
lock_file = root_dir / src / f"environment{tag}-{python}-{platform_value}"
env_file = root_dir / f"environment{tag}-{python}.yml"
lock_file = root_dir / f"environment{tag}-{python}-{platform_value}"
lock_file_gen = root_dir / f"environment{tag}-{python}-{platform_value}.yml"

if not env_file.exists():
continue
if not env_file.exists():
continue

print(f"Updating lock file for {env_file} at {lock_file}", flush=True)
subprocess.run(["conda-lock", "--channel", "conda-forge", "--kind", "env", "--platform", platform_key, "--file", str(env_file), "--lockfile", str(lock_file), "--filename-template", str(lock_file)])
print(f"Updating lock file for {env_file} at {lock_file_gen}", flush=True)
subprocess.run(
[
"conda-lock",
"--mamba",
"--channel",
"conda-forge",
"--kind",
"env",
"--platform",
platform_key,
"--file",
str(env_file),
"--lockfile",
str(lock_file),
"--filename-template",
str(lock_file),
],
check=True,
)

# Add conda env name to lock file at beginning
with open(lock_file_gen, "r+") as f:
content = f.read()
f.seek(0, 0)
f.write(f"name: sage{tag}\n{content}")
86 changes: 60 additions & 26 deletions .github/workflows/doc-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -114,59 +114,77 @@ jobs:
- name: Start container
id: container
# Try to continue when "exporting to GitHub Actions Cache" failed with timeout
if: (success() || failure())
run: |
docker run --name BUILD -dit \
--mount type=bind,src=$(pwd),dst=$(pwd) \
--workdir $(pwd) \
${{ env.BUILD_IMAGE }} /bin/sh
#
# On PRs and pushes to develop
# On pull request and push to develop events
#

- name: Get workflow run-id
id: get_run_id
if: steps.container.outcome == 'success' && !startsWith(github.ref, 'refs/tags/') && github.event_name == 'pull_request'
run: |
RESPONSE=$(curl -s -L \
-H "Accept: application/vnd.github+json" \
-H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
"https://api.github.com/repos/${{ github.repository }}/actions/runs?event=push&branch=develop&status=completed")
RUN_ID=$(echo "$RESPONSE" | jq -r --arg name "${{ github.workflow }}" --arg conclusion "success" \
'.workflow_runs[] | select(.name == $name and .conclusion == $conclusion) | .id' | head -n 1)
echo "RUN_ID=$RUN_ID" >> $GITHUB_ENV
- name: Download old doc
id: download-doc
if: steps.get_run_id.outcome == 'success'
uses: actions/download-artifact@v4
with:
name: doc-develop
github-token: ${{ secrets.GITHUB_TOKEN }}
repository: ${{ github.repository }}
run-id: ${{ env.RUN_ID }}

- name: Store old doc
id: worktree
if: (success() || failure()) && steps.container.outcome == 'success' && !startsWith(github.ref, 'refs/tags/')
if: steps.download-doc.outcome == 'success'
run: |
git config --global --add safe.directory $(pwd)
git config --global user.email "[email protected]"
git config --global user.name "Build documentation workflow"
mkdir -p doc
# Check if we are on PR
unzip doc.zip
PR_NUMBER=""
if [[ -n "$GITHUB_REF" ]]; then
if [[ "$GITHUB_REF" =~ refs/pull/([0-9]+)/merge ]]; then
PR_NUMBER="${BASH_REMATCH[1]}"
fi
if [[ "$GITHUB_REF" =~ refs/pull/([0-9]+)/merge ]]; then
PR_NUMBER="${BASH_REMATCH[1]}"
fi
# If so, then prepare to create CHANGES.html
# Create CHANGES.html
if [[ -n "$PR_NUMBER" ]]; then
# mathjax path in old doc (regex)
mathjax_path_from="[-./A-Za-z_]*/tex-chtml[.]js?v=[0-9a-f]*"
# mathjax path in new doc
mathjax_path_to=$(docker exec -e SAGE_USE_CDNS=yes BUILD /sage/sage -python -c "from sage_docbuild.conf import mathjax_path; print(mathjax_path)")
new_version=$(docker exec BUILD cat src/VERSION.txt)
docker cp BUILD:/sage/local/share/doc/sage/html doc/
# Wipe out chronic diffs between old doc and new doc
(cd doc && \
find . -name "*.html" | xargs sed -i -e '/class="sidebar-brand-text"/ s/Sage [0-9a-z.]* /Sage '"$new_version"' /' \
-e '/<link rel="stylesheet"/ s/?v=[0-9a-f]*"/"/' \
-e 's;?v=[0-9a-f]*";";' \
-e 's;'"$mathjax_path_from"';'"$mathjax_path_to"';' \
-e '\;<script type="application/vnd\.jupyter\.widget-state+json">;,\;</script>; d' \
-e 's;#L[0-9]*";";' \
-e 's;tab-set--[0-9]*-;tab-set-;' \
&& true)
# Create git repo from old doc
(cd doc && \
git init && \
(echo "*.svg binary"; echo "*.pdf binary") >> .gitattributes && \
(echo ".buildinfo"; echo '*.inv'; echo '.git*'; echo '*.svg'; echo '*.pdf'; echo '*.png'; echo 'searchindex.js') > .gitignore; \
(echo "*.svg binary"; echo "*.pdf binary") > .gitattributes && \
(echo ".buildinfo"; echo '*.inv'; echo '.git*'; echo '*.svg'; echo '*.pdf'; echo '*.png'; echo 'searchindex.js') > .gitignore && \
git add -A && git commit --quiet -m 'old')
fi
- name: Build doc
id: docbuild
if: (success() || failure()) && steps.worktree.outcome == 'success' && !startsWith(github.ref, 'refs/tags/')
if: steps.container.outcome == 'success' && !startsWith(github.ref, 'refs/tags/')
# Always non-incremental because of the concern that
# incremental docbuild may introduce broken links (inter-file references) though build succeeds
run: |
Expand All @@ -175,18 +193,22 @@ jobs:
export MAKE="make -j5 --output-sync=recurse" SAGE_NUM_THREADS=5
make doc-clean doc-uninstall
export SAGE_USE_CDNS=yes
export SAGE_DOCBUILD_OPTS="--include-tests-blocks"
./config.status && make sagemath_doc_html-no-deps
shell: sh .ci/docker-exec-script.sh BUILD /sage {0}

- name: Copy doc
id: copy
if: (success() || failure()) && steps.docbuild.outcome == 'success'
if: steps.docbuild.outcome == 'success'
run: |
set -ex
# We copy everything to a local folder
docker cp --follow-link BUILD:/sage/local/share/doc/sage/html doc
docker cp --follow-link BUILD:/sage/local/share/doc/sage/index.html doc
# Check if we are on PR
# Simpler "docker cp --follow-link ... doc" does not work
mkdir -p doc
mkdir -p temp
docker cp --follow-link BUILD:/sage/local/share/doc/sage/html temp
docker cp --follow-link BUILD:/sage/local/share/doc/sage/index.html temp
cp -r -L temp/* doc/
# Check if we are on pull request event
PR_NUMBER=""
if [[ -n "$GITHUB_REF" ]]; then
if [[ "$GITHUB_REF" =~ refs/pull/([0-9]+)/merge ]]; then
Expand All @@ -199,9 +221,10 @@ jobs:
# Wipe out chronic diffs of new doc against old doc before creating CHANGES.html
(cd doc && \
find . -name "*.html" | xargs sed -i -e '/This is documentation/ s/ built with GitHub PR .* for development/ for development/' \
-e '/<link rel="stylesheet"/ s/?v=[0-9a-f]*"/"/' \
-e 's;?v=[0-9a-f]*";";' \
-e '\;<script type="application/vnd\.jupyter\.widget-state+json">;,\;</script>; d' \
-e 's;#L[0-9]*";";' \
-e 's;tab-set--[0-9]*-;tab-set-;' \
&& git commit -a -m 'wipe-out')
# Since HEAD is at commit 'wipe-out', HEAD~1 is commit 'new' (new doc), HEAD~2 is commit 'old' (old doc)
(cd doc && git diff $(git rev-parse HEAD~2) -- "*.html") > diff.txt
Expand All @@ -221,20 +244,31 @@ jobs:
- name: Upload doc
id: upload
if: (success() || failure()) && steps.copy.outcome == 'success'
if: steps.copy.outcome == 'success'
uses: actions/upload-artifact@v4
with:
name: doc
path: doc.zip

- name: Upload doc-develop
# artifact doc-develop is used for doc build on pull request event
id: upload-push
if: steps.copy.outcome == 'success' && github.event_name == 'push'
uses: actions/upload-artifact@v4
with:
name: doc-${{ github.ref_name }}
path: doc.zip

#
# On release tags: live doc and wheels
# On release tag event
#

- name: Build live doc
id: buildlivedoc
if: (success() || failure()) && startsWith(github.ref, 'refs/tags/')
if: startsWith(github.ref, 'refs/tags/')
run: |
# Avoid running out of disk space
rm -rf upstream
export MAKE="make -j5 --output-sync=recurse" SAGE_NUM_THREADS=5
export PATH="build/bin:$PATH"
eval $(sage-print-system-package-command auto update)
Expand All @@ -249,7 +283,7 @@ jobs:

- name: Copy live doc
id: copylivedoc
if: (success() || failure()) && steps.buildlivedoc.outcome == 'success'
if: steps.buildlivedoc.outcome == 'success'
run: |
mkdir -p ./livedoc
# We copy everything to a local folder
Expand All @@ -259,7 +293,7 @@ jobs:
zip -r livedoc.zip livedoc
- name: Upload live doc
if: (success() || failure()) && steps.copylivedoc.outcome == 'success'
if: steps.copylivedoc.outcome == 'success'
uses: actions/upload-artifact@v4
with:
name: livedoc
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/docker.yml
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,7 @@ jobs:
ref: ${{ inputs.sage_ref }}
fetch-depth: 10000
- name: Download upstream artifact
uses: actions/download-artifact@v3
uses: actions/download-artifact@v4
with:
path: upstream
name: ${{ inputs.upstream_artifact }}
Expand Down Expand Up @@ -260,7 +260,7 @@ jobs:
cp -r .tox/$TOX_ENV/* "artifacts/$LOGS_ARTIFACT_NAME"
rm -rf "artifacts/$LOGS_ARTIFACT_NAME"/{bin,lib,pyvenv.cfg}
if: always()
- uses: actions/upload-artifact@v3
- uses: actions/upload-artifact@v4
with:
path: artifacts
name: ${{ env.LOGS_ARTIFACT_NAME }}
Expand Down
8 changes: 4 additions & 4 deletions .github/workflows/macos.yml
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ jobs:
run: |
"${{ steps.python.outputs.python-path }}" -m pip install pipx
- name: Download upstream artifact
uses: actions/download-artifact@v3
uses: actions/download-artifact@v4
with:
path: upstream
name: ${{ inputs.upstream_artifact }}
Expand All @@ -108,7 +108,7 @@ jobs:
GH_TOKEN: ${{ github.token }}
SAGE_CI_FIXES_FROM_REPOSITORIES: ${{ vars.SAGE_CI_FIXES_FROM_REPOSITORIES }}

- uses: actions/download-artifact@v3
- uses: actions/download-artifact@v4
with:
path: sage-local-artifact
name: ${{ env.LOCAL_ARTIFACT_NAME }}
Expand Down Expand Up @@ -147,7 +147,7 @@ jobs:
run: |
mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"; cp -r .tox/*/log "artifacts/$LOGS_ARTIFACT_NAME"
if: always()
- uses: actions/upload-artifact@v3
- uses: actions/upload-artifact@v4
with:
path: artifacts
name: ${{ env.LOGS_ARTIFACT_NAME }}
Expand All @@ -165,7 +165,7 @@ jobs:
run: |
mkdir -p sage-local-artifact && (cd .tox/$TOX_ENV && rm -f "local/lib64" && tar -cf - $(pwd)) > sage-local-artifact/sage-${{ env.TOX_ENV }}-${{ inputs.stage }}.tar
if: contains(inputs.stage, '1')
- uses: actions/upload-artifact@v3
- uses: actions/upload-artifact@v4
with:
path: sage-local-artifact/sage-${{ env.TOX_ENV }}-${{ inputs.stage }}.tar
name: ${{ env.LOCAL_ARTIFACT_NAME }}
Expand Down
Loading

0 comments on commit 03ffc18

Please sign in to comment.