Fix doc deployment to netlify for releases

.ci/create-changes-html.sh

@@ -19,7 +19,9 @@ echo '<script>hljs.highlightAll();</script>' >> CHANGES.html
 cat >> CHANGES.html << EOF
 <script>
 document.addEventListener('DOMContentLoaded', () => {
-const baseDocURL = 'https://sagemath.netlify.app'
+// This URL is hardcoded in the file .github/workflows/doc-publish.yml.
+// See NETLIFY_ALIAS of the "Deploy to Netlify" step.
+const baseDocURL = 'https://doc-develop--sagemath.netlify.app'
 const diffSite = 'https://pianomister.github.io/diffsite'
 const diffParagraphs = document.querySelectorAll('p.diff');
 diffParagraphs.forEach(paragraph => {

.github/workflows/doc-build-pdf.yml

@@ -30,7 +30,7 @@ env:
   EXTRA_CONFIGURE_ARGS:   --enable-fat-binary
 
 jobs:
-  build-docs-pdf:
+  build-doc-pdf:
     runs-on: ubuntu-latest
     services:
       # https://docs.docker.com/build/ci/github-actions/local-registry/
@@ -110,7 +110,9 @@ jobs:
                      --workdir $(pwd) \
                      ${{ env.BUILD_IMAGE }} /bin/sh
 
-      # Docs
+      #
+      # On PRs and pushes to branches
+      #
 
       - name: Update system packages
         id: packages
@@ -122,29 +124,27 @@ jobs:
           eval $(sage-print-system-package-command auto --spkg --yes --no-install-recommends install git texlive texlive_luatex free_fonts xindy)
         shell: sh .ci/docker-exec-script.sh BUILD /sage {0}
 
-      - name: Build docs (PDF)
+      - name: Build doc (PDF)
         id: docbuild
         if: (success() || failure()) && steps.packages.outcome == 'success'
         run: |
           export MAKE="make -j5 --output-sync=recurse" SAGE_NUM_THREADS=5
           make doc-clean doc-uninstall; make sagemath_doc_html-build-deps sagemath_doc_pdf-no-deps
         shell: sh .ci/docker-exec-script.sh BUILD /sage {0}
 
-      - name: Copy docs
+      - name: Copy doc
         id: copy
         if: (success() || failure()) && steps.docbuild.outcome == 'success'
         run: |
-          # For some reason the deploy step below cannot find /sage/...
-          # So copy everything from there to local folder
-          mkdir -p ./docs
-          cp -r -L /sage/local/share/doc/sage/pdf ./docs
+          mkdir -p ./doc
+          # We copy everything to a local folder
+          docker cp BUILD:/sage/local/share/doc/sage/pdf doc
           # Zip everything for increased performance
-          zip -r docs-pdf.zip docs
-        shell: sh .ci/docker-exec-script.sh BUILD /sage {0}
+          zip -r doc-pdf.zip doc
 
-      - name: Upload docs
+      - name: Upload doc
         if: (success() || failure()) && steps.copy.outcome == 'success'
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: docs-pdf
-          path: docs-pdf.zip
+          name: doc-pdf
+          path: doc-pdf.zip

.github/workflows/doc-build.yml

@@ -4,8 +4,15 @@ on:
   pull_request:
   merge_group:
   push:
+    tags:
+      # Match all release tags including beta, rc
+      - '[0-9]+.[0-9]+'
+      - '[0-9]+.[0-9]+.[0-9]+'
+      - '[0-9]+.[0-9]+.beta[0-9]+'
+      - '[0-9]+.[0-9]+.[0-9]+.beta[0-9]+'
+      - '[0-9]+.[0-9]+.rc[0-9]+'
+      - '[0-9]+.[0-9]+.[0-9]+.rc[0-9]+'
     branches:
-      - master
       - develop
   workflow_dispatch:
     # Allow to run manually
@@ -34,7 +41,7 @@ env:
   EXTRA_CONFIGURE_ARGS:   --enable-fat-binary
 
 jobs:
-  build-docs:
+  build-doc:
     runs-on: ubuntu-latest
     services:
       # https://docs.docker.com/build/ci/github-actions/local-registry/
@@ -114,11 +121,13 @@ jobs:
                      --workdir $(pwd) \
                      ${{ env.BUILD_IMAGE }} /bin/sh
 
-      # Docs
+      #
+      # On PRs and pushes to master or develop
+      #
 
-      - name: Store old docs
+      - name: Store old doc
         id: worktree
-        if: (success() || failure()) && steps.container.outcome == 'success'
+        if: (success() || failure()) && steps.container.outcome == 'success' && !startsWith(github.ref, 'refs/tags/')
         run: |
           git config --global --add safe.directory $(pwd)
           git config --global user.email "[email protected]"
@@ -128,24 +137,24 @@ jobs:
           # mathjax path in new doc
           mathjax_path_to=$(docker exec -e SAGE_USE_CDNS=yes BUILD /sage/sage -python -c "from sage_docbuild.conf import mathjax_path; print(mathjax_path)")
           new_version=$(docker exec BUILD cat src/VERSION.txt)
-          mkdir -p docs/
-          docker cp BUILD:/sage/local/share/doc/sage/html docs/
-          # Wipe out chronic diffs of old doc against new doc
-          (cd docs && \
+          mkdir -p doc/
+          docker cp BUILD:/sage/local/share/doc/sage/html doc/
+          # Wipe out chronic diffs between old doc and new doc
+          (cd doc && \
            find . -name "*.html" | xargs sed -i -e '/class="sidebar-brand-text"/ s/Sage [0-9a-z.]* /Sage '"$new_version"' /' \
                                                 -e '/<link rel="stylesheet"/ s/?v=[0-9a-f]*"/"/' \
                                                 -e 's;'"$mathjax_path_from"';'"$mathjax_path_to"';' \
                                                 -e '\;<script type="application/vnd\.jupyter\.widget-state+json">;,\;</script>; d')
           # Create git repo from old doc
-          (cd docs && \
+          (cd doc && \
            git init && \
            (echo "*.svg binary"; echo "*.pdf binary") >> .gitattributes && \
            (echo ".buildinfo"; echo '*.inv'; echo '.git*'; echo '*.svg'; echo '*.pdf'; echo '*.png'; echo 'searchindex.js') > .gitignore; \
            git add -A && git commit --quiet -m "old")
 
-      - name: Build docs
+      - name: Build doc
         id: docbuild
-        if: (success() || failure()) && steps.worktree.outcome == 'success'
+        if: (success() || failure()) && steps.worktree.outcome == 'success' && !startsWith(github.ref, 'refs/tags/')
         # Always non-incremental because of the concern that
         # incremental docbuild may introduce broken links (inter-file references) though build succeeds
         run: |
@@ -157,50 +166,43 @@ jobs:
           ./config.status && make sagemath_doc_html-no-deps
         shell: sh .ci/docker-exec-script.sh BUILD /sage {0}
 
-      - name: Copy docs
+      - name: Copy doc
         id: copy
         if: (success() || failure()) && steps.docbuild.outcome == 'success'
         run: |
           set -ex
           # We copy everything to a local folder
-          docker cp BUILD:/sage/local/share/doc/sage/html docs
-          docker cp BUILD:/sage/local/share/doc/sage/index.html docs
-          (cd docs && git commit -a -m 'new')
+          docker cp --follow-link BUILD:/sage/local/share/doc/sage/html doc
+          docker cp --follow-link BUILD:/sage/local/share/doc/sage/index.html doc
+          (cd doc && git commit -a -m 'new')
           # Wipe out chronic diffs of new doc against old doc
-          (cd docs && \
+          (cd doc && \
            find . -name "*.html" | xargs sed -i -e '/This is documentation for/ s/ built with GitHub PR .*. Doc/. Doc/' \
                                                 -e '/<link rel="stylesheet"/ s/?v=[0-9a-f]*"/"/')
-          .ci/create-changes-html.sh $(cd docs && git rev-parse HEAD^) docs
+          .ci/create-changes-html.sh $(cd doc && git rev-parse HEAD^) doc
           # Restore the new doc from changes by "wipe out"
-          (cd docs && git checkout -f)
-          (cd docs && rm -rf .git)
-          mv CHANGES.html docs
-          # We also need to replace the symlinks because netlify is not following them
-          # CHECK IF STILL NEEDED
-          #cp -r -L $DOC_DIR ./docs
+          (cd doc && git checkout -f)
+          # Sometimes rm -rf .git errors out because of some diehard hidden files
+          (cd doc && rm -rf .git 2>/dev/null && rm .gitattributes 2>/dev/null)
+          mv CHANGES.html doc
           # Zip everything for increased performance
-          zip -r docs.zip docs
+          zip -r doc.zip doc
 
-      - name: Upload docs
+      - name: Upload doc
         id: upload
         if: (success() || failure()) && steps.copy.outcome == 'success'
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: docs
-          path: docs.zip
+          name: doc
+          path: doc.zip
 
-      - name: Save space
-        id: savespace
-        if: (success() || failure()) && steps.upload.outcome == 'success' && github.repository == 'sagemath/sage' && github.ref == 'refs/heads/develop'
-        run: |
-          set -ex
-          # Save space on runner device before we start to build livedoc
-          rm -rf docs/
-          rm -f docs.zip
+      #
+      # On release tags: live doc and wheels
+      #
 
       - name: Build live doc
         id: buildlivedoc
-        if: (success() || failure()) && steps.savespace.outcome == 'success'
+        if: (success() || failure()) && startsWith(github.ref, 'refs/tags/')
         run: |
           export MAKE="make -j5 --output-sync=recurse" SAGE_NUM_THREADS=5
           export PATH="build/bin:$PATH"
@@ -227,7 +229,7 @@ jobs:
 
       - name: Upload live doc
         if: (success() || failure()) && steps.copylivedoc.outcome == 'success'
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: livedoc
           path: livedoc.zip

.github/workflows/doc-publish.yml

@@ -14,54 +14,45 @@ permissions:
   pull-requests: write
 
 jobs:
-  upload-docs:
+  publish-doc:
     runs-on: ubuntu-latest
     if: github.event.workflow_run.conclusion == 'success'
+    env:
+      CAN_DEPLOY: ${{ secrets.NETLIFY_AUTH_TOKEN != '' &&  secrets.NETLIFY_SITE_ID != '' }}
     steps:
       - name: Get information about workflow origin
         uses: potiuk/get-workflow-origin@v1_5
         id: source-run-info
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           sourceRunId: ${{ github.event.workflow_run.id }}
+        if: env.CAN_DEPLOY == 'true'
 
-      # Once https://github.com/actions/download-artifact/issues/172 and/or https://github.com/actions/download-artifact/issues/60 is implemented, we can use the official download-artifact action
-      # For now use the solution from https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#using-data-from-the-triggering-workflow
-      - name: Download docs
-        uses: actions/[email protected]
+      - name: Download doc
+        id: download-doc
+        uses: actions/download-artifact@v4
         with:
-          script: |
-            var artifacts = await github.rest.actions.listWorkflowRunArtifacts({
-               owner: context.repo.owner,
-               repo: context.repo.repo,
-               run_id: ${{github.event.workflow_run.id }},
-            });
-            var matchArtifact = artifacts.data.artifacts.filter((artifact) => {
-              return artifact.name == "docs"
-            })[0];
-            var download = await github.rest.actions.downloadArtifact({
-               owner: context.repo.owner,
-               repo: context.repo.repo,
-               artifact_id: matchArtifact.id,
-               archive_format: 'zip',
-            });
-            var fs = require('fs');
-            fs.writeFileSync('${{github.workspace}}/docs.zip', Buffer.from(download.data));
+          name: doc
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          repository: ${{ github.repository }}
+          run-id: ${{ github.event.workflow_run.id }}
+        if: steps.source-run-info.outputs.sourceEvent == 'pull_request' || (steps.source-run-info.outputs.sourceEvent == 'push' && steps.source-run-info.outputs.targetBranch == 'develop')
 
-      - name: Extract docs
-        run: unzip docs.zip -d docs && unzip docs/docs.zip -d docs/docs
+      - name: Extract doc
+        run: unzip doc.zip -d doc
+        if: steps.download-doc.outcome == 'success'
 
       - name: Deploy to Netlify
         id: deploy-netlify
         uses: netlify/actions/cli@master
         with:
-          args: deploy --dir=docs/docs/docs ${NETLIFY_PRODUCTION:+"--prod"} --message ${NETLIFY_MESSAGE} --alias ${NETLIFY_ALIAS}
+          args: deploy --dir=doc/doc --message ${NETLIFY_MESSAGE} --alias ${NETLIFY_ALIAS}
         env:
           NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
           NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }}
-          NETLIFY_PRODUCTION: ${{ github.ref == 'refs/heads/develop' }}
-          NETLIFY_MESSAGE: ${{ steps.source-run-info.outputs.pullRequestNumber }}
-          NETLIFY_ALIAS: preview-${{ steps.source-run-info.outputs.pullRequestNumber }}
+          NETLIFY_MESSAGE:  doc-${{ steps.source-run-info.outputs.pullRequestNumber && format('pr-{0}', steps.source-run-info.outputs.pullRequestNumber) || 'develop' }}
+          NETLIFY_ALIAS: doc-${{ steps.source-run-info.outputs.pullRequestNumber && format('pr-{0}', steps.source-run-info.outputs.pullRequestNumber) || 'develop' }}
+        if: steps.download-doc.outcome == 'success'
 
       # Add deployment as status check, PR comment and annotation
       # we could use the nwtgck/actions-netlify action for that, except for that it is not (yet) working in workflow_run context: https://github.com/nwtgck/actions-netlify/issues/545
@@ -74,10 +65,10 @@ jobs:
           message: |
             [Documentation preview for this PR](${{ steps.deploy-netlify.outputs.NETLIFY_URL }}/html/en) (built with commit ${{ steps.source-run-info.outputs.sourceHeadSha }}; [changes](${{ steps.deploy-netlify.outputs.NETLIFY_URL }}/CHANGES.html)) is ready! :tada:
             This preview will update shortly after each push to this PR.
+        if: steps.download-doc.outcome == 'success'
 
       - name: Update deployment status PR check
         uses: myrotvorets/[email protected]
-        if: ${{ always() }}
         env:
           DEPLOY_SUCCESS: Successfully deployed preview.
           DEPLOY_FAILURE: Failed to deploy preview.
@@ -88,50 +79,54 @@ jobs:
           context: Deploy Documentation
           targetUrl: ${{ steps.deploy-netlify.outputs.NETLIFY_URL }}
           description: ${{ job.status == 'success' && env.DEPLOY_SUCCESS || env.DEPLOY_FAILURE }}
+        if: steps.download-doc.outcome == 'success'
 
       - name: Report deployment url
         run: |
           echo "::notice::The documentation has been deployed - ${{ steps.deploy-netlify.outputs.NETLIFY_URL }}"
+        if: steps.download-doc.outcome == 'success'
 
   publish-live-doc:
     runs-on: ubuntu-latest
-    if: github.event.workflow_run.conclusion == 'success' && github.repository == 'sagemath/sage' && github.event.workflow_run.head_branch == 'develop'
+    if: github.event.workflow_run.conclusion == 'success'
+    env:
+      CAN_DEPLOY: ${{ secrets.NETLIFY_AUTH_TOKEN != '' &&  secrets.NETLIFY_SITE_ID != '' }}
     steps:
+      - name: Get information about workflow origin
+        uses: potiuk/get-workflow-origin@v1_5
+        id: source-run-info
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          sourceRunId: ${{ github.event.workflow_run.id }}
+        if: env.CAN_DEPLOY == 'true'
+
       - name: Download live doc
-        uses: actions/[email protected]
+        id: download-doc
+        uses: actions/download-artifact@v4
         with:
-          script: |
-            var artifacts = await github.rest.actions.listWorkflowRunArtifacts({
-               owner: context.repo.owner,
-               repo: context.repo.repo,
-               run_id: ${{github.event.workflow_run.id }},
-            });
-            var matchArtifact = artifacts.data.artifacts.filter((artifact) => {
-              return artifact.name == "livedoc"
-            })[0];
-            var download = await github.rest.actions.downloadArtifact({
-               owner: context.repo.owner,
-               repo: context.repo.repo,
-               artifact_id: matchArtifact.id,
-               archive_format: 'zip',
-            });
-            var fs = require('fs');
-            fs.writeFileSync('${{github.workspace}}/livedoc.zip', Buffer.from(download.data));
+          name: livedoc
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          repository: ${{ github.repository }}
+          run-id: ${{ github.event.workflow_run.id }}
+        if: steps.source-run-info.outputs.sourceEvent == 'push' && steps.source-run-info.outputs.targetBranch != 'develop'
 
       - name: Extract live doc
-        run: unzip livedoc.zip -d doc && unzip doc/livedoc.zip -d doc/doc
+        run: unzip livedoc.zip -d livedoc
+        if: steps.download-doc.outcome == 'success'
 
       - name: Deploy to Netlify
         id: deploy-netlify
         uses: netlify/actions/cli@master
         with:
-          args: deploy --dir=doc/doc/livedoc --message ${NETLIFY_MESSAGE} --alias ${NETLIFY_ALIAS}
+          args: deploy --dir=livedoc/livedoc --message ${NETLIFY_MESSAGE} --alias ${NETLIFY_ALIAS}
         env:
           NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
           NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }}
-          NETLIFY_MESSAGE: Deployed live doc
-          NETLIFY_ALIAS: livedoc
+          NETLIFY_MESSAGE: doc-${{ steps.source-run-info.outputs.targetBranch }}
+          NETLIFY_ALIAS: doc-${{ steps.source-run-info.outputs.targetBranch }}
+        if: steps.download-doc.outcome == 'success'
 
       - name: Report deployment url
         run: |
           echo "::notice::The live documentation has been deployed - ${{ steps.deploy-netlify.outputs.NETLIFY_URL }}"
+        if: steps.download-doc.outcome == 'success'