Skip to content

Commit

Permalink
Auto merge of #112407 - tgross35:ci-docs-publish, r=Mark-Simulacrum
Browse files Browse the repository at this point in the history
Publish docs as github artifacts during CI

Discussed here: https://rust-lang.zulipchat.com/#narrow/stream/242791-t-infra/topic/Building.20docs.20for.20PR.20CI

The goal is to make docs available for download after CI runs on PRs, for easy review of API changes.

Notes:

- Currently this only captures library documentation (`core`, `alloc`, `std`, `test`, `proc_macro`)
- You can't see artifacts until the entire workflow run has completed actions/upload-artifact#53
- There is currently a generic file name `ci-artifacts`. No way to customize this based on contained files unfortunately actions/upload-artifact#349

You can find the results at the bottom of the CI "summary" page:

<img width="379" alt="image" src="https://github.com/rust-lang/rust/assets/13724985/d3748e59-242c-40f8-9f54-82177b9b481b">
  • Loading branch information
bors committed Jun 17, 2023
2 parents 0cc541e + 696b0dd commit 1d7d824
Show file tree
Hide file tree
Showing 4 changed files with 97 additions and 0 deletions.
36 changes: 36 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ jobs:
env:
CI_JOB_NAME: "${{ matrix.name }}"
CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
HEAD_SHA: "${{ github.event.pull_request.head.sha || github.sha }}"
SCCACHE_BUCKET: rust-lang-ci-sccache2
TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate"
CACHE_DOMAIN: ci-caches.rust-lang.org
Expand Down Expand Up @@ -143,6 +144,17 @@ jobs:
AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}"
TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}"
if: success() && !env.SKIP_JOB
- name: create github artifacts
run: src/ci/scripts/create-doc-artifacts.sh
if: success() && !env.SKIP_JOB
- name: upload artifacts to github
uses: actions/upload-artifact@v3
with:
name: "${{ env.DOC_ARTIFACT_NAME }}"
path: obj/artifacts/doc
if-no-files-found: ignore
retention-days: 5
if: success() && !env.SKIP_JOB
- name: upload artifacts to S3
run: src/ci/scripts/upload-artifacts.sh
env:
Expand All @@ -156,6 +168,7 @@ jobs:
env:
CI_JOB_NAME: "${{ matrix.name }}"
CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
HEAD_SHA: "${{ github.event.pull_request.head.sha || github.sha }}"
SCCACHE_BUCKET: rust-lang-ci-sccache2
DEPLOY_BUCKET: rust-lang-ci2
TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate"
Expand Down Expand Up @@ -557,6 +570,17 @@ jobs:
AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}"
TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}"
if: success() && !env.SKIP_JOB
- name: create github artifacts
run: src/ci/scripts/create-doc-artifacts.sh
if: success() && !env.SKIP_JOB
- name: upload artifacts to github
uses: actions/upload-artifact@v3
with:
name: "${{ env.DOC_ARTIFACT_NAME }}"
path: obj/artifacts/doc
if-no-files-found: ignore
retention-days: 5
if: success() && !env.SKIP_JOB
- name: upload artifacts to S3
run: src/ci/scripts/upload-artifacts.sh
env:
Expand All @@ -571,6 +595,7 @@ jobs:
DIST_TRY_BUILD: 1
CI_JOB_NAME: "${{ matrix.name }}"
CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
HEAD_SHA: "${{ github.event.pull_request.head.sha || github.sha }}"
SCCACHE_BUCKET: rust-lang-ci-sccache2
DEPLOY_BUCKET: rust-lang-ci2
TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate"
Expand Down Expand Up @@ -672,6 +697,17 @@ jobs:
AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}"
TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}"
if: success() && !env.SKIP_JOB
- name: create github artifacts
run: src/ci/scripts/create-doc-artifacts.sh
if: success() && !env.SKIP_JOB
- name: upload artifacts to github
uses: actions/upload-artifact@v3
with:
name: "${{ env.DOC_ARTIFACT_NAME }}"
path: obj/artifacts/doc
if-no-files-found: ignore
retention-days: 5
if: success() && !env.SKIP_JOB
- name: upload artifacts to S3
run: src/ci/scripts/upload-artifacts.sh
env:
Expand Down
3 changes: 3 additions & 0 deletions src/ci/docker/host-x86_64/mingw-check/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,9 @@ ENV SCRIPT python3 ../x.py --stage 2 test src/tools/expand-yaml-anchors && \
python3 ../x.py test --stage 0 src/tools/compiletest && \
python3 ../x.py test --stage 0 core alloc std test proc_macro && \
# Build both public and internal documentation.
RUSTDOCFLAGS=\"--document-private-items --document-hidden-items\" python3 ../x.py doc --stage 0 library && \
mkdir -p /checkout/obj/staging/doc && \
cp -r build/x86_64-unknown-linux-gnu/doc /checkout/obj/staging && \
RUSTDOCFLAGS=\"--document-private-items --document-hidden-items\" python3 ../x.py doc --stage 0 compiler && \
RUSTDOCFLAGS=\"--document-private-items --document-hidden-items\" python3 ../x.py doc --stage 0 library/test && \
/scripts/validate-toolstate.sh && \
Expand Down
16 changes: 16 additions & 0 deletions src/ci/github-actions/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ x--expand-yaml-anchors--remove:
- &shared-ci-variables
CI_JOB_NAME: ${{ matrix.name }}
CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
# commit of PR sha or commit sha. `GITHUB_SHA` is not accurate for PRs.
HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}

- &public-variables
SCCACHE_BUCKET: rust-lang-ci-sccache2
Expand Down Expand Up @@ -229,6 +231,20 @@ x--expand-yaml-anchors--remove:
TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}
<<: *step

- name: create github artifacts
run: src/ci/scripts/create-doc-artifacts.sh
<<: *step

- name: upload artifacts to github
uses: actions/upload-artifact@v3
with:
# name is set in previous step
name: ${{ env.DOC_ARTIFACT_NAME }}
path: obj/artifacts/doc
if-no-files-found: ignore
retention-days: 5
<<: *step

- name: upload artifacts to S3
run: src/ci/scripts/upload-artifacts.sh
env:
Expand Down
42 changes: 42 additions & 0 deletions src/ci/scripts/create-doc-artifacts.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
#!/bin/bash
# Compress doc artifacts and name them based on the commit, or the date if
# commit is not available.

set -euox pipefail

# Try to get short commit hash, fallback to date
if [ -n "$HEAD_SHA" ]; then
short_rev=$(echo "${HEAD_SHA}" | cut -c1-8)
else
short_rev=$(git rev-parse --short HEAD || date -u +'%Y-%m-%dT%H%M%SZ')
fi

# Try to get branch, fallback to none
branch=$(git branch --show-current || echo)

if [ -n "$branch" ]; then
branch="${branch}-"
fi

if [ "${GITHUB_EVENT_NAME:=none}" = "pull_request" ]; then
pr_num=$(echo "$GITHUB_REF_NAME" | cut -d'/' -f1)
name="doc-${pr_num}-${short_rev}"
else
name="doc-${branch}${short_rev}"
fi


if [ -d "obj/staging/doc" ]; then
mkdir -p obj/artifacts/doc

# Level 12 seems to give a good tradeoff of time vs. space savings
ZSTD_CLEVEL=12 ZSTD_NBTHREADS=4 \
tar --zstd -cf "obj/artifacts/doc/${name}.tar.zst" -C obj/staging/doc .

ls -lh obj/artifacts/doc
fi

# Set this environment variable for future use if running in CI
if [ -n "$GITHUB_ENV" ]; then
echo "DOC_ARTIFACT_NAME=${name}" >> "$GITHUB_ENV"
fi

0 comments on commit 1d7d824

Please sign in to comment.