From 012c2a3b8a3c0d1484ab69249f4e813a0b06050f Mon Sep 17 00:00:00 2001 From: Dongdong Tian Date: Mon, 15 Mar 2021 00:52:27 -0400 Subject: [PATCH] Separate workflows for running tests and building documentation (#1033) Separates the current "Tests" workflow into two workflows to speedup our CI jobs. - ci_test.yaml: runs the full tests - ci_docs.yml: build the documentation and deploy it on master branch Co-authored-by: Wei Ji <23487320+weiji14@users.noreply.github.com> --- .github/workflows/ci_docs.yml | 164 ++++++++++++++++++++++++++++++++ .github/workflows/ci_tests.yaml | 76 ++------------- MAINTENANCE.md | 22 +++-- 3 files changed, 186 insertions(+), 76 deletions(-) create mode 100644 .github/workflows/ci_docs.yml diff --git a/.github/workflows/ci_docs.yml b/.github/workflows/ci_docs.yml new file mode 100644 index 00000000000..72a32f117d2 --- /dev/null +++ b/.github/workflows/ci_docs.yml @@ -0,0 +1,164 @@ +# This workflow installs PyGMT, builds and deploys documentation + +name: Docs + +on: + push: + branches: [ master ] + pull_request: + types: [opened, reopened, synchronize, ready_for_review] + paths-ignore: + - 'pygmt/tests/**' + - '*.md' + - '*.json' + - 'LICENSE.txt' + - '.gitignore' + - '.pylintrc' + release: + types: + - published + +jobs: + docs: + name: ${{ matrix.os }} - Python ${{ matrix.python-version }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: [3.9] + os: [ubuntu-latest, macOS-latest, windows-latest] + # Is it a draft Pull Request (true or false)? + isDraft: + - ${{ github.event.pull_request.draft }} + # Only run one job (Ubuntu + Python 3.9) for draft PRs + exclude: + - os: macOS-latest + isDraft: true + - os: windows-latest + isDraft: true + defaults: + run: + shell: bash -l {0} + + steps: + # Cancel previous runs that are not completed + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.8.0 + with: + access_token: ${{ github.token }} + + # Checkout current git repository + - name: Checkout + uses: actions/checkout@v2.3.4 + with: + # fecth all history so that setuptools-scm works + fetch-depth: 0 + + # Setup Miniconda + - name: Setup Miniconda + uses: conda-incubator/setup-miniconda@v2.0.1 + with: + activate-environment: pygmt + python-version: ${{ matrix.python-version }} + channels: conda-forge + miniconda-version: "latest" + + # Install GMT and other required dependencies from conda-forge + - name: Install dependencies + run: | + conda install gmt=6.1.1 numpy pandas xarray netCDF4 packaging \ + ipython make myst-parser \ + sphinx sphinx-copybutton sphinx-gallery sphinx_rtd_theme=0.4.3 + + # Show installed pkg information for postmortem diagnostic + - name: List installed packages + run: conda list + + # Download cached remote files (artifacts) from GitHub + - name: Download remote data from GitHub + uses: dawidd6/action-download-artifact@v2.12.0 + with: + workflow: cache_data.yaml + workflow_conclusion: success + name: gmt-cache + path: .gmt + + # Move downloaded files to ~/.gmt directory and list them + - name: Move and list downloaded remote files + run: | + mkdir -p ~/.gmt + mv .gmt/* ~/.gmt + # Change modification times of the two files, so GMT won't refresh it + touch ~/.gmt/server/gmt_data_server.txt ~/.gmt/server/gmt_hash_server.txt + ls -lhR ~/.gmt + + # Install the package that we want to test + - name: Install the package + run: | + python setup.py sdist --formats=zip + pip install dist/* + + # Build the documentation + - name: Build the documentation + run: make -C doc clean all + + - name: Checkout the gh-pages branch + uses: actions/checkout@28c7f3d2b5162b5ddd3dfd9a45aa55eaf396478b + with: + ref: gh-pages + # Checkout to this folder instead of the current one + path: deploy + # Download the entire history + fetch-depth: 0 + if: (github.event_name == 'release' || github.event_name == 'push') && (matrix.os == 'ubuntu-latest') + + - name: Push the built HTML to gh-pages + run: | + # Detect if this is a release or from the master branch + if [[ "${GITHUB_EVENT_NAME}" == "release" ]]; then + # Get the tag name without the "refs/tags/" part + version="${GITHUB_REF#refs/*/}" + else + version=dev + fi + echo "Deploying version: $version" + # Make the new commit message. Needs to happen before cd into deploy + # to get the right commit hash. + message="Deploy $version from $(git rev-parse --short HEAD)" + cd deploy + # Need to have this file so that Github doesn't try to run Jekyll + touch .nojekyll + # Delete all the files and replace with our new set + echo -e "\nRemoving old files from previous builds of ${version}:" + rm -rvf ${version} + echo -e "\nCopying HTML files to ${version}:" + cp -Rvf ../doc/_build/html/ ${version}/ + # If this is a new release, update the link from /latest to it + if [[ "${version}" != "dev" ]]; then + echo -e "\nSetup link from ${version} to 'latest'." + rm -f latest + ln -sf ${version} latest + fi + # Stage the commit + git add -A . + echo -e "\nChanges to be applied:" + git status + # Configure git to be the GitHub Actions account + git config user.email "github-actions[bot]@users.noreply.github.com" + git config user.name "github-actions[bot]" + # If this is a dev build and the last commit was from a dev build + # (detect if "dev" was in the previous commit message), reuse the + # same commit + if [[ "${version}" == "dev" && `git log -1 --format='%s'` == *"dev"* ]]; then + echo -e "\nAmending last commit:" + git commit --amend --reset-author -m "$message" + else + echo -e "\nMaking a new commit:" + git commit -m "$message" + fi + # Make the push quiet just in case there is anything that could leak + # sensitive information. + echo -e "\nPushing changes to gh-pages." + git push -fq origin gh-pages 2>&1 >/dev/null + echo -e "\nFinished uploading generated files." + if: (github.event_name == 'release' || github.event_name == 'push') && (matrix.os == 'ubuntu-latest') diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index 0ac19e209a4..73e4624ff4e 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -1,5 +1,4 @@ -# This workflow installs PyGMT dependencies, build documentation and run tests -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions +# This workflow installs PyGMT and runs tests name: Tests @@ -10,10 +9,13 @@ on: types: [opened, reopened, synchronize, ready_for_review] paths-ignore: - 'doc/**' + - 'examples/**' - '*.md' - '*.json' - 'README.rst' - 'LICENSE.txt' + - '.gitignore' + - '.pylintrc' release: types: - published @@ -79,7 +81,10 @@ jobs: # Install GMT and other required dependencies from conda-forge - name: Install dependencies - run: conda env update --file environment.yml + run: | + conda install gmt=6.1.1 numpy pandas xarray netCDF4 packaging \ + codecov coverage[toml] ipython make \ + pytest-cov pytest-mpl pytest>=6.0 # Show installed pkg information for postmortem diagnostic - name: List installed packages @@ -121,10 +126,6 @@ jobs: name: artifact-${{ runner.os }}-${{ matrix.python-version }} path: tmp-test-dir-with-unique-name - # Build the documentation - - name: Build the documentation - run: make -C doc clean all - # Upload coverage to Codecov - name: Upload coverage to Codecov uses: codecov/codecov-action@v1.2.1 @@ -132,64 +133,3 @@ jobs: file: ./coverage.xml # optional env_vars: OS,PYTHON fail_ci_if_error: false - - - name: Checkout the gh-pages branch - uses: actions/checkout@28c7f3d2b5162b5ddd3dfd9a45aa55eaf396478b - with: - ref: gh-pages - # Checkout to this folder instead of the current one - path: deploy - # Download the entire history - fetch-depth: 0 - if: (github.event_name == 'release' || github.event_name == 'push') && (matrix.os == 'ubuntu-latest') && (matrix.python-version == '3.9') - - - name: Push the built HTML to gh-pages - run: | - # Detect if this is a release or from the master branch - if [[ "${GITHUB_EVENT_NAME}" == "release" ]]; then - # Get the tag name without the "refs/tags/" part - version="${GITHUB_REF#refs/*/}" - else - version=dev - fi - echo "Deploying version: $version" - # Make the new commit message. Needs to happen before cd into deploy - # to get the right commit hash. - message="Deploy $version from $(git rev-parse --short HEAD)" - cd deploy - # Need to have this file so that Github doesn't try to run Jekyll - touch .nojekyll - # Delete all the files and replace with our new set - echo -e "\nRemoving old files from previous builds of ${version}:" - rm -rvf ${version} - echo -e "\nCopying HTML files to ${version}:" - cp -Rvf ../doc/_build/html/ ${version}/ - # If this is a new release, update the link from /latest to it - if [[ "${version}" != "dev" ]]; then - echo -e "\nSetup link from ${version} to 'latest'." - rm -f latest - ln -sf ${version} latest - fi - # Stage the commit - git add -A . - echo -e "\nChanges to be applied:" - git status - # Configure git to be the GitHub Actions account - git config user.email "github-actions[bot]@users.noreply.github.com" - git config user.name "github-actions[bot]" - # If this is a dev build and the last commit was from a dev build - # (detect if "dev" was in the previous commit message), reuse the - # same commit - if [[ "${version}" == "dev" && `git log -1 --format='%s'` == *"dev"* ]]; then - echo -e "\nAmending last commit:" - git commit --amend --reset-author -m "$message" - else - echo -e "\nMaking a new commit:" - git commit -m "$message" - fi - # Make the push quiet just in case there is anything that could leak - # sensitive information. - echo -e "\nPushing changes to gh-pages." - git push -fq origin gh-pages 2>&1 >/dev/null - echo -e "\nFinished uploading generated files." - if: (github.event_name == 'release' || github.event_name == 'push') && (matrix.os == 'ubuntu-latest') && (matrix.python-version == '3.9') diff --git a/MAINTENANCE.md b/MAINTENANCE.md index d4477f90b4b..69ba4ec77dc 100644 --- a/MAINTENANCE.md +++ b/MAINTENANCE.md @@ -65,7 +65,7 @@ build and test the project on Linux, macOS and Windows. They rely on the `environment.yml` file to install required dependencies using conda and the `Makefile` to run the tests and checks. -There are 8 configuration files located in `.github/workflows`: +There are 9 configuration files located in `.github/workflows`: 1. `style_checks.yaml` (Code lint and style checks) @@ -76,9 +76,15 @@ There are 8 configuration files located in `.github/workflows`: This is run on every commit to the *master* and Pull Request branches. It is also scheduled to run daily on the *master* branch. - In draft Pull Requests, only one job (Ubuntu + Python latest) + In draft Pull Requests, only one job (Linux + Python latest) is triggered to save on Continuous Integration resources. +3. `ci_docs.yml` (Build documentation on Linux/macOS/Windows) + + This is run on every commit to the *master* and Pull Request branches. + In draft Pull Requests, only the job on Linux is triggered to save on + Continuous Integration resources. + On the *master* branch, the workflow also handles the documentation deployment: @@ -86,35 +92,35 @@ There are 8 configuration files located in `.github/workflows`: from the *master* branch onto the `dev` folder of the *gh-pages* branch. * Updating the `latest` documentation link to the new release. -3. `ci_tests_dev.yaml` (GMT Dev Tests on Linux/macOS/Windows). +4. `ci_tests_dev.yaml` (GMT Dev Tests on Linux/macOS/Windows). This is triggered when a PR is marked as "ready for review", or using the slash command `/test-gmt-dev`. It is also scheduled to run daily on the *master* branch. -4. `cache_data.yaml` (Caches GMT remote data files needed for GitHub Actions CI) +5. `cache_data.yaml` (Caches GMT remote data files needed for GitHub Actions CI) This is scheduled to run every Sunday at 12:00 (UTC). If new remote files are needed urgently, maintainers can manually uncomment the 'pull_request:' line in that `cache_data.yaml` file to refresh the cache. -5. `publish-to-pypi.yml` (Publish wheels to PyPI and TestPyPI) +6. `publish-to-pypi.yml` (Publish wheels to PyPI and TestPyPI) This workflow is run to publish wheels to PyPI and TestPyPI (for testing only). Archives will be pushed to TestPyPI on every commit to the *master* branch and tagged releases, and to PyPI for tagged releases only. -6. `release-drafter.yml` (Drafts the next release notes) +7. `release-drafter.yml` (Drafts the next release notes) This workflow is run to update the next releases notes as pull requests are merged into master. -7. `check-links.yml` (Check links in the repository and website) +8. `check-links.yml` (Check links in the repository and website) This workflow is run weekly to check all external links in plaintext and HTML files. It will create an issue if broken links are found. -8. `format-command.yml` (Format the codes using slash command) +9. `format-command.yml` (Format the codes using slash command) This workflow is triggered in a PR if the slash command `/format` is used.