diff --git a/.github/workflows/render_example_gallery.yml b/.github/workflows/render_example_gallery.yml new file mode 100644 index 000000000..63bbd8ea7 --- /dev/null +++ b/.github/workflows/render_example_gallery.yml @@ -0,0 +1,147 @@ +# Action to render the example gallery, upload them as artifacts. After the examples are +# rendered, the ReadTheDocs build is triggered pulling the artifacts with the rendered +# example gallery. +# +# This workflow is inspired by the actions in the https://github.com/dfm/rtds-action and +# https://github.com/wradlib/wradlib-notebooks projects. +# +# Steps needed to setup this workflow +# =================================== +# +# Read-the-docs configuration +# --------------------------- +# +# 1. In Admin/Integrations, add a generic API incoming webhook. +# Save the webhook url and token value. +# 2. Add the RTD_GITHUB_TOKEN environmental variable in RTD (Settings/Environment Variables) +# with the pysteps-bot's Github access token. This access token should only +# have the "public_repo" scope. +# At the moment, this token is needed to download the artifacts in RTD's. +# +# Github configuration +# --------------------- +# +# 1. Add the following secrets to the pysteps repository: +# - RTD_WEBHOOK_URL: Read-the-docs webhook. Important: the RTD-Github integration +# must not be activated since this workflow triggers the RTD build. +# - RTD_TOKEN: RTD webhook access token. + +name: Render the example gallery + +on: + # Triggers the workflow on push or pull request events to the master branch + push: + branches: [ master, rtd_using_gh_artifacts ] +# pull_request: +# branches: [ master ] + release: + types: [ published ] + +jobs: + render_example_gallery: + name: Render the example gallery + runs-on: "ubuntu-latest" + + defaults: + run: + shell: bash -l {0} + + outputs: + rtd_branch: ${{steps.set_rendered_branch.outputs.render_branch}} + gallery_branch: ${{steps.set_rendered_branch.outputs.render_branch}} + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Install mamba and create environment + uses: mamba-org/provision-with-micromamba@main + with: + environment-file: ci/ci_test_env.yml + extra-specs: python=3.8 + environment-name: doc_builder + + - name: Install pygrib + run: mamba install --quiet pygrib + + - name: Install sphinx dependencies using pip + run: | + pip install -r doc/requirements.txt + + - name: Install pysteps + working-directory: ${{github.workspace}} + run: | + pip install -e . + + - name: Install pysteps-data + env: + PYSTEPS_DATA_PATH: ${{github.workspace}}/pysteps_data + working-directory: ${{github.workspace}}/ci + run: | + python fetch_pysteps_data.py + python -c "import pysteps; print(pysteps.config_fname())" + + - name: Build example gallery + working-directory: ./doc + env: + PYSTEPSRC: ${{github.workspace}}/pysteps_data/pystepsrc + run: | + make html + + - name: Export conda environment + run: conda env export > doc/source/auto_examples/environment.yml + + - name: Tar auto_examples files + # Needed to maintain permissions and case-sensitive files + # https://github.com/actions/upload-artifact#maintaining-file-permissions-and-case-sensitive-files + run: tar -cvf auto_examples.tar -C doc/source/auto_examples . + + - uses: actions/upload-artifact@v2 + with: + name: auto_examples-for-${{ github.sha }} + path: auto_examples.tar + + trigger_rtd: + # Task inspired in the following wradlib's action: + # https://github.com/wradlib/wradlib-notebooks/blob/main/.github/workflows/render_notebooks.yml + needs: [ render_example_gallery ] + name: Trigger readthedocs + runs-on: ubuntu-latest + defaults: + run: + shell: bash -l {0} + steps: + - name: Determine rendered branch name + id: set_rtd_branch + run: | + if [[ "${GITHUB_EVENT_NAME}" == "push" ]]; then + rtd_branch="${GITHUB_REF##*/}" + elif [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then + rtd_branch=$GITHUB_BASE_REF + elif [[ "${GITHUB_REF_TYPE}" == "tag" ]]; then + rtd_branch="${GITHUB_REF_NAME}" + else + rtd_branch="undefined" + fi + + echo "::set-output name=rtd_branch::${rtd_branch}" + + - name: Print debug information + env: + rtd_branch: ${{steps.set_rtd_branch.outputs.rtd_branch}} + run: | + echo "GITHUB_REF=${GITHUB_REF}" + echo "GITHUB_REF_TYPE=${GITHUB_REF_TYPE}" + echo "GITHUB_REF_NAME=${GITHUB_REF_NAME}" + echo "GITHUB_EVENT_NAME=${GITHUB_EVENT_NAME}" + echo "rtd_branch=${rtd_branch}" + + - name: Trigger readthedocs for the corresponding branch + env: + RTD_TOKEN: ${{ secrets.RTD_TOKEN }} + RTD_WEBHOOK_URL: ${{ secrets.RTD_WEBHOOK_URL }} + RTD_BRANCH: ${{steps.set_rtd_branch.outputs.rtd_branch}} + run: | + if [[ ! -z "${RTD_BRANCH}" ]]; then + curl -X POST -d "branches=${RTD_BRANCH}" -d "token=${RTD_TOKEN}" "${RTD_WEBHOOK_URL}" + fi diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c80571951..6ab8e1c23 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/psf/black - rev: 21.6b0 + rev: 22.1.0 hooks: - id: black language_version: python3 \ No newline at end of file diff --git a/ci/fetch_pysteps_data.py b/ci/fetch_pysteps_data.py index 7172447da..d9db279ed 100644 --- a/ci/fetch_pysteps_data.py +++ b/ci/fetch_pysteps_data.py @@ -15,7 +15,7 @@ tox_test_data_dir = os.environ["PYSTEPS_DATA_PATH"] -download_pysteps_data(tox_test_data_dir, force=True) +download_pysteps_data(tox_test_data_dir, force=True, show_progress=False) create_default_pystepsrc( tox_test_data_dir, config_dir=tox_test_data_dir, file_name="pystepsrc" diff --git a/doc/requirements.txt b/doc/requirements.txt index 05e2450cd..dce04b2f2 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,5 +1,7 @@ -# Additional requeriments related to the documentation build only +# Additional requirements related to the documentation build only sphinx +requests +numpydoc sphinxcontrib.bibtex sphinx-book-theme sphinx_gallery diff --git a/doc/source/conf.py b/doc/source/conf.py index 5ed946c71..b9fca144f 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -1,143 +1,50 @@ -# -*- coding: utf-8 -*- - -# All configuration values have a default; values that are commented out -# serve to show the default. - +# Pysteps-docs sphinx configuration import os -import subprocess import sys from datetime import datetime +from pathlib import Path -import json -from jsmin import jsmin - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -if "READTHEDOCS" not in os.environ: - sys.path.insert(1, os.path.abspath("../../")) - -# -- General configuration ------------------------------------------------ +sys.path.insert(0, str(Path(__file__).parent)) +import doc_utils -# If your documentation needs a minimal Sphinx version, state it here. -# -needs_sphinx = "1.6" +# REPO_URL="pySTEPS/pysteps" +REPO_URL = "aperezhortal/pysteps" -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. extensions = [ "sphinx.ext.autodoc", "sphinx.ext.autosummary", "sphinx.ext.coverage", "sphinx.ext.mathjax", - "sphinx.ext.napoleon", + "numpydoc", "sphinxcontrib.bibtex", "sphinx_gallery.gen_gallery", ] bibtex_bibfiles = ["references.bib"] - -# numpydoc_show_class_members = False -# Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] - -# The suffix(es) of source filenames. -# You can specify multiple suffix as a list of string: -# -# source_suffix = ['.rst', '.md'] source_suffix = ".rst" - -# The master toctree document. master_doc = "index" - -# General information about the project. +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "**.ipynb_checkpoints"] +todo_include_todos = False project = "pysteps" copyright = f"2018-{datetime.now():%Y}, pysteps developers" -author = "pysteps developers" - - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -def get_version(): - """Returns project version as string from 'git describe' command.""" - - from subprocess import check_output - - _version = check_output(["git", "describe", "--tags", "--always"]) - - if _version: - return _version.decode("utf-8") - else: - return "X.Y" - - -# The short X.Y version. -version = get_version().lstrip("v").rstrip().split("-")[0] - -# The full version, including alpha/beta/rc tags. -release = version - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# -# This is also used if you do content translation via gettext catalogs. -# Usually you set "language" from the command line for these cases. -language = None - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This patterns also effect to html_static_path and html_extra_path -exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = "sphinx" - -# If true, `todo` and `todoList` produce output, else they produce nothing. -todo_include_todos = False - - -# -- Read the Docs build -------------------------------------------------- - - -def set_root(): - fn = os.path.abspath(os.path.join("..", "..", "pysteps", "pystepsrc")) - with open(fn, "r") as f: - rcparams = json.loads(jsmin(f.read())) - - for key, value in rcparams["data_sources"].items(): - new_path = os.path.join("..", "..", "pysteps-data", value["root_path"]) - new_path = os.path.abspath(new_path) +author = "PySteps developers" - value["root_path"] = new_path +release = doc_utils.current_release() +version = release.lstrip("v").rstrip().split("-")[0] # The short X.Y version. - fn = os.path.abspath(os.path.join("..", "..", "pystepsrc.rtd")) - with open(fn, "w") as f: - json.dump(rcparams, f, indent=4) - - -if "READTHEDOCS" in os.environ: - repourl = "https://github.com/pySTEPS/pysteps-data.git" - dir = os.path.join(os.getcwd(), "..", "..", "pysteps-data") - dir = os.path.abspath(dir) - subprocess.check_call(["rm", "-rf", dir]) - subprocess.check_call(["git", "clone", repourl, dir]) - set_root() - pystepsrc = os.path.abspath(os.path.join("..", "..", "pystepsrc.rtd")) - os.environ["PYSTEPSRC"] = pystepsrc +is_run_in_read_the_docs = "READTHEDOCS" in os.environ +sphinx_gallery_conf = { + "examples_dirs": "../../examples", + "gallery_dirs": "auto_examples", + "filename_pattern": r"/*\.py", + "plot_gallery": not is_run_in_read_the_docs, + "abort_on_example_error": True +} # -- Options for HTML output ---------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -# html_theme = 'alabaster' -# html_theme = 'classic' html_theme = "sphinx_book_theme" html_title = "" - html_context = { "github_user": "pySTEPS", "github_repo": "pysteps", @@ -145,12 +52,8 @@ def set_root(): "doc_path": "doc", } -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -# html_theme_options = { - "repository_url": "https://github.com/pySTEPS/pysteps", + "repository_url": f"https://github.com/{REPO_URL}", "repository_branch": "master", "path_to_docs": "doc/source", "use_edit_page_button": True, @@ -158,108 +61,24 @@ def set_root(): "use_issues_button": True, } -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. html_logo = "../_static/pysteps_logo.png" - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ["../_static"] html_css_files = ["../_static/pysteps.css"] - -# Custom sidebar templates, must be a dictionary that maps document names -# to template names. -# -# This is required for the alabaster theme -# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars -# html_sidebars = { -# "**": [ -# "relations.html", # needs 'show_related': True theme option to display -# "searchbox.html", -# ] -# } - html_domain_indices = True - autosummary_generate = True - -# -- Options for HTMLHelp output ------------------------------------------ - -# Output file base name for HTML help builder. htmlhelp_basename = "pystepsdoc" -# -- Options for LaTeX output --------------------------------------------- - -# This hack is taken from numpy (https://github.com/numpy/numpy/blob/master/doc/source/conf.py). -latex_preamble = r""" - \usepackage{amsmath} - \DeclareUnicodeCharacter{00A0}{\nobreakspace} - - % In the parameters section, place a newline after the Parameters - % header - \usepackage{expdlist} - \let\latexdescription=\description - \def\description{\latexdescription{}{} \breaklabel} - - % Make Examples/etc section headers smaller and more compact - \makeatletter - \titleformat{\paragraph}{\normalsize\py@HeaderFamily}% - {\py@TitleColor}{0em}{\py@TitleColor}{\py@NormalColor} - \titlespacing*{\paragraph}{0pt}{1ex}{0pt} - \makeatother - - % Fix footer/header - \renewcommand{\chaptermark}[1]{\markboth{\MakeUppercase{\thechapter.\ #1}}{}} - \renewcommand{\sectionmark}[1]{\markright{\MakeUppercase{\thesection.\ #1}}} -""" - -latex_elements = { - "papersize": "a4paper", - "pointsize": "10pt", - "preamble": latex_preamble - # Latex figure (float) alignment - # - # 'figure_align': 'htbp', -} - -latex_domain_indices = False - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - (master_doc, "pysteps.tex", "pysteps reference", author, "manual"), -] - -# -- Options for manual page output --------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [(master_doc, "pysteps", "pysteps reference", [author], 1)] -# -- Options for Texinfo output ------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - ( - master_doc, - "pysteps", - "pysteps reference", - author, - "pysteps", - "One line description of project.", - "Miscellaneous", - ), -] - -# -- Options for Sphinx-Gallery ------------------------------------------- - -# The configuration dictionary for Sphinx-Gallery - -sphinx_gallery_conf = { - "examples_dirs": "../../examples", # path to your example scripts - "gallery_dirs": "auto_examples", # path where to save gallery generated examples - "filename_pattern": r"/*\.py", # Include all the files in the examples dir -} +# -- Download artifacts with rendered examples in RTD ---------------------------------- +if is_run_in_read_the_docs: + my_artifact_downloader = doc_utils.ArtifactDownloader( + REPO_URL, prefix="auto_examples-for-" + ) + + DOC_ROOT_DIR = Path(__file__).parent / "../" + token = os.environ.get("RTD_GITHUB_TOKEN") # never print this! + my_artifact_downloader.download_artifact( + token, + DOC_ROOT_DIR / "source/auto_examples", + retry=3, + tar_filename="auto_examples.tar", + ) diff --git a/doc/source/doc_utils.py b/doc/source/doc_utils.py new file mode 100644 index 000000000..c3dbcef37 --- /dev/null +++ b/doc/source/doc_utils.py @@ -0,0 +1,130 @@ +# Adapted from the rtds_action sphinx extension. +# https://github.com/dfm/rtds-action/blob/main/src/rtds_action/rtds_action.py +# MIT License +# Copyright (c) 2021 Pysteps developers +# Copyright (c) 2020 Dan Foreman-Mackey + +import os +import subprocess +import tarfile +import tempfile +from io import BytesIO +from zipfile import ZipFile + +import requests + + +def current_release(): + """Returns current version as string from 'git describe' command.""" + from subprocess import check_output + + _version = check_output(["git", "describe", "--tags", "--always"]) + if _version: + return _version.decode("utf-8") + else: + return "X.Y" + + +class ArtifactDownloader: + """ + Class used to download Github artifacts. + + Notes + ----- + Adapted from the rtds_action sphinx extension. + https://github.com/dfm/rtds-action/blob/main/src/rtds_action/rtds_action.py + """ + + def __init__(self, repo, prefix="auto_examples-for-"): + self.repo = repo + self.prefix = prefix + + @staticmethod + def current_commit_hash(): + try: + cmd = "git rev-parse HEAD" + git_hash = subprocess.check_output(cmd.split(" ")).strip().decode("ascii") + except subprocess.CalledProcessError as e: + raise RuntimeError( + f"Error retrieving git hash. Return code {e.returncode}.\n" + e.stdout + ) + return git_hash + + def available_artifacts(self): + request_result = requests.get( + f"https://api.github.com/repos/{self.repo}/actions/artifacts", + params=dict(per_page=100), + ) + + if request_result.status_code != 200: + raise RuntimeError( + f"Error retrieving artifacts: ({request_result.status_code})" + ) + request_result = request_result.json() + artifacts = request_result.get("artifacts", []) + return artifacts + + def current_hash_artifact(self): + commit_hash = self.current_commit_hash() + artifact_name = f"{self.prefix}{commit_hash}" + + artifacts = [ + artifact + for artifact in self.available_artifacts() + if artifact["name"] == artifact_name + ] + if len(artifacts) == 0: + raise RuntimeError(f"Artifact `{artifact_name}` not found.") + + # Return the first artifact that matches the name. Ignore the rest. + return artifacts[0] + + def download_artifact( + self, token, dest_dir, retry=3, tar_filename="auto_examples.tar" + ): + # --- IMPORTANT ---------------------------------------------------------------- + # + # This function uses a GitHub access token to download the artifact. + # + # To avoid leaking credentials: + # - NEVER hardcode the token + # - ALWAYS store the token value in environment variables as GitHub secrets + # or Read the docs environmental variables. + # - NEVER print or log the token value for debugging + # - NEVER print or log the request header since it contains the token. + # + # For testing: + # - Generate a temporary token that expires the next day. + # - Remove the temporary token from GitHub once the testing finished. + # ------------------------------------------------------------------------------ + dest_dir = str(dest_dir) + artifact = self.current_hash_artifact() + for n in range(retry): + # Access token needed to download artifacts + # https://github.com/actions/upload-artifact/issues/51 + request_result = requests.get( + artifact["archive_download_url"], + headers={"Authorization": f"token {token}"}, + ) + + if request_result.status_code != 200: + print( + f"Can't download artifact ({request_result.status_code})." + f"Retry {n + 1}/{retry}" + ) + continue + + with tempfile.TemporaryDirectory() as tmp_dir: + local_artifact_dir = os.path.join(tmp_dir, "artifact") + with ZipFile(BytesIO(request_result.content)) as f: + f.extractall(path=local_artifact_dir) + + _tar_filename = os.path.join(local_artifact_dir, tar_filename) + if not os.path.isfile(_tar_filename): + raise RuntimeError(f"{tar_filename} file not found in artifact.") + + my_tar = tarfile.open(_tar_filename) + my_tar.extractall(dest_dir) + return + + raise RuntimeError(f"Can't download artifact after {retry} retries.") diff --git a/examples/thunderstorm_detection_and_tracking.py b/examples/thunderstorm_detection_and_tracking.py index aa27ef75f..c0e793fed 100644 --- a/examples/thunderstorm_detection_and_tracking.py +++ b/examples/thunderstorm_detection_and_tracking.py @@ -1,8 +1,7 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- +#!/bin/env python """ Thunderstorm Detection and Tracking - T-DaTing -============================================ +============================================== This example shows how to use the thunderstorm DaTing module. The example is based on MeteoSwiss radar data and uses the Cartesian composite of maximum reflectivity on a diff --git a/pysteps/datasets.py b/pysteps/datasets.py index c75bdf02c..b1dfa4d8d 100644 --- a/pysteps/datasets.py +++ b/pysteps/datasets.py @@ -285,7 +285,7 @@ def delay(_counter): print(err) -def download_pysteps_data(dir_path, force=True): +def download_pysteps_data(dir_path, force=True, show_progress=True): """ Download pysteps data from github. @@ -298,6 +298,8 @@ def download_pysteps_data(dir_path, force=True): exception if raised. If force=True, the data will we downloaded in the destination directory and may override existing files. + show_progess: bool + If true, show the download progress. """ # Check if directory exists but is not empty @@ -317,9 +319,14 @@ def download_pysteps_data(dir_path, force=True): # the content is dynamically generated and we can't know the length a priori easily. pbar = ShowProgress() print("Downloading pysteps-data from github.") + + if show_progress: + reporthook = pbar + else: + reporthook = None tmp_file_name, _ = request.urlretrieve( "https://github.com/pySTEPS/pysteps-data/archive/master.zip", - reporthook=pbar, + reporthook=reporthook, ) pbar.end(message="Download complete\n") diff --git a/pysteps/verification/spatialscores.py b/pysteps/verification/spatialscores.py index afce35ee4..243ee9f21 100644 --- a/pysteps/verification/spatialscores.py +++ b/pysteps/verification/spatialscores.py @@ -368,7 +368,7 @@ def binary_mse_accum(bmse, X_f, X_o): """Accumulate forecast-observation pairs to an BMSE object. Parameters - ----------- + ---------- bmse: dict The BMSE object initialized with :py:func:`pysteps.verification.spatialscores.binary_mse_init`. @@ -577,7 +577,7 @@ def fss_accum(fss, X_f, X_o): """Accumulate forecast-observation pairs to an FSS object. Parameters - ----------- + ---------- fss: dict The FSS object initialized with :py:func:`pysteps.verification.spatialscores.fss_init`.