From eb78c244c15a1397b20ef9182005c2eab0e04def Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 13 Jul 2020 15:59:03 +0200 Subject: [PATCH] Options arg in read_config_user_file now optional (#709) * Remove utils section (#697) Moved the last script from esmvalcore/utils to ESMValTool * Fixed bug in time weights calculation (#695) * Fixed calculation of time weights * Fixed failing FLAKE8 test * Added more test for time weighting and fixed cube dimensions in test * Avoid pytest version that crashes (#707) * Suggested Documentation changes (#690) Update documentation on relative diagnostics paths and preprocessor order. Co-authored-by: Bouwe Andela * Options arg in read_config_user_file now optional * Fix codacy warning Co-authored-by: Bouwe Andela Co-authored-by: Manuel Schlund <32543114+schlunma@users.noreply.github.com> Co-authored-by: Steve Smith Co-authored-by: Bouwe Andela --- doc/gensidebar.py | 1 - doc/quickstart/find_data.rst | 2 +- doc/recipe/overview.rst | 9 +++ doc/recipe/preprocessor.rst | 3 +- doc/utils.rst | 12 --- esmvalcore/_config.py | 4 +- esmvalcore/utils/__init__.py | 0 esmvalcore/utils/draft_release_notes.py | 99 ------------------------- setup.py | 2 +- 9 files changed, 16 insertions(+), 116 deletions(-) delete mode 100644 doc/utils.rst delete mode 100644 esmvalcore/utils/__init__.py delete mode 100644 esmvalcore/utils/draft_release_notes.py diff --git a/doc/gensidebar.py b/doc/gensidebar.py index ac527f8da8..a076f9df59 100644 --- a/doc/gensidebar.py +++ b/doc/gensidebar.py @@ -76,7 +76,6 @@ def _header(project, text): _write("esmvalcore", "Diagnostic script interfaces", "interfaces") _write("esmvalcore", "Development", "develop/index") _write("esmvalcore", "Contributing", "contributing") - _write("esmvalcore", "Utilities", "utils") _write("esmvalcore", "ESMValCore API Reference", "api/esmvalcore") _write("esmvalcore", "Changelog", "changelog") _endl() diff --git a/doc/quickstart/find_data.rst b/doc/quickstart/find_data.rst index cbd1253727..d6769ef64c 100644 --- a/doc/quickstart/find_data.rst +++ b/doc/quickstart/find_data.rst @@ -79,7 +79,7 @@ structures. Explaining ``config-user/drs: CMIP5:`` or ``config-user/drs: CMIP6:`` --------------------------------------------------------------------- -Whreas ESMValTool will **always** use the CMOR standard for file naming (please +Whereas ESMValTool will **always** use the CMOR standard for file naming (please refer above), by setting the ``drs`` parameter the user tells the tool what type of root paths they need the data from, e.g.: diff --git a/doc/recipe/overview.rst b/doc/recipe/overview.rst index 93f3151de0..85a80f5180 100644 --- a/doc/recipe/overview.rst +++ b/doc/recipe/overview.rst @@ -179,6 +179,11 @@ arguments): data, applying CMOR checks and fixes (:ref:`CMOR check and dataset-specific fixes`) and saving the data to disk. +Preprocessor operations will be applied using the default order +as listed in :ref:`preprocessor_functions`. +Preprocessor tasks can be set to run in the order they are listed in the recipe +by adding ``custom_order: true`` to the preprocessor definition. + .. _Diagnostics: Recipe section: ``diagnostics`` @@ -237,6 +242,10 @@ The path to the script provided in the ``script`` option should be either the absolute path to the script, or the path relative to the ``esmvaltool/diag_scripts`` directory. +Depending on the installation configuration, you may get an error of +"file does not exist" when the system tries to run the diagnostic script +using relative paths. If this happens, use an absolute path instead. + Ancestor tasks -------------- Some tasks require the result of other tasks to be ready before they can start, diff --git a/doc/recipe/preprocessor.rst b/doc/recipe/preprocessor.rst index dd97ebe360..7f10a90ae7 100644 --- a/doc/recipe/preprocessor.rst +++ b/doc/recipe/preprocessor.rst @@ -59,7 +59,8 @@ Overview The ESMValTool preprocessor can be used to perform a broad range of operations on the input data before diagnostics or metrics are applied. The preprocessor performs these operations in a centralized, documented and efficient way, thus -reducing the data processing load on the diagnostics side. +reducing the data processing load on the diagnostics side. For an overview of +the preprocessor structure see the :ref:`Preprocessors`. Each of the preprocessor operations is written in a dedicated python module and all of them receive and return an Iris `cube diff --git a/doc/utils.rst b/doc/utils.rst deleted file mode 100644 index a27244e6cc..0000000000 --- a/doc/utils.rst +++ /dev/null @@ -1,12 +0,0 @@ -.. _utils: - -Utilities -********* - -This section provides information on small tools that are available in the ``esmvalcore/utils`` directory. - - -draft_release_notes.py -====================== - -Script for drafting release notes based on the titles of the GitHub pull requests that have been merged since the previous release. diff --git a/esmvalcore/_config.py b/esmvalcore/_config.py index 1e16eebe4c..0947f74c6d 100644 --- a/esmvalcore/_config.py +++ b/esmvalcore/_config.py @@ -31,7 +31,7 @@ def find_diagnostics(): DIAGNOSTICS_PATH = find_diagnostics() -def read_config_user_file(config_file, folder_name, options): +def read_config_user_file(config_file, folder_name, options=None): """Read config user file and store settings in a dictionary.""" config_file = os.path.abspath( os.path.expandvars(os.path.expanduser(config_file))) @@ -42,6 +42,8 @@ def read_config_user_file(config_file, folder_name, options): with open(config_file, 'r') as file: cfg = yaml.safe_load(file) + if options is None: + options = dict() for key, value in options.items(): cfg[key] = value diff --git a/esmvalcore/utils/__init__.py b/esmvalcore/utils/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/esmvalcore/utils/draft_release_notes.py b/esmvalcore/utils/draft_release_notes.py deleted file mode 100644 index ad3af567b4..0000000000 --- a/esmvalcore/utils/draft_release_notes.py +++ /dev/null @@ -1,99 +0,0 @@ -"""Draft release notes. - -To use this tool, follow these steps: -1) `pip install pygithub` -2) Create an access token and store it in the file ~/.github_api_key, see: -https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line -3) set PREVIOUS_RELEASE to the date/time of the previous release in code below - -""" -import datetime -from pathlib import Path - -try: - from github import Github -except ImportError: - print("Please `pip install pygithub`") - -try: - GITHUB_API_KEY = Path("~/.github_api_key").expanduser().read_text().strip() -except FileNotFoundError: - print("Please create an access token and store it in the file " - "~/.github_api_key, see:\nhttps://help.github.com/en/github/" - "authenticating-to-github/creating-a-personal-access-token-" - "for-the-command-line") - -from esmvalcore import __version__ - -VERSION = f"v{__version__}" -GITHUB_REPO = "ESMValGroup/ESMValCore" - -TITLES = { - 'bug': 'Bug fixes', - 'enhancement': 'Improvements', -} - - -def draft_notes_since(previous_release_date, labels): - """Draft release notes containing the merged pull requests. - - Arguments - --------- - previous_release_date: datetime.datetime - date of the previous release - labels: list - list of GitHub labels that deserve separate sections - """ - session = Github(GITHUB_API_KEY) - repo = session.get_repo(GITHUB_REPO) - pulls = repo.get_pulls( - state='closed', - sort='updated', - direction='desc', - ) - - lines = {} - for pull in pulls: - if pull.merged: - if pull.closed_at < previous_release_date: - break - pr_labels = {label.name for label in pull.labels} - for label in labels: - if label in pr_labels: - break - else: - label = 'enhancement' - - user = pull.user - username = user.login if user.name is None else user.name - line = ( - f"- {pull.title} (`#{pull.number} " - f"`__) " - f"`{username} `__") - if label not in lines: - lines[label] = [] - lines[label].append((pull.closed_at, line)) - - # Create sections - sections = [ - VERSION, - '-' * len(VERSION), - '', - "This release includes", - ] - for label in sorted(lines): - entries = sorted(lines[label]) # sort by merge time - label = TITLES.get(label, label) - sections.append('\n'.join(['', label, '~' * len(label), ''])) - sections.append('\n'.join(entry for _, entry in entries)) - notes = '\n'.join(sections) - - print(notes) - - -if __name__ == '__main__': - - PREVIOUS_RELEASE = datetime.datetime(2020, 3, 6) - LABELS = ('bug', 'fix for dataset') - - draft_notes_since(PREVIOUS_RELEASE, LABELS) diff --git a/setup.py b/setup.py index f884855685..6e038e7f9b 100755 --- a/setup.py +++ b/setup.py @@ -48,7 +48,7 @@ # Test dependencies # Execute 'python setup.py test' to run tests 'test': [ - 'pytest>=3.9', + 'pytest>=3.9,!=6.0.0rc1', 'pytest-cov', 'pytest-env', 'pytest-flake8',