Merge branch origin/main into this branch and resolve conflicts

ESMValGroup · Jun 9, 2021 · 3ac43b5 · 3ac43b5
1 parent 99159ac
commit 3ac43b5
Show file tree

Hide file tree

Showing 137 changed files with 5,427 additions and 1,688 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -8,8 +8,8 @@ commands:
   check_changes:
     steps:
       - run: |
-          if (test "$CIRCLE_BRANCH" = master ||
-            git --no-pager diff --name-only origin/master... |
+          if (test "$CIRCLE_BRANCH" = main ||
+            git --no-pager diff --name-only origin/main... |
             grep -q -E -f .circleci/install_triggers)
           then
             echo Running installation tests
@@ -33,7 +33,8 @@ jobs:
             . /opt/conda/etc/profile.d/conda.sh
             conda activate esmvaltool
             pip install .[test]
-            pytest -n 2 -m "not installation"
+            pytest -n 2 -m "not installation and not sequential"
+            pytest -n 0 -m "sequential"
       - save_cache:
           key: test-{{ .Branch }}
           paths:
@@ -62,19 +63,25 @@ jobs:
             . /opt/conda/etc/profile.d/conda.sh
             set -x
             mkdir /logs
+            # Add additional requirements for running all tests
+            echo "
+              - r-base
+              - r-yaml
+              - ncl
+            " >> environment.yml
             # Install
-            # conda update -y conda > /logs/conda.txt 2>&1
-            conda env update >> /logs/conda.txt 2>&1
+            conda env create >> /logs/conda.txt 2>&1
             set +x; conda activate esmvaltool; set -x
-            conda install -yS r-base r-yaml ncl -c conda-forge
             pip install .[test] > /logs/install.txt 2>&1
             # Log versions
             dpkg -l > /logs/versions.txt
             conda env export > /logs/environment.yml
             pip freeze > /logs/requirements.txt
             # Test installation
-            pytest -n 2
+            pytest -n 2 -m "not sequential"
+            pytest -n 0 -m "sequential"
             esmvaltool version
+          no_output_timeout: 30m
       - save_cache:
           key: install-{{ .Branch }}
           paths:
@@ -127,20 +134,25 @@ jobs:
           command: |
             . /opt/conda/etc/profile.d/conda.sh
             mkdir /logs
+            # Add additional requirements for running all tests
+            echo "
+              - r-base
+              - r-yaml
+              - ncl
+            " >> environment.yml
             # Install
-            # conda update -y conda > /logs/conda.txt 2>&1
-            conda env update >> /logs/conda.txt 2>&1
+            conda env create >> /logs/conda.txt 2>&1
             conda activate esmvaltool
             pip install -e .[develop] > /logs/install.txt 2>&1
-            # install additional requirements for running all tests
-            conda install -yS r-base r-yaml ncl -c conda-forge
             # Log versions
             dpkg -l > /logs/versions.txt
             conda env export > /logs/environment.yml
             pip freeze > /logs/requirements.txt
             # Test installation
             esmvaltool version
-            pytest -n 2
+            pytest -n 2 -m "not sequential"
+            pytest -n 0 -m "sequential"
+          no_output_timeout: 30m
       - store_artifacts:
           path: /logs
 
@@ -164,7 +176,8 @@ jobs:
             dpkg -l > /logs/versions.txt
             conda env export -n base > /logs/build_environment.yml
             # Build conda package
-            conda build package -c conda-forge -c esmvalgroup > /logs/build_log.txt
+            conda build package -c conda-forge > /logs/build_log.txt
+          no_output_timeout: 60m
       - store_artifacts:
           path: /logs
 
@@ -206,7 +219,7 @@ workflows:
           filters:
             branches:
               only:
-                - master
+                - main
     jobs:
       - test
       - install

diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
@@ -1,45 +1,54 @@
 <!--
     Thank you for contributing to our project!
+
+    Please do not delete this text completely, but read the text below and keep
+    items that seem relevant. If in doubt, just keep everything and add your
+    own text at the top, a reviewer will update the checklist for you.
+
 -->
 
 ## Description
 
 <!--
-    Please describe your changes here, especially focusing on why this PR makes
-    ESMValCore better and what problem it solves.
+    Please describe your changes here, especially focusing on why this pull
+    request makes ESMValCore better and what problem it solves.
 
-    Before you start, please read our [contribution guidelines](https://docs.esmvaltool.org/projects/ESMValCore/en/latest/contributing.html).
+    Before you start, please read our contribution guidelines: https://docs.esmvaltool.org/projects/ESMValCore/en/latest/contributing.html
 
-    Please fill in the GitHub issue that is closed by this pull request, e.g. Closes #1903
+    Please fill in the GitHub issue that is closed by this pull request,
+    e.g. Closes #1903
 -->
 
--   Closes #issue_number
--   Link to documentation:
+Closes #issue_number
+
+Link to documentation:
 
 ***
 
-## Before you get started
+## [Before you get started](https://docs.esmvaltool.org/projects/ESMValCore/en/latest/contributing.html#getting-started)
 
--   [ ] [☝ Create an issue](https://github.com/ESMValGroup/ESMValCore/issues) to discuss what you are going to do
+- [ ] [☝ Create an issue](https://github.com/ESMValGroup/ESMValCore/issues) to discuss what you are going to do
 
-## Checklist
+## [Checklist](https://docs.esmvaltool.org/projects/ESMValCore/en/latest/contributing.html#checklist-for-pull-requests)
 
--   [ ] PR has a descriptive title for the [changelog](https://docs.esmvaltool.org/projects/esmvalcore/en/latest/contributing.html#branches-pull-requests-and-code-review)
--   [ ] Labels are assigned so they can be used in the [changelog](https://docs.esmvaltool.org/projects/esmvalcore/en/latest/contributing.html#branches-pull-requests-and-code-review)
--   [ ] Code follows the [style guide](https://docs.esmvaltool.org/projects/esmvalcore/en/latest/contributing.html#code-style)
--   [ ] [Documentation](https://docs.esmvaltool.org/projects/esmvalcore/en/latest/contributing.html#documentation) is available for new functionality
--   [ ] YAML files pass [`pre-commit`](https://docs.esmvaltool.org/projects/esmvalcore/en/latest/contributing.html#pre-commit) or [`yamllint`](https://docs.esmvaltool.org/projects/esmvalcore/en/latest/community/introduction.html#yaml) checks
--   [ ] [Circle/CI tests pass](https://docs.esmvaltool.org/projects/esmvalcore/en/latest/contributing.html#branches-pull-requests-and-code-review)
--   [ ] [Codacy code quality checks pass](https://docs.esmvaltool.org/projects/esmvalcore/en/latest/contributing.html#branches-pull-requests-and-code-review)
--   [ ] [Documentation builds successfully](https://docs.esmvaltool.org/projects/esmvalcore/en/latest/contributing.html#branches-pull-requests-and-code-review) on readthedocs
--   [ ] [Unit tests](https://docs.esmvaltool.org/projects/esmvalcore/projects/esmvalcore/en/latest/contributing.html#contributing-to-the-esmvalcore-package) are available
+It is the responsibility of the author to make sure the pull request is ready to review. The icons indicate whether the item will be subject to the [🛠 Technical][1] or [🧪 Scientific][2] review.
 
-If you make backwards incompatible changes to the recipe format:
+<!-- The next two lines turn the 🛠 and 🧪 below into hyperlinks -->
+[1]: https://docs.esmvaltool.org/en/latest/community/review.html#technical-review
+[2]: https://docs.esmvaltool.org/en/latest/community/review.html#scientific-review
 
--   [ ] Update [ESMValTool](https://github.com/esmvalgroup/esmvaltool) and link the pull request(s) in the description
+- [ ] [🧪][2] The new functionality is [relevant and scientifically sound](https://docs.esmvaltool.org/projects/ESMValCore/en/latest/contributing.html#scientific-relevance)
+- [ ] [🛠][1] This pull request has a [descriptive title and labels](https://docs.esmvaltool.org/projects/ESMValCore/en/latest/contributing.html#pull-request-title-and-label)
+- [ ] [🛠][1] Code is written according to the [code quality guidelines](https://docs.esmvaltool.org/projects/ESMValCore/en/latest/contributing.html#code-quality)
+- [ ] [🧪][2] and [🛠][1] [Documentation](https://docs.esmvaltool.org/projects/ESMValCore/en/latest/contributing.html#documentation) is available
+- [ ] [🛠][1] [Unit tests](https://docs.esmvaltool.org/projects/ESMValCore/en/latest/contributing.html#tests) have been added
+- [ ] [🛠][1] Changes are [backward compatible](https://docs.esmvaltool.org/projects/ESMValCore/en/latest/contributing.html#backward-compatibility)
+- [ ] [🛠][1] Any changed [dependencies have been added or removed](https://docs.esmvaltool.org/projects/ESMValCore/en/latest/contributing.html#dependencies) correctly
+- [ ] [🛠][1] The [list of authors](https://docs.esmvaltool.org/projects/ESMValCore/en/latest/contributing.html#list-of-authors) is up to date
+- [ ] [🛠][1] All [checks below this pull request](https://docs.esmvaltool.org/projects/ESMValCore/en/latest/contributing.html#pull-request-checks) were successful
 
 ***
 
 To help with the number pull requests:
 
--   🙏 We kindly ask you to [review](https://docs.esmvaltool.org/en/latest/community/review.html#review-of-pull-requests) two other [open pull requests](https://github.com/ESMValGroup/ESMValTool/pulls) in this repository
+- 🙏 We kindly ask you to [review](https://docs.esmvaltool.org/en/latest/community/review.html#review-of-pull-requests) two other [open pull requests](https://github.com/ESMValGroup/ESMValCore/pulls) in this repository
diff --git a/.github/workflows/action-conda-publish.yml b/.github/workflows/action-conda-publish.yml
@@ -6,7 +6,7 @@ on:
   # use this to test before actual release and publish
   push:
     branches:
-      - master
+      - main
 
 jobs:
 
@@ -43,7 +43,7 @@ jobs:
           export BUILD_FOLDER=/tmp/esmvalcore/_build
           mkdir -p $BUILD_FOLDER
           conda build package \
-            --channel esmvalgroup --channel conda-forge \
+            --channel conda-forge \
             --croot $BUILD_FOLDER \
       - name: Push the package to anaconda cloud
         if: startsWith(github.ref, 'refs/tags')
@@ -70,7 +70,7 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
           miniconda-version: "latest"
-          channels: esmvalgroup,conda-forge
+          channels: conda-forge
       - shell: bash -l {0}
         run: conda --version
       - shell: bash -l {0}

diff --git a/.github/workflows/action-install-from-conda.yml b/.github/workflows/action-install-from-conda.yml
@@ -13,20 +13,19 @@
 
 name: Conda Base Install
 
-# runs on a push on master and at the end of every day
+# runs on a push on main and at the end of every day
 on:
   # triggering on push without branch name will run tests everytime
   # there is a push on any branch
   # turn it on only if needed
   push:
     branches:
-    - master
-    - github-actions2
-  # run the test only if the PR is to master
+    - main
+  # run the test only if the PR is to maain
   # turn it on if required
   #pull_request:
   #  branches:
-  #  - master
+  #  - main
   schedule:
     - cron: '0 4 * * *'
 
@@ -46,7 +45,7 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
           miniconda-version: "latest"
-          channels: esmvalgroup,conda-forge
+          channels: conda-forge
       - shell: bash -l {0}
         run: mkdir -p conda_install_linux_artifacts_python_${{ matrix.python-version }}
       - shell: bash -l {0}
@@ -82,7 +81,7 @@ jobs:
           activate-environment: esmvalcore
           python-version: ${{ matrix.python-version }}
           miniconda-version: "latest"
-          channels: esmvalgroup,conda-forge
+          channels: conda-forge
       - shell: bash -l {0}
         run: mkdir -p conda_install_osx_artifacts_python_${{ matrix.python-version }}
       - shell: bash -l {0}

diff --git a/.github/workflows/action-install-from-pypi.yml b/.github/workflows/action-install-from-pypi.yml
@@ -13,21 +13,20 @@
 
 name: PyPi Install
 
-# runs on a push on master and at the end of every day
+# runs on a push on main and at the end of every day
 on:
   # triggering on push without branch name will run tests everytime
   # there is a push on any branch
   # turn it on only if needed
   push:
     branches:
-    - master
-    - github-actions2
+    - main
 
-  # run the test only if the PR is to master
+  # run the test only if the PR is to main
   # turn it on if required
   #pull_request:
   #  branches:
-  #  - master
+  #  - main
   schedule:
     - cron: '0 0 * * *'
 

diff --git a/.github/workflows/action-install-from-source.yml b/.github/workflows/action-install-from-source.yml
@@ -14,20 +14,20 @@
 
 name: Source Install
 
-# runs on a push on master and at the end of every day
+# runs on a push on main and at the end of every day
 on:
   # triggering on push without branch name will run tests everytime
   # there is a push on any branch
   # turn it on only if needed
   push:
     branches:
-    - master
+    - main
     - github-actions2
-  # run the test only if the PR is to master
+  # run the test only if the PR is to main
   # turn it on if required
   #pull_request:
   #  branches:
-  #  - master
+  #  - main
   schedule:
     - cron: '0 0 * * *'
 

diff --git a/.github/workflows/action-pypi-build-and-deploy.yml b/.github/workflows/action-pypi-build-and-deploy.yml
@@ -6,7 +6,7 @@ on:
   # use this for testing
   push:
     branches:
-      - master
+      - main
 
 jobs:
   build-n-publish:

diff --git a/.github/workflows/action-test.yml b/.github/workflows/action-test.yml
@@ -13,19 +13,19 @@
 
 name: Test
 
-# runs on a push on master and at the end of every day
+# runs on a push on main and at the end of every day
 on:
   # triggering on push without branch name will run tests everytime
   # there is a push on any branch
   # turn it on only if needed
   push:
     branches:
-    - master
-  # run the test only if the PR is to master
+    - main
+  # run the test only if the PR is to main
   # turn it on if required
   #pull_request:
   #  branches:
-  #  - master
+  #  - main
   schedule:
     - cron: '0 0 * * *'  # nightly
 
@@ -55,7 +55,9 @@ jobs:
       - shell: bash -l {0}
         run: pip install -e .[develop] 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/install.txt
       - shell: bash -l {0}
-        run: pytest -n 2 -m "not installation" 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/test_report.txt
+        run: pytest -n 2 -m "not installation and not sequential" 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/test_report.txt
+      - shell: bash -l {0}
+        run: pytest -n 0 -m "sequential"
       - name: Upload artifacts
         if: ${{ always() }}  # upload artifacts even if fail
         uses: actions/upload-artifact@v2
@@ -88,7 +90,9 @@ jobs:
       - shell: bash -l {0}
         run: pip install -e .[develop] --use-feature=2020-resolver 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/install.txt
       - shell: bash -l {0}
-        run: pytest -n 2 -m "not installation" 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/test_report.txt
+        run: pytest -n 2 -m "not installation and not sequential" 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/test_report.txt
+      - shell: bash -l {0}
+        run: pytest -n 0 -m "sequential"
       - name: Upload artifacts
         if: ${{ always() }}  # upload artifacts even if fail
         uses: actions/upload-artifact@v2

diff --git a/.gitignore b/.gitignore
@@ -74,6 +74,7 @@ nosetests.xml
 coverage.xml
 *.cover
 .hypothesis/
+.mypy_cache
 
 # Jupyter Notebook
 .ipynb_checkpoints

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -42,3 +42,7 @@ repos:
     rev: '3.8.4'
     hooks:
       - id: flake8
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: 'v0.812'
+    hooks:
+      - id: mypy
diff --git a/.prospector.yml b/.prospector.yml
@@ -15,6 +15,9 @@ pyroma:
 pep8:
     full: true
 
+mypy:
+  run: true
+
 pep257:
     # disable rules that are allowed by the numpy convention
     # see https://github.com/PyCQA/pydocstyle/blob/master/src/pydocstyle/violations.py

diff --git a/.zenodo.json b/.zenodo.json
@@ -152,6 +152,11 @@
             "affiliation": "Barcelona Supercomputing Center",
             "name": "Jury, Martin",
             "orcid": "0000-0003-0590-7843"
+        },
+        {
+            "affiliation": "Stéphane Sénési EIRL, Colomiers, France",
+            "name": "Sénési, Stéphane",
+            "orcid": "0000-0003-0892-5967"
         }
     ],
     "description": "ESMValCore: A community tool for pre-processing data from Earth system models in CMIP and running analysis scripts.",
@@ -193,4 +198,5 @@
             "id": "10.13039/501100000780::824084"
         }
     ]
+
 }