Speed up tests on CircleCI and use pytest to run them

.circleci/config.yml

@@ -1,19 +1,22 @@
 ---
-version: 2
+version: 2.1
 
-aliases:
-  check_changes: &check_changes
-    name: Check changes
-    command: |
-      if (test "$CIRCLE_BRANCH" = master ||
-          git --no-pager diff --name-only origin/master... |
-          grep -q -E -f .circleci/install_triggers)
-      then
-        echo Running installation tests
-      else
-        echo Skipping installation tests
-        circleci step halt
-      fi
+orbs:
+  coverage-reporter: codacy/[email protected]
+
+commands:
+  check_changes:
+    steps:
+      - run: |
+          if (test "$CIRCLE_BRANCH" = master ||
+            git --no-pager diff --name-only origin/master... |
+            grep -q -E -f .circleci/install_triggers)
+          then
+            echo Running installation tests
+          else
+            echo Skipping installation tests
+            circleci step halt
+          fi
 
 jobs:
   test:
@@ -26,27 +29,19 @@ jobs:
       - run:
           # Update/Create Conda environment and run tests
           command: |
-            julia esmvaltool/install/Julia/setup.jl
-            Rscript esmvaltool/install/R/setup.R
-            python setup.py test
-          no_output_timeout: 20m
-      - store_artifacts:
-          path: /logs
+            pip install .[test]
+            esmvaltool install Julia
+            esmvaltool install R
+            # Remove source to test installed software
+            rm -r esmvaltool
+            pytest -n 2
       - store_artifacts:
           path: test-reports/
       - store_test_results:
           path: test-reports/
-      - run:
-          when: always
-          command: |
-            if [[ -v CODACY_PROJECT_TOKEN ]]
-            then
-              echo Uploading coverage report
-              pip install codacy-coverage
-              python-codacy-coverage -r test-reports/coverage.xml
-            else
-              echo Not uploading coverage report
-            fi
+      - coverage-reporter/send_report:
+          coverage-reports: 'test-reports/coverage.xml'
+          project-token: $CODACY_PROJECT_TOKEN
 
   test_installation:
     # Test Python 3 installation
@@ -55,7 +50,7 @@ jobs:
       - image: continuumio/miniconda3
     steps:
       - checkout
-      - run: *check_changes
+      - check_changes
       - restore_cache:
           key: test-install-{{ .Branch }}
       - run:
@@ -80,7 +75,7 @@ jobs:
             conda env export > /logs/environment.yml
             pip freeze > /logs/requirements.txt
             # Test installation
-            pytest
+            pytest -n 2
             esmvaltool -h
             ncl -V
             # cdo test, check that it supports hdf5
@@ -108,7 +103,7 @@ jobs:
       - image: continuumio/miniconda3
     steps:
       - checkout
-      - run: *check_changes
+      - check_changes
       - run:
           command: |
             . /opt/conda/etc/profile.d/conda.sh
@@ -129,8 +124,8 @@ jobs:
             conda env export > /logs/environment.yml
             pip freeze > /logs/requirements.txt
             # Test installation
+            pytest -n 2
             esmvaltool -h
-            python setup.py test
             ncl -V
             cdo --version
           no_output_timeout: 20m
@@ -171,7 +166,7 @@ jobs:
       - image: continuumio/miniconda3
     steps:
       - checkout
-      - run: *check_changes
+      - check_changes
       - run:
           command: |
             . /opt/conda/etc/profile.d/conda.sh
@@ -220,6 +215,7 @@ jobs:
             esmvaltool -h
             ncl -V
             cdo --version
+          no_output_timeout: 30m
 
   ncl_cdo_test:
     # Test ncl and cdo conda packages
@@ -248,7 +244,6 @@ jobs:
           path: /logs
 
 workflows:
-  version: 2
   commit:
     jobs:
       - test

doc/sphinx/source/community/introduction.rst

@@ -123,7 +123,7 @@ Running tests
 -------------
 
 Go to the directory where the repository is cloned and run
-``python setup.py test``. Tests will also be run automatically by
+``pytest``. Tests will also be run automatically by
 `CircleCI <https://circleci.com/gh/ESMValGroup/ESMValTool>`__.
 
 Code style

package/meta.yaml

@@ -42,10 +42,11 @@ test:
     - pytest-flake8
     - pytest-html!=2.1.0
     - pytest-metadata>=1.5.1
+    - pytest-xdist
   imports:
     - esmvaltool
   commands:
-    - pytest --ignore=run_test.py
+    - pytest -n 2 --ignore=run_test.py
     - cmorize_obs --help
     - nclcodestyle --help
     - esmvaltool colortables --help
@@ -60,7 +61,6 @@ outputs:
     requirements:
       build:
         - git
-        - pytest-runner
         - python>=3.6
         - setuptools_scm
       run:

setup.cfg

@@ -1,6 +1,3 @@
-[aliases]
-test=pytest
-
 [build_sphinx]
 source-dir = doc/sphinx/source
 build-dir = doc/sphinx/build
@@ -16,6 +13,7 @@ addopts =
     --cov-report=xml:test-reports/coverage.xml
     --cov-report=html:test-reports/coverage_html
     --html=test-reports/report.html
+    --numprocesses auto
 env =
     MPLBACKEND = Agg
 flake8-ignore =

setup.py

@@ -17,7 +17,6 @@
 REQUIREMENTS = {
     # Installation script (this file) dependencies
     'setup': [
-        'pytest-runner',
         'setuptools_scm',
     ],
     # Installation dependencies
@@ -57,14 +56,15 @@
         'xlsxwriter',
     ],
     # Test dependencies
-    # Execute 'python setup.py test' to run tests
+    # Execute `pip install .[test]` once and the use `pytest` to run tests
     'test': [
         'pytest>=3.9,!=6.0.0rc1,!=6.0.0',
-        'pytest-cov',
+        'pytest-cov>=2.10.1',
         'pytest-env',
-        'pytest-flake8',
+        'pytest-flake8>=1.0.6',
         'pytest-html!=2.1.0',
         'pytest-metadata>=1.5.1',
+        'pytest-xdist',
     ],
     # Development dependencies
     # Use pip install -e .[develop] to install in development mode