Support test data in benchmark workflows (#4402)

* Support test data in benchmark workflows

* Use existing test data for now to test

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Set OVERRIDE_TEST_DATA_REPOSITORY properly

* Manually set version number to mirror cirrus.yml

* Add licence to top of file

* Specify absolute path of test data

* Changes to regridding to remove unnecessary print statements

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Can't refer to environment variables in env block

* Precommit checks benchmarks too

* Set the env var for data location

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Add some logging to noxfile for now

* Do we have the test data path set right?

* Rearrange imports and fix logging

* More logging because iris.config.TEST_DATA_DIR was None

* More logging and changed mv

* Revert logging changes

* Enable caching

* Update to use new test data

* Trim out last bit of logging

* Update cache test data directory

* Reprompt ci

* Fix cache location

* Bit of logging to reprompt CI

* Remove that logging

* Added type hinting

* Review changes

* Prevent piping hiding benchmark failures

Co-authored-by: Martin Yeo <[email protected]>

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Martin Yeo <[email protected]>

.cirrus.yml

@@ -38,7 +38,7 @@ env:
   # Conda packages to be installed.
   CONDA_CACHE_PACKAGES: "nox pip"
   # Git commit hash for iris test data.
-  IRIS_TEST_DATA_VERSION: "2.4"
+  IRIS_TEST_DATA_VERSION: "2.5"
   # Base directory for the iris-test-data.
   IRIS_TEST_DATA_DIR: ${HOME}/iris-test-data
 

.github/workflows/benchmark.yml

@@ -10,6 +10,13 @@ jobs:
   benchmark:
     runs-on: ubuntu-latest
 
+    env:
+      IRIS_TEST_DATA_LOC_PATH: benchmarks
+      IRIS_TEST_DATA_PATH: benchmarks/iris-test-data
+      IRIS_TEST_DATA_VERSION: "2.5"
+      # Lets us manually bump the cache to rebuild
+      TEST_DATA_CACHE_BUILD: "2"
+
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
       - uses: actions/checkout@v2
@@ -38,9 +45,31 @@ jobs:
           key: ${{ runner.os }}-${{ github.sha }}
           restore-keys: ${{ runner.os }}
 
+      - name: Cache test data directory
+        id: cache-test-data
+        uses: actions/cache@v2
+        with:
+          path: |
+            ${{ env.IRIS_TEST_DATA_PATH }}
+          key:
+            test-data-${{ env.IRIS_TEST_DATA_VERSION }}-${{ env.TEST_DATA_CACHE_BUILD }}
+
+      - name: Fetch the test data
+        if: steps.cache-test-data.outputs.cache-hit != 'true'
+        run: |
+          wget --quiet https://github.com/SciTools/iris-test-data/archive/v${IRIS_TEST_DATA_VERSION}.zip -O iris-test-data.zip
+          unzip -q iris-test-data.zip
+          mkdir --parents ${GITHUB_WORKSPACE}/${IRIS_TEST_DATA_LOC_PATH}
+          mv iris-test-data-${IRIS_TEST_DATA_VERSION} ${GITHUB_WORKSPACE}/${IRIS_TEST_DATA_PATH}
+          
+      - name: Set test data var
+        run: |
+          echo "OVERRIDE_TEST_DATA_REPOSITORY=${GITHUB_WORKSPACE}/${IRIS_TEST_DATA_PATH}/test_data" >> $GITHUB_ENV
+
       - name: Run CI benchmarks
         run: |
           mkdir --parents benchmarks/.asv
+          set -o pipefail
           nox --session="benchmarks(ci compare)" | tee benchmarks/.asv/ci_compare.txt
 
       - name: Archive asv results

.pre-commit-config.yaml

@@ -6,7 +6,8 @@ files: |
         noxfile\.py|
         setup\.py|
         docs\/.+\.py|
-        lib\/.+\.py
+        lib\/.+\.py|
+        benchmarks\/.+\.py
     )
 minimum_pre_commit_version: 1.21.0
 

benchmarks/benchmarks/metadata_manager_factory.py

@@ -9,13 +9,13 @@
 """
 
 from iris.common import (
-    metadata_manager_factory,
     AncillaryVariableMetadata,
     BaseMetadata,
     CellMeasureMetadata,
     CoordMetadata,
     CubeMetadata,
     DimCoordMetadata,
+    metadata_manager_factory,
 )
 
 

benchmarks/benchmarks/mixin.py

@@ -14,7 +14,6 @@
 from iris import coords
 from iris.common.metadata import AncillaryVariableMetadata
 
-
 LONG_NAME = "air temperature"
 STANDARD_NAME = "air_temperature"
 VAR_NAME = "air_temp"

benchmarks/benchmarks/regridding.py

@@ -0,0 +1,40 @@
+# Copyright Iris contributors
+#
+# This file is part of Iris and is released under the LGPL license.
+# See COPYING and COPYING.LESSER in the root of the repository for full
+# licensing details.
+"""
+Regridding benchmark test
+
+"""
+
+# import iris tests first so that some things can be initialised before
+# importing anything else
+from iris import tests  # isort:skip
+
+import iris
+from iris.analysis import AreaWeighted
+
+
+class HorizontalChunkedRegridding:
+    def setup(self) -> None:
+        # Prepare a cube and a template
+
+        cube_file_path = tests.get_data_path(
+            ["NetCDF", "regrid", "regrid_xyt.nc"]
+        )
+        self.cube = iris.load_cube(cube_file_path)
+
+        template_file_path = tests.get_data_path(
+            ["NetCDF", "regrid", "regrid_template_global_latlon.nc"]
+        )
+        self.template_cube = iris.load_cube(template_file_path)
+
+        # Chunked data makes the regridder run repeatedly
+        self.cube.data = self.cube.lazy_data().rechunk((1, -1, -1))
+
+    def time_regrid_area_w(self) -> None:
+        # Regrid the cube onto the template.
+        out = self.cube.regrid(self.template_cube, AreaWeighted())
+        # Realise the data
+        out.data

benchmarks/nox_asv_plugin.py

@@ -13,12 +13,12 @@
 from shutil import copy2, copytree
 from tempfile import TemporaryDirectory
 
+from asv import util as asv_util
 from asv.config import Config
 from asv.console import log
 from asv.environment import get_env_name
 from asv.plugins.conda import Conda, _find_conda
-from asv.repo import get_repo, Repo
-from asv import util as asv_util
+from asv.repo import Repo, get_repo
 
 
 class NoxConda(Conda):