From 99fa551474c46f24ff8021e2b4dea90635c74e69 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Tue, 26 Oct 2021 16:59:36 +0100 Subject: [PATCH 01/12] Proposal for fixed generation of benchmark data. --- .cirrus.yml | 9 ++ .gitignore | 1 + benchmarks/benchmarks/ci/esmf_regridder.py | 13 +-- benchmarks/benchmarks/generate_data.py | 108 +++++++++++++++++++++ 4 files changed, 122 insertions(+), 9 deletions(-) create mode 100644 benchmarks/benchmarks/generate_data.py diff --git a/.cirrus.yml b/.cirrus.yml index 8199466f..ffdc0ef6 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -134,6 +134,15 @@ benchmark_task: fingerprint_script: - echo "${CIRRUS_TASK_NAME}" - if [ -n "${IRIS_SOURCE}" ]; then echo "${IRIS_SOURCE}"; fi + nox_cache: + folder: ${CIRRUS_WORKING_DIR}/.nox + reupload_on_changes: true + fingerprint_script: + - echo "${CIRRUS_TASK_NAME}" + - echo "${NOX_CACHE_BUILD}" + - if [ -n "${IRIS_SOURCE}" ]; then echo "${IRIS_SOURCE}"; fi benchmarks_script: - export CONDA_OVERRIDE_LINUX="$(uname -r | cut -d'+' -f1)" + - nox --session=tests --install-only + - export DATA_GEN_PYTHON=$(find .nox -path "*bin/python") - nox --session="benchmarks(ci compare)" diff --git a/.gitignore b/.gitignore index ac0c0ab9..1f8ccf4c 100644 --- a/.gitignore +++ b/.gitignore @@ -51,6 +51,7 @@ coverage.xml .hypothesis/ .pytest_cache/ benchmarks/.asv* +benchmarks/.data # Translations *.mo diff --git a/benchmarks/benchmarks/ci/esmf_regridder.py b/benchmarks/benchmarks/ci/esmf_regridder.py index bbd48a38..9db9c767 100644 --- a/benchmarks/benchmarks/ci/esmf_regridder.py +++ b/benchmarks/benchmarks/ci/esmf_regridder.py @@ -5,12 +5,12 @@ import numpy as np import dask.array as da import iris -from iris.coord_systems import RotatedGeogCS from iris.cube import Cube from esmf_regrid.esmf_regridder import GridInfo from esmf_regrid.schemes import ESMFAreaWeightedRegridder -from esmf_regrid.tests.unit.schemes.test__cube_to_GridInfo import _grid_cube + +from ..generate_data import _grid_cube def _make_small_grid_args(): @@ -65,16 +65,12 @@ def setup(self, type): if type == "large target": n_lons_tgt = 100 n_lats_tgt = 200 - if type == "mixed": - coord_system_src = RotatedGeogCS(0, 90, 90) - else: - coord_system_src = None grid = _grid_cube( n_lons_src, n_lats_src, lon_bounds, lat_bounds, - coord_system=coord_system_src, + alt_coord_system=(type == "mixed"), ) tgt = _grid_cube(n_lons_tgt, n_lats_tgt, lon_bounds, lat_bounds) src_data = np.arange(n_lats_src * n_lons_src * h).reshape( @@ -115,13 +111,12 @@ def setup_cache(self): h = 2000 # Rotated coord systems prevent pickling of the regridder so are # removed for the time being. - # coord_system_src = RotatedGeogCS(0, 90, 90) grid = _grid_cube( n_lons_src, n_lats_src, lon_bounds, lat_bounds, - # coord_system=coord_system_src, + # alt_coord_system=True, ) tgt = _grid_cube(n_lons_tgt, n_lats_tgt, lon_bounds, lat_bounds) diff --git a/benchmarks/benchmarks/generate_data.py b/benchmarks/benchmarks/generate_data.py new file mode 100644 index 00000000..8b246438 --- /dev/null +++ b/benchmarks/benchmarks/generate_data.py @@ -0,0 +1,108 @@ +""" +Scripts for generating supporting data for benchmarking. + +Data generated using iris-esmf-regrid should use +:func:`run_function_elsewhere`, which means that data is generated using a +fixed version of iris-esmf-regrid and a fixed environment, rather than those +that get changed when the benchmarking run checks out a new commit. + +The python executable for :func:`run_function_elsewhere` is provided via the +``DATA_GEN_PYTHON`` environment variable, and should be the path of an +executable within an environment that supports iris-esmf-regrid and has +iris-esmf-regrid installed via ``pip install -e``. + +Downstream use of data generated 'elsewhere' requires saving; usually in a +NetCDF file. Could also use pickling but there is a potential risk if the +benchmark sequence runs over two different Python versions. + +""" +from inspect import getsource +from subprocess import CalledProcessError, check_output, run +from os import environ +from textwrap import dedent + +from iris import load_cube + +PYTHON_EXE = environ.get("DATA_GEN_PYTHON", "") +try: + _ = check_output([PYTHON_EXE, "-c", "a = True"]) +except (CalledProcessError, FileNotFoundError, PermissionError): + error = ( + f"Expected valid python executable path from env variable " + f"DATA_GEN_PYTHON. Got: {PYTHON_EXE}" + ) + raise ValueError(error) + + +def run_function_elsewhere(python_exe, func_to_run, func_call_string): + """ + Run a given function using an alternative python executable. + + This structure allows the function to be written natively, with only the + function call needing to be written as a string. + + Uses print (stdout) as the only available 'return'. + + Parameters + ---------- + python_exe : pathlib.Path or str + Path to the alternative python executable. + func_to_run : FunctionType + The function object to be run using the alternative python executable. + func_call_string: str + A string that, when executed, will call the function with the desired arguments. + + """ + func_string = dedent(getsource(func_to_run)) + python_string = "\n".join([func_string, func_call_string]) + result = run([python_exe, "-c", python_string], capture_output=True, check=True) + return result.stdout.decode().strip() + + +def _grid_cube( + n_lons, + n_lats, + lon_outer_bounds, + lat_outer_bounds, + circular=False, + alt_coord_system=False, +): + def func(*args, **kwargs): + """ + Prep and run _grid_args, saving to a NetCDF file. + + Saving to a file allows the original python executable to pick back up. + + Remember that all arguments must work as strings, hence the fresh + construction of a ``coord_system`` within the function. + + """ + from pathlib import Path + + from iris import save + from iris.coord_systems import RotatedGeogCS + + from esmf_regrid.tests.unit.schemes.test__cube_to_GridInfo import _grid_cube + + save_dir = (Path().parent.parent / ".data").resolve() + save_dir.mkdir(exist_ok=True) + # TODO: caching? Currently written assuming overwrite every time. + save_path = save_dir / "_grid_cube.nc" + + if kwargs.pop("alt_coord_system"): + kwargs["coord_system"] = RotatedGeogCS(0, 90, 90) + + cube = _grid_cube(*args, **kwargs) + save(cube, str(save_path)) + # Print the path for downstream use - is returned by run_function_elsewhere(). + print(save_path) + + call_string = ( + "func(" + f"{n_lons}, {n_lats}, {lon_outer_bounds}, {lat_outer_bounds}, " + f"{circular}, alt_coord_system={alt_coord_system}" + ")" + ) + cube_file = run_function_elsewhere(PYTHON_EXE, func, call_string) + return_cube = load_cube(cube_file) + return return_cube From 50daddc15ca36738c88ec25af33360d87f1ab4f7 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Tue, 26 Oct 2021 17:06:34 +0100 Subject: [PATCH 02/12] Docstring improvements. --- benchmarks/benchmarks/generate_data.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/benchmarks/benchmarks/generate_data.py b/benchmarks/benchmarks/generate_data.py index 8b246438..b3af1252 100644 --- a/benchmarks/benchmarks/generate_data.py +++ b/benchmarks/benchmarks/generate_data.py @@ -67,9 +67,11 @@ def _grid_cube( circular=False, alt_coord_system=False, ): + """Wrapper for calling _grid_cube using an alternative python executable.""" + def func(*args, **kwargs): """ - Prep and run _grid_args, saving to a NetCDF file. + Prep and run _grid_cube, saving to a NetCDF file. Saving to a file allows the original python executable to pick back up. From d3bed8f23850ba4cbf2d9a506be4da0ec2f55e15 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Tue, 26 Oct 2021 17:08:20 +0100 Subject: [PATCH 03/12] Cirrus ASV PY_VER 3.8. --- .cirrus.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.cirrus.yml b/.cirrus.yml index ffdc0ef6..94ee4070 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -115,6 +115,8 @@ test_task: benchmark_task: only_if: ${SKIP_BENCHMARK_TASK} == "" auto_cancellation: true + env: + PY_VER: "3.8" name: "${CIRRUS_OS}: performance benchmarking" # Custom clone behaviour to enable ASV to access the PR base branch (if on a # PR). From 2e78be1597f3d9a0a43f07eeda192734e0ba02d4 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Tue, 26 Oct 2021 17:17:38 +0100 Subject: [PATCH 04/12] Better benchmark python versioning. --- .cirrus.yml | 2 +- noxfile.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 94ee4070..0d70fc17 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -145,6 +145,6 @@ benchmark_task: - if [ -n "${IRIS_SOURCE}" ]; then echo "${IRIS_SOURCE}"; fi benchmarks_script: - export CONDA_OVERRIDE_LINUX="$(uname -r | cut -d'+' -f1)" - - nox --session=tests --install-only + - nox --session=tests --python --install-only - export DATA_GEN_PYTHON=$(find .nox -path "*bin/python") - nox --session="benchmarks(ci compare)" diff --git a/noxfile.py b/noxfile.py index c45646e6..6cbc01d6 100644 --- a/noxfile.py +++ b/noxfile.py @@ -336,7 +336,7 @@ def tests(session: nox.sessions.Session): session.run("pytest") -@nox.session +@nox.session(python=PY_VER, venv_backend="conda") @nox.parametrize( ["ci_mode", "gh_pages"], [(True, False), (False, False), (False, True)], @@ -372,7 +372,7 @@ def asv_exec(*sub_args: str) -> None: help_output = session.run(*run_args, "--help", silent=True) if "--python" in help_output: # Not all asv commands accept the --python kwarg. - run_args.append(f"--python={PY_VER[-1]}") + run_args.append(f"--python={session.python}") session.run(*run_args) if ci_mode: From f9babb151ac59c963afbdc9f3e2934d9c87569c7 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Tue, 26 Oct 2021 17:26:36 +0100 Subject: [PATCH 05/12] DATA_GEN_PYTHON realpath. --- .cirrus.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cirrus.yml b/.cirrus.yml index 0d70fc17..991688a3 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -146,5 +146,5 @@ benchmark_task: benchmarks_script: - export CONDA_OVERRIDE_LINUX="$(uname -r | cut -d'+' -f1)" - nox --session=tests --python --install-only - - export DATA_GEN_PYTHON=$(find .nox -path "*bin/python") + - export DATA_GEN_PYTHON=$(realpath $(find .nox -path "*bin/python")) - nox --session="benchmarks(ci compare)" From 1759248450f0ffa0345041df376cdc63f941a05b Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Wed, 27 Oct 2021 10:08:45 +0100 Subject: [PATCH 06/12] Better DATA_GEN_PYTHON handling. --- benchmarks/benchmarks/generate_data.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/benchmarks/benchmarks/generate_data.py b/benchmarks/benchmarks/generate_data.py index b3af1252..1ef83a6d 100644 --- a/benchmarks/benchmarks/generate_data.py +++ b/benchmarks/benchmarks/generate_data.py @@ -4,12 +4,10 @@ Data generated using iris-esmf-regrid should use :func:`run_function_elsewhere`, which means that data is generated using a fixed version of iris-esmf-regrid and a fixed environment, rather than those -that get changed when the benchmarking run checks out a new commit. - -The python executable for :func:`run_function_elsewhere` is provided via the -``DATA_GEN_PYTHON`` environment variable, and should be the path of an -executable within an environment that supports iris-esmf-regrid and has -iris-esmf-regrid installed via ``pip install -e``. +that get changed when the benchmarking run checks out a new commit. The passed +python executable path in such a case should be within an environment that +supports iris-esmf-regrid and has iris-esmf-regrid installed via +``pip install -e``. See also :const:`DATA_GEN_PYTHON`. Downstream use of data generated 'elsewhere' requires saving; usually in a NetCDF file. Could also use pickling but there is a potential risk if the @@ -23,13 +21,15 @@ from iris import load_cube -PYTHON_EXE = environ.get("DATA_GEN_PYTHON", "") +# Allows the command line to pass in a python executable for use within +# run_function_elsewhere()· +DATA_GEN_PYTHON = environ.get("DATA_GEN_PYTHON", "") try: - _ = check_output([PYTHON_EXE, "-c", "a = True"]) + _ = check_output([DATA_GEN_PYTHON, "-c", "a = True"]) except (CalledProcessError, FileNotFoundError, PermissionError): error = ( f"Expected valid python executable path from env variable " - f"DATA_GEN_PYTHON. Got: {PYTHON_EXE}" + f"DATA_GEN_PYTHON. Got: {DATA_GEN_PYTHON}" ) raise ValueError(error) @@ -105,6 +105,6 @@ def func(*args, **kwargs): f"{circular}, alt_coord_system={alt_coord_system}" ")" ) - cube_file = run_function_elsewhere(PYTHON_EXE, func, call_string) + cube_file = run_function_elsewhere(DATA_GEN_PYTHON, func, call_string) return_cube = load_cube(cube_file) return return_cube From dfe4c596ed488869358ec9fb528cab14d606cf6a Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Wed, 27 Oct 2021 10:14:37 +0100 Subject: [PATCH 07/12] Better DATA_GEN_PYTHON handling. --- .cirrus.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cirrus.yml b/.cirrus.yml index 991688a3..a1adc2ec 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -146,5 +146,5 @@ benchmark_task: benchmarks_script: - export CONDA_OVERRIDE_LINUX="$(uname -r | cut -d'+' -f1)" - nox --session=tests --python --install-only - - export DATA_GEN_PYTHON=$(realpath $(find .nox -path "*bin/python")) + - export DATA_GEN_PYTHON=$(realpath $(find .nox -path "*tests*bin/python")) - nox --session="benchmarks(ci compare)" From 02cb1d2d45a6d3bbe890482189d30dec741ad657 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Wed, 27 Oct 2021 12:32:50 +0100 Subject: [PATCH 08/12] Safer handling of save_path. --- benchmarks/benchmarks/generate_data.py | 33 ++++++++++++++------------ 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/benchmarks/benchmarks/generate_data.py b/benchmarks/benchmarks/generate_data.py index 1ef83a6d..fcbd8072 100644 --- a/benchmarks/benchmarks/generate_data.py +++ b/benchmarks/benchmarks/generate_data.py @@ -17,6 +17,7 @@ from inspect import getsource from subprocess import CalledProcessError, check_output, run from os import environ +from pathlib import Path from textwrap import dedent from iris import load_cube @@ -41,8 +42,6 @@ def run_function_elsewhere(python_exe, func_to_run, func_call_string): This structure allows the function to be written natively, with only the function call needing to be written as a string. - Uses print (stdout) as the only available 'return'. - Parameters ---------- python_exe : pathlib.Path or str @@ -52,11 +51,16 @@ def run_function_elsewhere(python_exe, func_to_run, func_call_string): func_call_string: str A string that, when executed, will call the function with the desired arguments. + Returns + ------- + str + The ``stdout`` from the run. + """ func_string = dedent(getsource(func_to_run)) python_string = "\n".join([func_string, func_call_string]) result = run([python_exe, "-c", python_string], capture_output=True, check=True) - return result.stdout.decode().strip() + return result.stdout def _grid_cube( @@ -79,32 +83,31 @@ def func(*args, **kwargs): construction of a ``coord_system`` within the function. """ - from pathlib import Path - from iris import save from iris.coord_systems import RotatedGeogCS from esmf_regrid.tests.unit.schemes.test__cube_to_GridInfo import _grid_cube - save_dir = (Path().parent.parent / ".data").resolve() - save_dir.mkdir(exist_ok=True) - # TODO: caching? Currently written assuming overwrite every time. - save_path = save_dir / "_grid_cube.nc" + save_path = kwargs.pop("save_path") if kwargs.pop("alt_coord_system"): kwargs["coord_system"] = RotatedGeogCS(0, 90, 90) cube = _grid_cube(*args, **kwargs) - save(cube, str(save_path)) - # Print the path for downstream use - is returned by run_function_elsewhere(). - print(save_path) + save(cube, save_path) + + save_dir = (Path(__file__).parent.parent / ".data").resolve() + save_dir.mkdir(exist_ok=True) + # TODO: caching? Currently written assuming overwrite every time. + save_path = save_dir / "_grid_cube.nc" call_string = ( "func(" f"{n_lons}, {n_lats}, {lon_outer_bounds}, {lat_outer_bounds}, " - f"{circular}, alt_coord_system={alt_coord_system}" + f"{circular}, alt_coord_system={alt_coord_system}, " + f"save_path='{save_path}'" ")" ) - cube_file = run_function_elsewhere(DATA_GEN_PYTHON, func, call_string) - return_cube = load_cube(cube_file) + _ = run_function_elsewhere(DATA_GEN_PYTHON, func, call_string) + return_cube = load_cube(str(save_path)) return return_cube From f3dec96009adfd0b84dc4e1b3958c30f2b3b6b82 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Fri, 29 Oct 2021 11:03:57 +0100 Subject: [PATCH 09/12] Better naming. --- benchmarks/benchmarks/generate_data.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/benchmarks/benchmarks/generate_data.py b/benchmarks/benchmarks/generate_data.py index fcbd8072..db88294b 100644 --- a/benchmarks/benchmarks/generate_data.py +++ b/benchmarks/benchmarks/generate_data.py @@ -73,9 +73,9 @@ def _grid_cube( ): """Wrapper for calling _grid_cube using an alternative python executable.""" - def func(*args, **kwargs): + def external(*args, **kwargs): """ - Prep and run _grid_cube, saving to a NetCDF file. + Prep and call _grid_cube, saving to a NetCDF file. Saving to a file allows the original python executable to pick back up. @@ -86,14 +86,16 @@ def func(*args, **kwargs): from iris import save from iris.coord_systems import RotatedGeogCS - from esmf_regrid.tests.unit.schemes.test__cube_to_GridInfo import _grid_cube + from esmf_regrid.tests.unit.schemes.test__cube_to_GridInfo import ( + _grid_cube as original, + ) save_path = kwargs.pop("save_path") if kwargs.pop("alt_coord_system"): kwargs["coord_system"] = RotatedGeogCS(0, 90, 90) - cube = _grid_cube(*args, **kwargs) + cube = original(*args, **kwargs) save(cube, save_path) save_dir = (Path(__file__).parent.parent / ".data").resolve() @@ -101,13 +103,13 @@ def func(*args, **kwargs): # TODO: caching? Currently written assuming overwrite every time. save_path = save_dir / "_grid_cube.nc" - call_string = ( + external_call = ( "func(" f"{n_lons}, {n_lats}, {lon_outer_bounds}, {lat_outer_bounds}, " f"{circular}, alt_coord_system={alt_coord_system}, " f"save_path='{save_path}'" ")" ) - _ = run_function_elsewhere(DATA_GEN_PYTHON, func, call_string) + _ = run_function_elsewhere(DATA_GEN_PYTHON, external, external_call) return_cube = load_cube(str(save_path)) return return_cube From 906623c58d543427357aec48b7ca2926b2a42fa7 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Tue, 2 Nov 2021 17:21:29 +0000 Subject: [PATCH 10/12] Improvements to run_function_elsewhere(). --- benchmarks/benchmarks/generate_data.py | 60 +++++++++++++++----------- 1 file changed, 36 insertions(+), 24 deletions(-) diff --git a/benchmarks/benchmarks/generate_data.py b/benchmarks/benchmarks/generate_data.py index db88294b..eed50f53 100644 --- a/benchmarks/benchmarks/generate_data.py +++ b/benchmarks/benchmarks/generate_data.py @@ -4,10 +4,7 @@ Data generated using iris-esmf-regrid should use :func:`run_function_elsewhere`, which means that data is generated using a fixed version of iris-esmf-regrid and a fixed environment, rather than those -that get changed when the benchmarking run checks out a new commit. The passed -python executable path in such a case should be within an environment that -supports iris-esmf-regrid and has iris-esmf-regrid installed via -``pip install -e``. See also :const:`DATA_GEN_PYTHON`. +that get changed when the benchmarking run checks out a new commit. Downstream use of data generated 'elsewhere' requires saving; usually in a NetCDF file. Could also use pickling but there is a potential risk if the @@ -22,8 +19,10 @@ from iris import load_cube -# Allows the command line to pass in a python executable for use within -# run_function_elsewhere()· +#: Python executable used by :func:`run_function_elsewhere`, set via env +#: variable of same name. Must be path of Python within an environment that +#: supports iris-esmf-regrid and has iris-esmf-regrid installed via +#: ``pip install -e``. DATA_GEN_PYTHON = environ.get("DATA_GEN_PYTHON", "") try: _ = check_output([DATA_GEN_PYTHON, "-c", "a = True"]) @@ -35,21 +34,25 @@ raise ValueError(error) -def run_function_elsewhere(python_exe, func_to_run, func_call_string): +def run_function_elsewhere(func_to_run, *args, **kwargs): """ - Run a given function using an alternative python executable. + Run a given function using the :const:`DATA_GEN_PYTHON` executable. - This structure allows the function to be written natively, with only the - function call needing to be written as a string. + This structure allows the function to be written natively. Parameters ---------- - python_exe : pathlib.Path or str - Path to the alternative python executable. func_to_run : FunctionType - The function object to be run using the alternative python executable. - func_call_string: str - A string that, when executed, will call the function with the desired arguments. + The function object to be run. + NOTE: the function must be completely self-contained, i.e. perform all + its own imports (within the target :const:`DATA_GEN_PYTHON` + environment). + *args : tuple, optional + Function call arguments. Must all be expressible as simple literals, + i.e. the ``repr`` must be a valid literal expression. + **kwargs: dict, optional + Function call keyword arguments. All values must be expressible as + simple literals (see ``*args``). Returns ------- @@ -58,8 +61,15 @@ def run_function_elsewhere(python_exe, func_to_run, func_call_string): """ func_string = dedent(getsource(func_to_run)) + func_call_term_strings = [repr(arg) for arg in args] + func_call_term_strings += [f"{name}={repr(val)}" for name, val in kwargs.items()] + func_call_string = ( + f"{func_to_run.__name__}(" + ",".join(func_call_term_strings) + ")" + ) python_string = "\n".join([func_string, func_call_string]) - result = run([python_exe, "-c", python_string], capture_output=True, check=True) + result = run( + [DATA_GEN_PYTHON, "-c", python_string], capture_output=True, check=True + ) return result.stdout @@ -71,7 +81,7 @@ def _grid_cube( circular=False, alt_coord_system=False, ): - """Wrapper for calling _grid_cube using an alternative python executable.""" + """Wrapper for calling _grid_cube via :func:`run_function_elsewhere`.""" def external(*args, **kwargs): """ @@ -103,13 +113,15 @@ def external(*args, **kwargs): # TODO: caching? Currently written assuming overwrite every time. save_path = save_dir / "_grid_cube.nc" - external_call = ( - "func(" - f"{n_lons}, {n_lats}, {lon_outer_bounds}, {lat_outer_bounds}, " - f"{circular}, alt_coord_system={alt_coord_system}, " - f"save_path='{save_path}'" - ")" + _ = run_function_elsewhere( + external, + n_lons, + n_lats, + lon_outer_bounds, + lat_outer_bounds, + circular, + alt_coord_system=alt_coord_system, + save_path=str(save_path), ) - _ = run_function_elsewhere(DATA_GEN_PYTHON, external, external_call) return_cube = load_cube(str(save_path)) return return_cube From b9de1fcc580e2df822231ca637513bd9da839ca5 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Mon, 8 Nov 2021 10:56:22 +0000 Subject: [PATCH 11/12] More pythonic checking of DATA_GEN_PYTHON. --- benchmarks/benchmarks/generate_data.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/benchmarks/benchmarks/generate_data.py b/benchmarks/benchmarks/generate_data.py index eed50f53..b0700a3a 100644 --- a/benchmarks/benchmarks/generate_data.py +++ b/benchmarks/benchmarks/generate_data.py @@ -23,14 +23,14 @@ #: variable of same name. Must be path of Python within an environment that #: supports iris-esmf-regrid and has iris-esmf-regrid installed via #: ``pip install -e``. -DATA_GEN_PYTHON = environ.get("DATA_GEN_PYTHON", "") try: + DATA_GEN_PYTHON = environ["DATA_GEN_PYTHON"] _ = check_output([DATA_GEN_PYTHON, "-c", "a = True"]) +except KeyError: + error = "Env variable DATA_GEN_PYTHON not defined." + raise KeyError(error) except (CalledProcessError, FileNotFoundError, PermissionError): - error = ( - f"Expected valid python executable path from env variable " - f"DATA_GEN_PYTHON. Got: {DATA_GEN_PYTHON}" - ) + error = "Env variable DATA_GEN_PYTHON not a runnable python executable path." raise ValueError(error) From 5340ce67f126d66f44d71acf9dc1024def210765 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Mon, 8 Nov 2021 11:23:42 +0000 Subject: [PATCH 12/12] Remove superfluous --python from Cirrus Nox run. --- .cirrus.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cirrus.yml b/.cirrus.yml index a1adc2ec..969c465d 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -145,6 +145,6 @@ benchmark_task: - if [ -n "${IRIS_SOURCE}" ]; then echo "${IRIS_SOURCE}"; fi benchmarks_script: - export CONDA_OVERRIDE_LINUX="$(uname -r | cut -d'+' -f1)" - - nox --session=tests --python --install-only + - nox --session=tests --install-only - export DATA_GEN_PYTHON=$(realpath $(find .nox -path "*tests*bin/python")) - nox --session="benchmarks(ci compare)"