From 068bd48afa697901650fe8cbd2e47afef4210a84 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Tue, 1 Aug 2023 19:35:26 +0100 Subject: [PATCH] WIP more unifying change. --- lib/iris/cube.py | 2 +- .../integration/test_netcdf__loadsaveattrs.py | 205 +++++++++++++----- 2 files changed, 156 insertions(+), 51 deletions(-) diff --git a/lib/iris/cube.py b/lib/iris/cube.py index d9fd105f771..0d1f531bf9c 100644 --- a/lib/iris/cube.py +++ b/lib/iris/cube.py @@ -1,4 +1,4 @@ - # Copyright Iris contributors +# Copyright Iris contributors # # This file is part of Iris and is released under the LGPL license. # See COPYING and COPYING.LESSER in the root of the repository for full diff --git a/lib/iris/tests/integration/test_netcdf__loadsaveattrs.py b/lib/iris/tests/integration/test_netcdf__loadsaveattrs.py index 3c7b0fb5fae..dd69b508ba8 100644 --- a/lib/iris/tests/integration/test_netcdf__loadsaveattrs.py +++ b/lib/iris/tests/integration/test_netcdf__loadsaveattrs.py @@ -25,8 +25,8 @@ import pytest import iris -from iris.coords import DimCoord import iris.coord_systems +from iris.coords import DimCoord from iris.cube import Cube, CubeAttrsDict import iris.fileformats.netcdf import iris.fileformats.netcdf._thread_safe_nc as threadsafe_nc4 @@ -138,7 +138,7 @@ def create_testcase_files_or_cubes( var_values_file1: Union[None, str, dict] = None, global_value_file2: Optional[str] = None, var_values_file2: Union[None, str, dict] = None, - cubes=False + cubes=False, ): """ Create temporary input netcdf files with specific content. @@ -178,9 +178,9 @@ def make_cubes(var_name, global_value=None, var_values=None): cubes = [] var_values = self._default_vars_and_attrvalues(var_values) for varname, local_value in var_values.items(): - cube = Cube(np.arange(3.), var_name=var_name) + cube = Cube(np.arange(3.0), var_name=var_name) cubes.append(cube) - dimco = DimCoord(np.arange(3.), var_name='x') + dimco = DimCoord(np.arange(3.0), var_name="x") cube.add_dim_coord(dimco, 0) cube.attributes.globals[attr_name] = global_value if local_value is not None: @@ -188,10 +188,10 @@ def make_cubes(var_name, global_value=None, var_values=None): return cubes if cubes: - results = make_cubes('v1', global_value_file1, var_values_file1) + results = make_cubes("v1", global_value_file1, var_values_file1) if global_value_file2 is not None or var_values_file2 is not None: results.extend( - make_cubes('v2', global_value_file2, var_values_file2) + make_cubes("v2", global_value_file2, var_values_file2) ) else: results = [ @@ -204,21 +204,39 @@ def make_cubes(var_name, global_value=None, var_values=None): ) return results - def create_testcase_files(self, *args, **kwargs): - return self.create_testcase_files_or_cubes(*args, cubes=False, **kwargs) + def create_testcase_files(self, *args, **kwargs): + return self.create_testcase_files_or_cubes( + *args, cubes=False, **kwargs + ) def create_testcase_cubes(self, *args, **kwargs): return self.create_testcase_files_or_cubes(*args, cubes=True, **kwargs) - def fetch_results(self, filepath=None, cubes=None, oldstyle_combined=False): - # Unpick the global+local values of an attribute resulting from an operation. + def fetch_results( + self, filepath=None, cubes=None, oldstyle_combined=False + ): + """ + Return testcase results from an output file or cubes in a standardised form. + + Unpick the global+local values of an attribute resulting from an operation. + A file result is always [global_value, *local_values] + A cubes result is [*[global_value, *local_values]] (over different global vals) + + When "oldstyle_combined" simulate the "legacy" result, when each cube had a + single combined attribute dictionary. This enables us to check against former + behaviour (and behaviour of results treated as a single dictionary). + If results are from a *file*, this has no effect. + + """ attr_name = self.attrname if filepath is not None: # Fetch global and local values from a file try: ds = threadsafe_nc4.DatasetWrapper(filepath) global_result = ( - ds.getncattr(attr_name) if attr_name in ds.ncattrs() else None + ds.getncattr(attr_name) + if attr_name in ds.ncattrs() + else None ) # Fetch local attr value from all data variables (except dimcoord vars) local_vars_results = [ @@ -228,7 +246,7 @@ def fetch_results(self, filepath=None, cubes=None, oldstyle_combined=False): var.getncattr(attr_name) if attr_name in var.ncattrs() else None - ) + ), ) for var in ds.variables.values() if var.name not in ds.dimensions @@ -237,14 +255,19 @@ def fetch_results(self, filepath=None, cubes=None, oldstyle_combined=False): ds.close() # This version always returns a single result set [global, local1[, local2]] # Return global, plus locals sorted by varname - local_vars_results = sorted( - local_vars_results, - key=lambda x: x[0] - ) + local_vars_results = sorted(local_vars_results, key=lambda x: x[0]) results = [global_result] + [val for _, val in local_vars_results] else: - # Fetch globals and locals from cubes. assert cubes is not None + # Sort result cubes according to a standard ordering. + cubes = sorted( + cubes, + key=lambda cube: ( + cube.name(), + cube.attributes.locals.get(self.attrname, None), + ), + ) + # Fetch globals and locals from cubes. if oldstyle_combined: # Replace cubes attributes with all-combined dictionaries cubes = [cube.copy() for cube in cubes] @@ -253,20 +276,44 @@ def fetch_results(self, filepath=None, cubes=None, oldstyle_combined=False): cube.attributes.clear() cube.attributes.locals = combined global_values = set( - cube.attributes.globals.get(attr_name, None) - for cube in cubes + cube.attributes.globals.get(attr_name, None) for cube in cubes ) # This way returns *multiple* result 'sets', one for each global value results = [ - [globalval] + [ + [globalval] + + [ cube.attributes.locals.get(attr_name, None) for cube in cubes - if cube.attributes.globals.get(attr_name, None) == globalval + if cube.attributes.globals.get(attr_name, None) + == globalval ] - for globalval in global_values + for globalval in sorted(global_values) ] return results + def check_results(self, results, global_values=None, local_values=None): + """ + Fetch results + check that they are as "expected". + + For cubes results, global values may be plural, and if so the locals are + in a nested list (one for each global value). + For file results, there can be only one global + one set of local values. + """ + # Ensure input matches are global=list(values), locals=list(list(value)) + if not isinstance(global_values, list): + global_values = [global_values] + local_values = [local_values] + # Ensure results are in same form + assert isinstance(results, list) + if not isinstance(results[0], list): + results = [results] + assert len(results) == len(global_values) == len(local_values) + for result, vglobal, vlocals in zip( + results, global_values, local_values + ): + assert result == [vglobal] + vlocals + + class TestRoundtrip(MixinAttrsTesting): """ Test handling of attributes in roundtrip netcdf-iris-netcdf. @@ -357,6 +404,25 @@ def check_roundtrip_results( # assert self.attrname in v.ncattrs() # assert v.getncattr(self.attrname) == value + def check_roundtrip_results_NEWSTYLE(self, expected): + """ + Run checks on the generated output file. + + The counterpart to create_roundtrip_testcase, with similar control arguments. + Check existence (or not) of a global attribute, and a number of local + (variable) attributes. + Values of 'None' mean to check that the relevant global/local attribute does + *not* exist. + """ + # N.B. there is only ever one result-file, but it can contain various variables + # which came from different input files. + results = self.fetch_results(filepath=self.result_filepath) + self.check_results( + results=results, + global_values=expected[0], + local_values=expected[1:], + ) + ####################################################### # Tests on "user-style" attributes. # This means any arbitrary attribute which a user might have added -- i.e. one with @@ -379,6 +445,7 @@ def test_01_userstyle_single_global(self): "myvar": None }, # the variable has no such attribute ) + self.check_roundtrip_results_NEWSTYLE(["single-value", None]) def test_02_userstyle_single_local(self): # Default behaviour for a general local user-attribute. @@ -391,11 +458,12 @@ def test_02_userstyle_single_local(self): global_attr_value="single-value", # local values eclipse the global ones # N.B. the output var has NO such attribute ) + self.check_roundtrip_results_NEWSTYLE(["single-value", None]) def test_03_userstyle_multiple_different(self): # Default behaviour for general user-attributes. # The global attribute is lost because there are local ones. - vars1 = {"f1_v1": "f1v1", "f1_v2": "f2v2"} + vars1 = {"f1_v1": "f1v1", "f1_v2": "f1v2"} vars2 = {"f2_v1": "x1", "f2_v2": "x2"} self.create_roundtrip_testcase( attr_name="random", # A generic "user" attribute with no special handling @@ -415,6 +483,9 @@ def test_03_userstyle_multiple_different(self): global_attr_value=None, # local values eclipse the global ones var_attr_vals=all_vars_and_attrs, ) + self.check_roundtrip_results_NEWSTYLE( + [None, "f1v1", "f1v2", "x1", "x2"] + ) def test_04_userstyle_matching_promoted(self): # matching local user-attributes are "promoted" to a global one. @@ -427,6 +498,7 @@ def test_04_userstyle_matching_promoted(self): global_attr_value="same-value", var_attr_vals={"v1": None, "v2": None}, ) + self.check_roundtrip_results_NEWSTYLE(["same-value", None, None]) def test_05_userstyle_matching_crossfile_promoted(self): # matching user-attributes are promoted, even across input files. @@ -440,6 +512,9 @@ def test_05_userstyle_matching_crossfile_promoted(self): global_attr_value="same-value", var_attr_vals={x: None for x in ("v1", "v2", "f2_v1", "f2_v2")}, ) + self.check_roundtrip_results_NEWSTYLE( + ["same-value", None, None, None, None] + ) def test_06_userstyle_nonmatching_remainlocal(self): # Non-matching user attributes remain 'local' to the individual variables. @@ -452,6 +527,7 @@ def test_06_userstyle_nonmatching_remainlocal(self): global_attr_value=None, # NB it still destroys the global one !! var_attr_vals={"v1": "value-1", "v2": "value-2"}, ) + self.check_roundtrip_results_NEWSTYLE([None, "value-1", "value-2"]) ####################################################### # Tests on "Conventions" attribute. @@ -475,6 +551,7 @@ def test_07_conventions_var_local(self): global_attr_value="CF-1.7", # standard content from Iris save var_attr_vals=None, ) + self.check_roundtrip_results_NEWSTYLE(["CF-1.7", None]) def test_08_conventions_var_both(self): # What happens if 'Conventions' appears as both global + local attribute. @@ -487,6 +564,7 @@ def test_08_conventions_var_both(self): global_attr_value="CF-1.7", # standard content from Iris save var_attr_vals=None, ) + self.check_roundtrip_results_NEWSTYLE(["CF-1.7", None]) ####################################################### # Tests on "global" style attributes @@ -500,6 +578,7 @@ def test_09_globalstyle__global(self, global_attr): global_value_file1=attr_content, ) self.check_roundtrip_results(global_attr_value=attr_content) + self.check_roundtrip_results_NEWSTYLE([attr_content, None]) def test_10_globalstyle__local(self, global_attr): # Strictly, not correct CF, but let's see what it does with it. @@ -511,6 +590,7 @@ def test_10_globalstyle__local(self, global_attr): self.check_roundtrip_results( global_attr_value=attr_content ) # "promoted" + self.check_roundtrip_results_NEWSTYLE([attr_content, None]) def test_11_globalstyle__both(self, global_attr): attr_global = f"Global-{global_attr}" @@ -523,6 +603,7 @@ def test_11_globalstyle__both(self, global_attr): self.check_roundtrip_results( global_attr_value=attr_local # promoted local setting "wins" ) + self.check_roundtrip_results_NEWSTYLE([attr_local, None]) def test_12_globalstyle__multivar_different(self, global_attr): # Multiple *different* local settings are retained, not promoted @@ -540,6 +621,7 @@ def test_12_globalstyle__multivar_different(self, global_attr): global_attr_value=None, var_attr_vals={"v1": attr_1, "v2": attr_2}, ) + self.check_roundtrip_results_NEWSTYLE([None, attr_1, attr_2]) def test_13_globalstyle__multivar_same(self, global_attr): # Multiple *same* local settings are promoted to a common global one @@ -552,6 +634,7 @@ def test_13_globalstyle__multivar_same(self, global_attr): global_attr_value=attrval, var_attr_vals={"v1": None, "v2": None}, ) + self.check_roundtrip_results_NEWSTYLE([attrval, None, None]) def test_14_globalstyle__multifile_different(self, global_attr): # Different global attributes from multiple files are retained as local ones @@ -673,7 +756,7 @@ class TestLoad(MixinAttrsTesting): """ - def _run_load_testcase( + def run_load_testcase( self, attr_name, global_value_file1=None, @@ -690,18 +773,6 @@ def _run_load_testcase( var_values_file2=vars_values_file2, ) - def check_results(self, results_expected): - result_cubes = iris.load(self.input_filepaths) - # Ensure result cube order - result_cubes = sorted( - result_cubes, - key=lambda cube: ( - cube.name(), - cube.attributes.results.locals.get(self.attrname, None) - ) - ) - return self.fetch_results(result_cubes) - def create_load_testcase(self, *args, **kwargs): """ Initialise the testcase from the passed-in controls, configure the input @@ -711,11 +782,20 @@ def create_load_testcase(self, *args, **kwargs): on the instance, from where "self.check_load_results()" can get them. """ - self._run_load_testcase(*args, **kwargs) + self.run_load_testcase(*args, **kwargs) result_cubes = iris.load(self.input_filepaths) result_cubes = sorted(result_cubes, key=lambda cube: cube.name()) return result_cubes + def check_load_results(self, expected, oldstyle_combined=False): + result_cubes = iris.load(self.input_filepaths) + results = self.fetch_results( + cubes=result_cubes, oldstyle_combined=oldstyle_combined + ) + self.check_results( + results, global_values=expected[0], local_values=expected[1:] + ) + ####################################################### # Tests on "user-style" attributes. # This means any arbitrary attribute which a user might have added -- i.e. one with @@ -741,13 +821,40 @@ def test_01_userstyle_single_global(self): # #2 : exact expected result, viewed as newstyle split-attributes assert cube1.attributes == CubeAttrsDict(globals=expected_dict) - assert self.fetch_results(cubes=[cube1, cube2], oldstyle_combined=True) == [ - [None, "single-value", "single-value",] + assert self.fetch_results( + cubes=[cube1, cube2], oldstyle_combined=True + ) == [ + [ + None, + "single-value", + "single-value", + ] ] assert self.fetch_results(cubes=[cube1, cube2]) == [ ["single-value", None, None] ] + self.check_load_results( + oldstyle_combined=True, + expected=[None, "single-value", "single-value"], + ) + self.check_load_results(["single-value", None, None]) + def test_01_userstyle_single_global_NEWSTYLE(self): + self.run_load_testcase( + attr_name="myname", # A generic "user" attribute with no special handling + global_value_file1="single-value", + vars_values_file1={ + "myvar": None, + "myvar2": None, + }, # the variable has no such attribute + ) + # Legacy-equivalent result check (single attributes dict per cube) + self.check_load_results( + [None, "single-value", "single-value"], + oldstyle_combined=True, + ) + # Full result check + self.check_load_results(["single-value", None, None]) def test_02_userstyle_single_local(self): # Default behaviour for a general local user-attribute. @@ -759,12 +866,10 @@ def test_02_userstyle_single_local(self): assert cube1.attributes == {"myname": "single-value"} assert cube2.attributes == {} - assert self.fetch_results(cubes=[cube1, cube2], oldstyle_combined=True) == [ - [None, "single-value", None] - ] - assert self.fetch_results(cubes=[cube1, cube2]) == [ - [None, "single-value", None] - ] + self.check_load_results( + [None, "single-value", None], oldstyle_combined=True + ) + self.check_load_results([None, "single-value", None]) def test_03_userstyle_multiple_different(self): # Default behaviour for differing local user-attributes. @@ -814,7 +919,7 @@ def test_03_userstyle_multiple_different(self): ] assert self.fetch_results(cubes=cubes) == [ ["global_file1", "f1v1", "f1v2"], - ["global_file2", "x1", "x2"] + ["global_file2", "x1", "x2"], ] def test_04_userstyle_multiple_same(self): @@ -1017,16 +1122,16 @@ def create_save_testcase(self, attr_name, value1, value2=None): Create cubes(s) and save to temporary file, then return the global and all variable-local attributes of that name (or None-s) from the file. """ - vars_vals = {'cube_0': value1} + vars_vals = {"cube_0": value1} if value2 is not None: - vars_vals['cube_1'] = value2 + vars_vals["cube_1"] = value2 cubes = self.create_testcase_cubes( attr_name=attr_name, global_value_file1=None, - var_values_file1=vars_vals + var_values_file1=vars_vals, ) # Required for common testfile-naming function. - self.attrname = (attr_name) + self.attrname = attr_name # if value2 is None: # n_cubes = 1 # values = [value1]